diff options
author | olaaun <ola.aunroe@gmail.com> | 2017-07-25 20:37:02 +0800 |
---|---|---|
committer | GitHub <noreply@github.com> | 2017-07-25 20:37:02 +0800 |
commit | a44477bcd943021fb80dda761e485e81b7c929ac (patch) | |
tree | b1a7a40b4cbb31612c3f65261860a84b4a0cc61d | |
parent | 674749e424fe50980479e462b4c1a3fb8b77147a (diff) |
Node verification (#2999)
* Initial node verification commit
77 files changed, 3584 insertions, 0 deletions
diff --git a/node-maintainer/pom.xml b/node-maintainer/pom.xml index 6e54afe2d90..f511f5c2a08 100644 --- a/node-maintainer/pom.xml +++ b/node-maintainer/pom.xml @@ -57,6 +57,16 @@ <artifactId>mockito-core</artifactId> <scope>test</scope> </dependency> + <dependency> + <groupId>com.fasterxml.jackson.core</groupId> + <artifactId>jackson-annotations</artifactId> + <version>2.8.3</version> + </dependency> + <dependency> + <groupId>org.apache.commons</groupId> + <artifactId>commons-exec</artifactId> + <version>1.3</version> + </dependency> </dependencies> <build> diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/CommandExecutor.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/CommandExecutor.java new file mode 100644 index 00000000000..c03e90d298a --- /dev/null +++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/CommandExecutor.java @@ -0,0 +1,44 @@ +package com.yahoo.vespa.hosted.node.verification.commons; + +import org.apache.commons.exec.CommandLine; +import org.apache.commons.exec.DefaultExecutor; +import org.apache.commons.exec.PumpStreamHandler; + +import java.io.BufferedReader; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.StringReader; +import java.util.ArrayList; + +/** + * Created by olaa on 03/07/2017. + * Wrapper for executing terminal commands + */ +public class CommandExecutor { + + public ArrayList<String> executeCommand(String command) throws IOException { + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + ArrayList<String> results = new ArrayList<>(); + writeToOutputStream(outputStream, command); + writeOutputStreamToResults(outputStream, results); + return results; + } + + private void writeToOutputStream(ByteArrayOutputStream outputStream, String command) throws IOException { + CommandLine cmdLine = CommandLine.parse(command); + DefaultExecutor executor = new DefaultExecutor(); + PumpStreamHandler streamHandler = new PumpStreamHandler(outputStream); + executor.setStreamHandler(streamHandler); + executor.execute(cmdLine); + } + + private void writeOutputStreamToResults(ByteArrayOutputStream outputStream, ArrayList<String> results) throws IOException { + String out = outputStream.toString(); + BufferedReader br = new BufferedReader(new StringReader(out)); + String line; + while ((line = br.readLine()) != null) { + results.add(line); + } + } + +}
\ No newline at end of file diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/OutputParser.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/OutputParser.java new file mode 100644 index 00000000000..09060bdebdf --- /dev/null +++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/OutputParser.java @@ -0,0 +1,83 @@ +package com.yahoo.vespa.hosted.node.verification.commons; + +import java.util.ArrayList; +import java.util.regex.Pattern; + +/** + * Created by sgrostad on 17/07/2017. + * Parses terminal command output, and returns results based on ParseInstructions + */ +public class OutputParser { + + public static ArrayList<ParseResult> parseOutput(ParseInstructions parseInstructions, ArrayList<String> commandOutput) { + ArrayList<ParseResult> results = new ArrayList<>(); + int searchElementIndex = parseInstructions.getSearchElementIndex(); + int valueElementIndex = parseInstructions.getValueElementIndex(); + ArrayList<String> searchWords = parseInstructions.getSearchWords(); + for (String line : commandOutput) { + String[] lineSplit = line.trim().split(parseInstructions.getSplitRegex()); + if (lineSplit.length <= Math.max(searchElementIndex, valueElementIndex)) { + continue; + } + String searchWordCandidate = lineSplit[searchElementIndex].trim(); + boolean searchWordCandidateMatch = matchingSearchWord(searchWords, searchWordCandidate); + if (searchWordCandidateMatch) { + String value = lineSplit[valueElementIndex]; + results.add(new ParseResult(searchWordCandidate, value.trim())); + } + } + return results; + } + + public static ParseResult parseSingleOutput(ParseInstructions parseInstructions, ArrayList<String> commandOutput) { + ArrayList<ParseResult> parseResults = parseOutput(parseInstructions, commandOutput); + if (parseResults.size() == 0) { + return new ParseResult("invalid", "invalid"); + } + return parseResults.get(0); + } + + public static ArrayList<ParseResult> parseOutPutWithSkips(ParseInstructions parseInstructions, ArrayList<String> commandOutput) { + ArrayList<ParseResult> results = new ArrayList<>(); + int searchElementIndex = parseInstructions.getSearchElementIndex(); + int valueElementIndex = parseInstructions.getValueElementIndex(); + String skipWord = parseInstructions.getSkipWord(); + ArrayList<String> searchWords = parseInstructions.getSearchWords(); + for (int i = 0; i < commandOutput.size(); i++) { + String line = commandOutput.get(i); + String[] lineSplit = line.trim().split(parseInstructions.getSplitRegex()); + if (lineSplit.length <= Math.max(searchElementIndex, valueElementIndex)) { + continue; + } + if (lineSplit[searchElementIndex].equals(skipWord)) { + i = skipToIndex(i, parseInstructions, commandOutput); + continue; + } + String searchWordCandidate = lineSplit[searchElementIndex]; + boolean searchWordCandidateMatch = matchingSearchWord(searchWords, searchWordCandidate); + if (searchWordCandidateMatch) { + String value = lineSplit[valueElementIndex]; + results.add(new ParseResult(searchWordCandidate, value.trim())); + } + } + return results; + } + + protected static int skipToIndex(int index, ParseInstructions parseInstructions, ArrayList<String> commandOutput) { + String skipUntilKeyword = parseInstructions.getSkipUntilKeyword(); + int returnIndex = commandOutput.size(); + for (int i = index; i < commandOutput.size(); i++) { + String line = commandOutput.get(i); + if (line.equals(skipUntilKeyword)) { + returnIndex = i; + break; + } + } + return returnIndex - 1; + } + + private static boolean matchingSearchWord(ArrayList<String> searchWords, String searchWordCandidate) { + return searchWords.stream().anyMatch(w -> Pattern.compile(w).matcher(searchWordCandidate).matches()); + } + +} diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/ParseInstructions.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/ParseInstructions.java new file mode 100644 index 00000000000..1e8531512f2 --- /dev/null +++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/ParseInstructions.java @@ -0,0 +1,56 @@ +package com.yahoo.vespa.hosted.node.verification.commons; + +import java.util.ArrayList; + +/** + * Created by sgrostad on 17/07/2017. + */ +public class ParseInstructions { + + private final int searchElementIndex; + private final int valueElementIndex; + private final String splitRegex; + private final ArrayList<String> searchWords; + private String skipWord; + private String skipUntilKeyword; + + public ParseInstructions(int searchElementIndex, int returnElementNum, String splitRegex, ArrayList<String> searchWords) { + this.searchElementIndex = searchElementIndex; + this.valueElementIndex = returnElementNum; + this.splitRegex = splitRegex; + this.searchWords = searchWords; + } + + public int getSearchElementIndex() { + return searchElementIndex; + } + + public int getValueElementIndex() { + return valueElementIndex; + } + + public String getSplitRegex() { + return splitRegex; + } + + public ArrayList<String> getSearchWords() { + return searchWords; + } + + public void setSkipWord(String skipWord) { + this.skipWord = skipWord; + } + + public void setSkipUntilKeyword(String skipUntilWord) { + this.skipUntilKeyword = skipUntilWord; + } + + public String getSkipWord() { + return this.skipWord; + } + + public String getSkipUntilKeyword() { + return this.skipUntilKeyword; + } + +} diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/ParseResult.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/ParseResult.java new file mode 100644 index 00000000000..57b42aa5dc9 --- /dev/null +++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/ParseResult.java @@ -0,0 +1,48 @@ +package com.yahoo.vespa.hosted.node.verification.commons; + +import java.util.Objects; + +/** + * Created by sgrostad on 17/07/2017. + */ +public class ParseResult { + + private final String searchWord; + private final String value; + + public ParseResult(String searchWord, String value) { + this.searchWord = searchWord; + this.value = value; + } + + public String getSearchWord() { + return searchWord; + } + + public String getValue() { + return value; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) return true; + if (obj instanceof ParseResult) { + ParseResult parseResult = (ParseResult) obj; + if (this.searchWord.equals(parseResult.getSearchWord()) && this.value.equals(parseResult.getValue())) { + return true; + } + } + return false; + } + + @Override + public int hashCode() { + return Objects.hash(searchWord, value); + } + + @Override + public String toString() { + return "Search word: " + searchWord + ", Value: " + value; + } + +} diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/HardwareVerifier.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/HardwareVerifier.java new file mode 100644 index 00000000000..61d26b322f4 --- /dev/null +++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/HardwareVerifier.java @@ -0,0 +1,47 @@ +package com.yahoo.vespa.hosted.node.verification.hardware; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.yahoo.vespa.hosted.node.verification.commons.CommandExecutor; +import com.yahoo.vespa.hosted.node.verification.hardware.benchmarks.Benchmark; +import com.yahoo.vespa.hosted.node.verification.hardware.benchmarks.CPUBenchmark; +import com.yahoo.vespa.hosted.node.verification.hardware.benchmarks.DiskBenchmark; +import com.yahoo.vespa.hosted.node.verification.hardware.benchmarks.HardwareResults; +import com.yahoo.vespa.hosted.node.verification.hardware.benchmarks.MemoryBenchmark; +import com.yahoo.vespa.hosted.node.verification.hardware.benchmarks.NetBenchmark; +import com.yahoo.vespa.hosted.node.verification.hardware.yamasreport.YamasHardwareReport; + +import java.util.ArrayList; +import java.util.Arrays; + +/** + * Benchmarks different hardware components and creates report + */ +public class HardwareVerifier { + + public static void verifyHardware() { + HardwareResults hardwareResults = new HardwareResults(); + CommandExecutor commandExecutor = new CommandExecutor(); + ArrayList<Benchmark> benchmarks = new ArrayList<>(Arrays.asList( + new DiskBenchmark(hardwareResults, commandExecutor), + new CPUBenchmark(hardwareResults, commandExecutor), + new MemoryBenchmark(hardwareResults, commandExecutor), + new NetBenchmark(hardwareResults, commandExecutor))); + + for (Benchmark benchmark : benchmarks) { + benchmark.doBenchmark(); + } + YamasHardwareReport yamasHardwareReport = new YamasHardwareReport(); + yamasHardwareReport.createFromHardwareResults(hardwareResults); + ObjectMapper om = new ObjectMapper(); + try { + System.out.println(om.writeValueAsString(yamasHardwareReport)); + } catch (Exception e) { + e.printStackTrace(); + } + } + + public static void main(String[] args) { + HardwareVerifier.verifyHardware(); + } + +} diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/README.md b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/README.md new file mode 100644 index 00000000000..2797a6b5a9c --- /dev/null +++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/README.md @@ -0,0 +1,2 @@ +# Hardware Verification +Verification of behaviour and performance of hardware
\ No newline at end of file diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/Benchmark.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/Benchmark.java new file mode 100644 index 00000000000..a76555d79d3 --- /dev/null +++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/Benchmark.java @@ -0,0 +1,10 @@ +package com.yahoo.vespa.hosted.node.verification.hardware.benchmarks; + +/** + * Created by sgrostad on 11/07/2017. + */ +public interface Benchmark { + + void doBenchmark(); + +} diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/CPUBenchmark.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/CPUBenchmark.java new file mode 100644 index 00000000000..836b8213ffb --- /dev/null +++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/CPUBenchmark.java @@ -0,0 +1,113 @@ +package com.yahoo.vespa.hosted.node.verification.hardware.benchmarks; + +import com.yahoo.vespa.hosted.node.verification.commons.CommandExecutor; +import com.yahoo.vespa.hosted.node.verification.commons.OutputParser; +import com.yahoo.vespa.hosted.node.verification.commons.ParseInstructions; +import com.yahoo.vespa.hosted.node.verification.commons.ParseResult; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.logging.Level; +import java.util.logging.Logger; + +/** + * Created by sgrostad on 11/07/2017. + */ +public class CPUBenchmark implements Benchmark { + + private static final String CPU_BENCHMARK_COMMAND = "perf stat -e cycles dd if=/dev/zero of=/dev/null count=100000 2>&1 | grep 'cycles\\|seconds'"; + private static final String CYCLES_SEARCH_WORD = "cycles"; + private static final String SECONDS_SEARCH_WORD = "seconds"; + private static final String SPLIT_REGEX_STRING = "\\s+"; + private static final int SEARCH_ELEMENT_INDEX = 1; + private static final int RETURN_ELEMENT_INDEX = 0; + private static final Logger logger = Logger.getLogger(CPUBenchmark.class.getName()); + private final HardwareResults hardwareResults; + + private final CommandExecutor commandExecutor; + + public CPUBenchmark(HardwareResults hardwareResults, CommandExecutor commandExecutor) { + this.hardwareResults = hardwareResults; + this.commandExecutor = commandExecutor; + } + + public void doBenchmark() { + try { + ArrayList<String> commandOutput = commandExecutor.executeCommand(CPU_BENCHMARK_COMMAND); + ArrayList<ParseResult> parseResults = parseCpuCyclesPerSec(commandOutput); + setCpuCyclesPerSec(parseResults); + } catch (IOException e) { + logger.log(Level.WARNING, "Failed to perform CPU benchmark", e); + } + } + + protected ArrayList<ParseResult> parseCpuCyclesPerSec(ArrayList<String> commandOutput) { + ArrayList<String> searchWords = new ArrayList<>(Arrays.asList(CYCLES_SEARCH_WORD, SECONDS_SEARCH_WORD)); + ParseInstructions parseInstructions = new ParseInstructions(SEARCH_ELEMENT_INDEX, RETURN_ELEMENT_INDEX, SPLIT_REGEX_STRING, searchWords); + return OutputParser.parseOutput(parseInstructions, commandOutput); + } + + + protected void setCpuCyclesPerSec(ArrayList<ParseResult> parseResults) { + double cpuCyclesPerSec = getCyclesPerSecond(parseResults); + if (cpuCyclesPerSec > 0) { + hardwareResults.setCpuCyclesPerSec(cpuCyclesPerSec); + } + } + + protected double getCyclesPerSecond(ArrayList<ParseResult> parseResults) { + double cycles = -1; + double seconds = -1; + for (ParseResult parseResult : parseResults) { + switch (parseResult.getSearchWord()) { + case CYCLES_SEARCH_WORD: + cycles = makeCyclesDouble(parseResult.getValue()); + break; + case SECONDS_SEARCH_WORD: + seconds = makeSecondsDouble(parseResult.getValue()); + break; + default: + throw new RuntimeException("Invalid ParseResult searchWord"); + } + } + if (cycles > 0 && seconds > 0) { + return convertToGHz(cycles, seconds); + } + return -1; + } + + protected double makeCyclesDouble(String cycles) { + cycles = cycles.replaceAll("[^\\d]", ""); + if (checkIfNumber(cycles)) { + return Double.parseDouble(cycles); + } + return -1; + } + + protected double makeSecondsDouble(String seconds) { + seconds = seconds.replaceAll(",", "."); + if (checkIfNumber(seconds)) { + return Double.parseDouble(seconds); + } + return -1; + } + + protected boolean checkIfNumber(String numberCandidate) { + if (numberCandidate == null || numberCandidate.equals("")) { + return false; + } + try { + Double.parseDouble(numberCandidate); + } catch (NumberFormatException e) { + return false; + } + return true; + } + + protected double convertToGHz(double cycles, double seconds) { + double giga = 1000000000.0; + return (cycles / seconds) / giga; + } + +} diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/DiskBenchmark.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/DiskBenchmark.java new file mode 100644 index 00000000000..8b5502cd415 --- /dev/null +++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/DiskBenchmark.java @@ -0,0 +1,96 @@ +package com.yahoo.vespa.hosted.node.verification.hardware.benchmarks; + +import com.yahoo.vespa.hosted.node.verification.commons.CommandExecutor; +import com.yahoo.vespa.hosted.node.verification.commons.OutputParser; +import com.yahoo.vespa.hosted.node.verification.commons.ParseInstructions; +import com.yahoo.vespa.hosted.node.verification.commons.ParseResult; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.logging.Level; +import java.util.logging.Logger; + + +/** + * Created by olaa on 10/07/2017. + */ +public class DiskBenchmark implements Benchmark { + + private static final String DISK_BENCHMARK_COMMAND = "time (dd if=/dev/zero of=/tmp/tempfile bs=16k count=16k > /dev/null; sync; rm /tmp/tempfile) 2>&1 | grep bytes | awk '{ print $8 \" \" $9 }'"; + private static final String KILO_BYTE_SEARCH_WORD = "kB/s"; + private static final String MEGA_BYTE_SEARCH_WORD = "MB/s"; + private static final String GIGA_BYTE_SEARCH_WORD = "GB/s"; + private static final String SPLIT_REGEX_STRING = " "; + private static final int SEARCH_ELEMENT_INDEX = 1; + private static final int RETURN_ELEMENT_INDEX = 0; + private static final Logger logger = Logger.getLogger(DiskBenchmark.class.getName()); + private final HardwareResults hardwareResults; + private final CommandExecutor commandExecutor; + + public DiskBenchmark(HardwareResults hardwareResults, CommandExecutor commandExecutor) { + this.hardwareResults = hardwareResults; + this.commandExecutor = commandExecutor; + } + + public void doBenchmark() { + try { + ArrayList<String> commandOutput = commandExecutor.executeCommand(DISK_BENCHMARK_COMMAND); + ParseResult parseResult = parseDiskSpeed(commandOutput); + setDiskSpeed(parseResult); + } catch (IOException e) { + logger.log(Level.WARNING, "Failed to perform disk benchmark", e); + } + } + + protected ParseResult parseDiskSpeed(ArrayList<String> commandOutput) { + ArrayList<String> searchWords = new ArrayList<>(Arrays.asList(KILO_BYTE_SEARCH_WORD, MEGA_BYTE_SEARCH_WORD, GIGA_BYTE_SEARCH_WORD)); + ParseInstructions parseInstructions = new ParseInstructions(SEARCH_ELEMENT_INDEX, RETURN_ELEMENT_INDEX, SPLIT_REGEX_STRING, searchWords); + return OutputParser.parseSingleOutput(parseInstructions, commandOutput); + } + + protected void setDiskSpeed(ParseResult parseResult) { + double diskSpeedMBs = getDiskSpeedInMBs(parseResult); + hardwareResults.setDiskSpeedMbs(diskSpeedMBs); + } + + protected double getDiskSpeedInMBs(ParseResult parseResult) { + double diskSpeedMBs = 0; + double convertKBsToMBs = 1 / 1000.0; + double convertGBsToMBs = 1000.0; + double convertMbsToMBs = 1.0; + String diskSpeed = parseResult.getValue(); + if (checkSpeedValidity(diskSpeed)) { + switch (parseResult.getSearchWord()) { + case KILO_BYTE_SEARCH_WORD: + diskSpeedMBs = convertToMBs(diskSpeed, convertKBsToMBs); + break; + case MEGA_BYTE_SEARCH_WORD: + diskSpeedMBs = convertToMBs(diskSpeed, convertMbsToMBs); + break; + case GIGA_BYTE_SEARCH_WORD: + diskSpeedMBs = convertToMBs(diskSpeed, convertGBsToMBs); + break; + default: + throw new RuntimeException("Invalid ParseResult searchWord"); + } + } + + return diskSpeedMBs; + } + + protected boolean checkSpeedValidity(String speed) { + try { + Double.parseDouble(speed); + } catch (NullPointerException | NumberFormatException e) { + return false; + } + return true; + } + + protected Double convertToMBs(String speed, double numberToConvert) { + double speedMbs = Double.parseDouble(speed); + return speedMbs * numberToConvert; + } + +} diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/HardwareResults.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/HardwareResults.java new file mode 100644 index 00000000000..6d1cb01fdbb --- /dev/null +++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/HardwareResults.java @@ -0,0 +1,56 @@ +package com.yahoo.vespa.hosted.node.verification.hardware.benchmarks; + +/** + * Created by sgrostad on 11/07/2017. + * Stores results from benchmarks + */ +public class HardwareResults { + + private double cpuCyclesPerSec; + private double diskSpeedMbs; + private boolean ipv6Connectivity; + private Double memoryWriteSpeedGBs; + private Double memoryReadSpeedGBs; + + + public Double getMemoryWriteSpeedGBs() { + return memoryWriteSpeedGBs; + } + + public void setMemoryWriteSpeedGBs(Double memoryWriteSpeedGBs) { + this.memoryWriteSpeedGBs = memoryWriteSpeedGBs; + } + + public Double getMemoryReadSpeedGBs() { + return memoryReadSpeedGBs; + } + + public void setMemoryReadSpeedGBs(Double memoryReadSpeedGBs) { + this.memoryReadSpeedGBs = memoryReadSpeedGBs; + } + + public double getCpuCyclesPerSec() { + return cpuCyclesPerSec; + } + + public void setCpuCyclesPerSec(double cpuCycles) { + this.cpuCyclesPerSec = cpuCycles; + } + + public double getDiskSpeedMbs() { + return diskSpeedMbs; + } + + public void setDiskSpeedMbs(double diskSpeedMbs) { + this.diskSpeedMbs = diskSpeedMbs; + } + + public boolean isIpv6Connectivity() { + return ipv6Connectivity; + } + + public void setIpv6Connectivity(boolean ipv6Connectivity) { + this.ipv6Connectivity = ipv6Connectivity; + } + +} diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/MemoryBenchmark.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/MemoryBenchmark.java new file mode 100644 index 00000000000..898c2071b92 --- /dev/null +++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/MemoryBenchmark.java @@ -0,0 +1,99 @@ +package com.yahoo.vespa.hosted.node.verification.hardware.benchmarks; + +import com.yahoo.vespa.hosted.node.verification.commons.CommandExecutor; +import com.yahoo.vespa.hosted.node.verification.commons.OutputParser; +import com.yahoo.vespa.hosted.node.verification.commons.ParseInstructions; +import com.yahoo.vespa.hosted.node.verification.commons.ParseResult; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.logging.Level; +import java.util.logging.Logger; + +/** + * Created by sgrostad on 11/07/2017. + */ +public class MemoryBenchmark implements Benchmark { + + private static final String MEM_BENCHMARK_CREATE_FOLDER = "mkdir -p RAM_test"; + private static final String MEM_BENCHMARK_MOUNT_TMPFS = "sudo mount tmpfs -t tmpfs RAM_test/"; + private static final String MEM_BENCHMARK_UNMOUNT_TMPFS = "umount RAM_test"; + private static final String MEM_BENCHMARK_DELETE_FOLDER = "rm -rf RAM_test"; + private static final String MEM_BENCHMARK_WRITE_SPEED = "dd if=/dev/zero of=RAM_test/data_tmp bs=1M count=512"; + private static final String MEM_BENCHMARK_READ_SPEED = "dd if=RAM_test/data_tmp of=/dev/null bs=1M count=512"; + private static final String READ_AND_WRITE_SEARCH_WORD = "GB/s"; + private static final String SPLIT_REGEX_STRING = " "; + private static final int SEARCH_ELEMENT_INDEX = 8; + private static final int RETURN_ELEMENT_INDEX = 7; + private static final Logger logger = Logger.getLogger(MemoryBenchmark.class.getName()); + private final HardwareResults hardwareResults; + private final CommandExecutor commandExecutor; + + public MemoryBenchmark(HardwareResults hardwareResults, CommandExecutor commandExecutor) { + this.hardwareResults = hardwareResults; + this.commandExecutor = commandExecutor; + } + + public void doBenchmark() { + try { + setupMountPoint(); + ArrayList<String> commandOutput = commandExecutor.executeCommand(MEM_BENCHMARK_WRITE_SPEED); + ParseResult parseResult = parseMemorySpeed(commandOutput); + updateMemoryWriteSpeed(parseResult.getValue()); + commandOutput = commandExecutor.executeCommand(MEM_BENCHMARK_READ_SPEED); + parseResult = parseMemorySpeed(commandOutput); + updateMemoryReadSpeed(parseResult.getValue()); + } catch (IOException e) { + logger.log(Level.WARNING, "Failed to perform memory benchmark", e); + } finally { + breakDownMountPoint(); + } + } + + private void setupMountPoint() throws IOException { + commandExecutor.executeCommand(MEM_BENCHMARK_CREATE_FOLDER); + commandExecutor.executeCommand(MEM_BENCHMARK_MOUNT_TMPFS); + } + + private void breakDownMountPoint() { + try { + commandExecutor.executeCommand(MEM_BENCHMARK_UNMOUNT_TMPFS); + } catch (IOException e) { + logger.log(Level.WARNING, "Failed to unmount tmpfs folder", e); + } + try { + commandExecutor.executeCommand(MEM_BENCHMARK_DELETE_FOLDER); + } catch (IOException e) { + logger.log(Level.WARNING, "Failed to delete memory benchmark folder", e); + } + } + + protected ParseResult parseMemorySpeed(ArrayList<String> commandOutput) { + ArrayList<String> searchWords = new ArrayList<>(Arrays.asList(READ_AND_WRITE_SEARCH_WORD)); + ParseInstructions parseInstructions = new ParseInstructions(SEARCH_ELEMENT_INDEX, RETURN_ELEMENT_INDEX, SPLIT_REGEX_STRING, searchWords); + return OutputParser.parseSingleOutput(parseInstructions, commandOutput); + } + + protected void updateMemoryWriteSpeed(String memorySpeed) { + if (!isValidMemory(memorySpeed)) return; + double memoryWriteSpeedGbs = Double.parseDouble(memorySpeed); + hardwareResults.setMemoryWriteSpeedGBs(memoryWriteSpeedGbs); + } + + protected void updateMemoryReadSpeed(String memorySpeed) { + if (!isValidMemory(memorySpeed)) return; + double memoryReadSpeedGbs = Double.parseDouble(memorySpeed); + hardwareResults.setMemoryReadSpeedGBs(memoryReadSpeedGbs); + } + + protected boolean isValidMemory(String benchmarkOutput) { + try { + Double.parseDouble(benchmarkOutput); + } catch (NumberFormatException | NullPointerException e) { + return false; + } + return true; + } + +} diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/NetBenchmark.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/NetBenchmark.java new file mode 100644 index 00000000000..9133cb47690 --- /dev/null +++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/NetBenchmark.java @@ -0,0 +1,60 @@ +package com.yahoo.vespa.hosted.node.verification.hardware.benchmarks; + +import com.yahoo.vespa.hosted.node.verification.commons.CommandExecutor; +import com.yahoo.vespa.hosted.node.verification.commons.OutputParser; +import com.yahoo.vespa.hosted.node.verification.commons.ParseInstructions; +import com.yahoo.vespa.hosted.node.verification.commons.ParseResult; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.logging.Level; +import java.util.logging.Logger; + +/** + * Created by sgrostad on 11/07/2017. + */ +public class NetBenchmark implements Benchmark { + + private static final String NET_BENCHMARK_COMMAND = "ping6 -c 10 www.yahoo.com | grep transmitted"; + private static final String PING_SEARCH_WORD = "loss,"; + private static final String SPLIT_REGEX_STRING = "\\s+"; + private static final int SEARCH_ELEMENT_INDEX = 7; + private static final int RETURN_ELEMENT_INDEX = 5; + private static final Logger logger = Logger.getLogger(NetBenchmark.class.getName()); + private final HardwareResults hardwareResults; + private final CommandExecutor commandExecutor; + + public NetBenchmark(HardwareResults hardwareResults, CommandExecutor commandExecutor) { + this.hardwareResults = hardwareResults; + this.commandExecutor = commandExecutor; + } + + public void doBenchmark() { + try { + ArrayList<String> commandOutput = commandExecutor.executeCommand(NET_BENCHMARK_COMMAND); + ParseResult parseResult = parsePingResponse(commandOutput); + setIpv6Connectivity(parseResult); + } catch (IOException e) { + logger.log(Level.WARNING, "Failed to perform net benchmark", e); + } + } + + protected ParseResult parsePingResponse(ArrayList<String> commandOutput) { + ArrayList<String> searchWords = new ArrayList<>(Arrays.asList(PING_SEARCH_WORD)); + ParseInstructions parseInstructions = new ParseInstructions(SEARCH_ELEMENT_INDEX, RETURN_ELEMENT_INDEX, SPLIT_REGEX_STRING, searchWords); + return OutputParser.parseSingleOutput(parseInstructions, commandOutput); + + } + + protected void setIpv6Connectivity(ParseResult parseResult) { + if (parseResult.getSearchWord().equals(PING_SEARCH_WORD)) { + String pingResponse = parseResult.getValue(); + String packetLoss = pingResponse.replaceAll("[^\\d.]", ""); + if (packetLoss.equals("")) return; + if (Double.parseDouble(packetLoss) > 99) return; + hardwareResults.setIpv6Connectivity(true); + } + } + +} diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/net/Client.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/net/Client.java new file mode 100644 index 00000000000..26c97e0707b --- /dev/null +++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/net/Client.java @@ -0,0 +1,46 @@ +package com.yahoo.vespa.hosted.node.verification.hardware.net; + +import java.io.DataOutputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.net.Socket; + +/** + * Created by olaa on 19/07/2017. + * Sends file to server, for checking connection speed. + * Not used, can be deleted + */ +public class Client { + + public void sendFile(Socket socket, File file) throws IOException { + try (FileInputStream inputStream = new FileInputStream(file); + DataOutputStream outputStream = new DataOutputStream(socket.getOutputStream())) { + int fileSize = (int) file.length(); + byte[] buffer = new byte[fileSize]; + outputStream.writeUTF(file.getName()); + int receivedBytesCount; + while ((receivedBytesCount = inputStream.read(buffer)) != -1) { + outputStream.write(buffer, 0, receivedBytesCount); + } + } catch (IOException e) { + e.printStackTrace(); + } finally { + socket.close(); + } + } + + public static void main(String[] args) { + Client client = new Client(); + File file = new File("src/test/resources/testReadFile.txt"); + double start = System.currentTimeMillis() / 1000.0; + try { + client.sendFile(new Socket("localhost", 10000), file); + } catch (IOException e) { + e.printStackTrace(); + } + double finish = System.currentTimeMillis() / 1000.0; + System.out.println(((double) file.length() / (finish - start)) + " B/s"); + } + +} diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/net/Server.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/net/Server.java new file mode 100644 index 00000000000..96d363f8da2 --- /dev/null +++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/net/Server.java @@ -0,0 +1,41 @@ +package com.yahoo.vespa.hosted.node.verification.hardware.net; + +import java.io.DataInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.net.ServerSocket; +import java.net.Socket; + +/** + * Created by olaa on 19/07/2017. + * Receives file from client. + * Not used, can be deleted + */ +public class Server { + + public void serve(int portNumber) throws IOException { + ServerSocket server; + Socket client; + server = new ServerSocket(portNumber); + client = server.accept(); + DataInputStream dataInputStream = new DataInputStream(client.getInputStream()); + DataInputStream inputStream = new DataInputStream(client.getInputStream()); + String fileName = dataInputStream.readUTF(); + FileOutputStream fileOutputStream = new FileOutputStream("./" + fileName); + byte[] buffer = new byte[65535]; + int currentLength; + while ((currentLength = inputStream.read(buffer)) != -1) { + fileOutputStream.write(buffer, 0, currentLength); + } + } + + public static void main(String[] args) { + Server server = new Server(); + try { + server.serve(10000); + } catch (IOException e) { + e.printStackTrace(); + } + } + +} diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/yamasreport/HardwareReportDimensions.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/yamasreport/HardwareReportDimensions.java new file mode 100644 index 00000000000..44c7ff44c67 --- /dev/null +++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/yamasreport/HardwareReportDimensions.java @@ -0,0 +1,13 @@ +package com.yahoo.vespa.hosted.node.verification.hardware.yamasreport; + +import com.fasterxml.jackson.annotation.JsonProperty; + +/** + * Created by sgrostad on 12/07/2017. + */ +public class HardwareReportDimensions { + + @JsonProperty + private String hostname = "hostname.something"; + +} diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/yamasreport/HardwareReportMetrics.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/yamasreport/HardwareReportMetrics.java new file mode 100644 index 00000000000..6e4d868b710 --- /dev/null +++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/yamasreport/HardwareReportMetrics.java @@ -0,0 +1,41 @@ +package com.yahoo.vespa.hosted.node.verification.hardware.yamasreport; + +import com.fasterxml.jackson.annotation.JsonProperty; + +/** + * Created by sgrostad on 12/07/2017. + */ +public class HardwareReportMetrics { + + @JsonProperty + private Double cpuCyclesPerSec; + @JsonProperty + private Double diskSpeedMbs; + @JsonProperty + private Boolean ipv6Connectivity; + @JsonProperty + private Double memoryWriteSpeedGBs; + @JsonProperty + private Double memoryReadSpeedGBs; + + public void setCpuCyclesPerSec(Double cpuCyclesPerSec) { + this.cpuCyclesPerSec = cpuCyclesPerSec; + } + + public void setDiskSpeedMbs(Double diskSpeedMbs) { + this.diskSpeedMbs = diskSpeedMbs != null ? diskSpeedMbs : -1; + } + + public void setIpv6Connectivity(Boolean ipv6Connectivity) { + this.ipv6Connectivity = ipv6Connectivity; + } + + public void setMemoryWriteSpeedGBs(Double memoryWriteSpeedGBs) { + this.memoryWriteSpeedGBs = memoryWriteSpeedGBs != null ? memoryWriteSpeedGBs : -1; + } + + public void setMemoryReadSpeedGBs(Double memoryReadSpeedGBs) { + this.memoryReadSpeedGBs = memoryReadSpeedGBs != null ? memoryReadSpeedGBs : -1; + } + +} diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/yamasreport/YamasHardwareReport.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/yamasreport/YamasHardwareReport.java new file mode 100644 index 00000000000..7efb1d8c31a --- /dev/null +++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/yamasreport/YamasHardwareReport.java @@ -0,0 +1,80 @@ +package com.yahoo.vespa.hosted.node.verification.hardware.yamasreport; + +import com.fasterxml.jackson.annotation.JsonAnyGetter; +import com.fasterxml.jackson.annotation.JsonAnySetter; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.yahoo.vespa.hosted.node.verification.hardware.benchmarks.HardwareResults; + +import java.util.HashMap; +import java.util.Map; + +/** + * Created by sgrostad on 12/07/2017. + * JSON-mapped class for reporting to YAMAS + */ +public class YamasHardwareReport { + + @JsonProperty + private long timestamp; + @JsonProperty + private HardwareReportDimensions dimensions; + @JsonProperty + private HardwareReportMetrics metrics; + @JsonProperty + JsonObjectWrapper routing; + + public YamasHardwareReport() { + this.timestamp = System.currentTimeMillis() / 1000L; + setRouting(); + } + + public HardwareReportDimensions getDimensions() { + return dimensions; + } + + public void setDimensions(HardwareReportDimensions dimensions) { + this.dimensions = dimensions; + } + + public HardwareReportMetrics getMetrics() { + return metrics; + } + + public void setMetrics(HardwareReportMetrics metrics) { + this.metrics = metrics; + } + + private void setRouting() { + JsonObjectWrapper wrap = new JsonObjectWrapper("namespace", new String[]{"Vespa"}); + routing = new JsonObjectWrapper("yamas", wrap); + } + + public void createFromHardwareResults(HardwareResults hardwareResults) { + metrics = new HardwareReportMetrics(); + dimensions = new HardwareReportDimensions(); + metrics.setCpuCyclesPerSec(hardwareResults.getCpuCyclesPerSec()); + metrics.setDiskSpeedMbs(hardwareResults.getDiskSpeedMbs()); + metrics.setIpv6Connectivity(hardwareResults.isIpv6Connectivity()); + metrics.setMemoryWriteSpeedGBs(hardwareResults.getMemoryWriteSpeedGBs()); + metrics.setMemoryReadSpeedGBs(hardwareResults.getMemoryReadSpeedGBs()); + } + + class JsonObjectWrapper<T> { + private Map<String, T> wrappedObjects = new HashMap<String, T>(); + + public JsonObjectWrapper(String name, T wrappedObject) { + this.wrappedObjects.put(name, wrappedObject); + } + + @JsonAnyGetter + public Map<String, T> any() { + return wrappedObjects; + } + + @JsonAnySetter + public void set(String name, T value) { + wrappedObjects.put(name, value); + } + } + +} diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/HardwareNodeComparator.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/HardwareNodeComparator.java new file mode 100644 index 00000000000..adec38c2ea4 --- /dev/null +++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/HardwareNodeComparator.java @@ -0,0 +1,128 @@ +package com.yahoo.vespa.hosted.node.verification.spec; + +import com.yahoo.vespa.hosted.node.verification.spec.retrievers.HardwareInfo; +import com.yahoo.vespa.hosted.node.verification.spec.yamasreport.SpecReportDimensions; +import com.yahoo.vespa.hosted.node.verification.spec.yamasreport.SpecReportMetrics; +import com.yahoo.vespa.hosted.node.verification.spec.yamasreport.YamasSpecReport; + +/** + * Created by olaa on 04/07/2017. + * Compares two HardwareInfo objects + */ +public class HardwareNodeComparator { + + public static YamasSpecReport compare(HardwareInfo node, HardwareInfo actualHardware) { + Boolean equalHardware = true; + YamasSpecReport yamasSpecReport = new YamasSpecReport(); + SpecReportDimensions specReportDimensions = new SpecReportDimensions(); + SpecReportMetrics specReportMetrics = new SpecReportMetrics(); + + if (node == null || actualHardware == null) { + return yamasSpecReport; + } + + setReportMetrics(node, actualHardware, specReportMetrics); + + equalHardware &= compareMemory(node, actualHardware, specReportDimensions); + equalHardware &= compareCPU(node, actualHardware, specReportDimensions); + equalHardware &= compareNetInterface(node, actualHardware, specReportDimensions); + equalHardware &= compareDisk(node, actualHardware, specReportDimensions); + + specReportMetrics.setMatch(equalHardware); + yamasSpecReport.setDimensions(specReportDimensions); + yamasSpecReport.setMetrics(specReportMetrics); + + return yamasSpecReport; + } + + private static void setReportMetrics(HardwareInfo node, HardwareInfo actualHardware, SpecReportMetrics specReportMetrics) { + setMemoryMetrics(node, actualHardware, specReportMetrics); + setCpuMetrics(node, actualHardware, specReportMetrics); + setDiskTypeMetrics(node, actualHardware, specReportMetrics); + setDiskSpaceMetrics(node, actualHardware, specReportMetrics); + setNetMetrics(node, actualHardware, specReportMetrics); + } + + private static void setMemoryMetrics(HardwareInfo node, HardwareInfo actualHardware, SpecReportMetrics specReportMetrics) { + double expectedMemory = node.getMinMainMemoryAvailableGb(); + double actualMemory = actualHardware.getMinMainMemoryAvailableGb(); + if (!insideThreshold(expectedMemory, actualMemory)) { + specReportMetrics.setExpectedMemoryAvailable(expectedMemory); + specReportMetrics.setActualMemoryAvailable(actualMemory); + } + } + + private static void setCpuMetrics(HardwareInfo node, HardwareInfo actualHardware, SpecReportMetrics specReportMetrics) { + int expectedCpuCores = node.getMinCpuCores(); + int actualCpuCores = actualHardware.getMinCpuCores(); + if (expectedCpuCores != actualCpuCores) { + specReportMetrics.setExpectedcpuCores(expectedCpuCores); + specReportMetrics.setActualcpuCores(actualCpuCores); + } + } + + private static void setDiskTypeMetrics(HardwareInfo node, HardwareInfo actualHardware, SpecReportMetrics specReportMetrics) { + Boolean expectedFastDisk = node.getFastDisk(); + Boolean actualFastDisk = actualHardware.getFastDisk(); + if (expectedFastDisk != null && actualFastDisk != null && expectedFastDisk != actualFastDisk) { + specReportMetrics.setExpectedDiskType(expectedFastDisk); + specReportMetrics.setActualDiskType(actualFastDisk); + } + } + + private static void setDiskSpaceMetrics(HardwareInfo node, HardwareInfo actualHardware, SpecReportMetrics specReportMetrics) { + double expectedDiskSpace = node.getMinDiskAvailableGb(); + double actualDiskSpace = actualHardware.getMinDiskAvailableGb(); + if (!insideThreshold(expectedDiskSpace, actualDiskSpace)) { + specReportMetrics.setExpectedDiskSpaceAvailable(expectedDiskSpace); + specReportMetrics.setActualDiskSpaceAvailable(actualDiskSpace); + } + } + + private static void setNetMetrics(HardwareInfo node, HardwareInfo actualHardware, SpecReportMetrics specReportMetrics) { + double expectedInterfaceSpeed = node.getInterfaceSpeedMbs(); + double actualInterfaceSpeed = actualHardware.getInterfaceSpeedMbs(); + if (!insideThreshold(expectedInterfaceSpeed, actualInterfaceSpeed)) { + specReportMetrics.setExpectedInterfaceSpeed(expectedInterfaceSpeed); + specReportMetrics.setActualInterfaceSpeed(actualInterfaceSpeed); + } + } + + private static boolean compareCPU(HardwareInfo node, HardwareInfo actualHardware, SpecReportDimensions specReportDimensions) { + boolean equalCPU = node.getMinCpuCores() == actualHardware.getMinCpuCores(); + specReportDimensions.setCpuCoresMatch(equalCPU); + return equalCPU; + } + + private static boolean compareMemory(HardwareInfo node, HardwareInfo actualHardware, SpecReportDimensions specReportDimensions) { + boolean equalMemory = insideThreshold(node.getMinMainMemoryAvailableGb(), actualHardware.getMinMainMemoryAvailableGb()); + specReportDimensions.setMemoryMatch(equalMemory); + return equalMemory; + } + + private static boolean compareNetInterface(HardwareInfo node, HardwareInfo actualHardware, SpecReportDimensions specReportDimensions) { + boolean equalNetInterfaceSpeed = insideThreshold(node.getInterfaceSpeedMbs(), actualHardware.getInterfaceSpeedMbs()); + boolean equalIpv6 = node.getIpv6Connectivity() == actualHardware.getIpv6Connectivity(); + boolean equalIpv4 = node.getIpv4Connectivity() == actualHardware.getIpv4Connectivity(); + specReportDimensions.setNetInterfaceSpeedMatch(equalNetInterfaceSpeed); + specReportDimensions.setIpv6Match(equalIpv6); + specReportDimensions.setIpv4Match(equalIpv4); + return equalNetInterfaceSpeed && equalIpv6 && equalIpv4; + + } + + private static boolean compareDisk(HardwareInfo node, HardwareInfo actualHardware, SpecReportDimensions specReportDimensions) { + boolean equalDiskType = node.getFastDisk() == actualHardware.getFastDisk(); + boolean equalDiskSize = insideThreshold(node.getMinDiskAvailableGb(), actualHardware.getMinDiskAvailableGb()); + specReportDimensions.setFastDiskMatch(equalDiskType); + specReportDimensions.setDiskAvailableMatch(equalDiskSize); + return equalDiskType && equalDiskSize; + } + + private static boolean insideThreshold(double value1, double value2) { + double lowerThresholdPercentage = 0.8; + double upperThresholdPercentage = 1.2; + return value1 > lowerThresholdPercentage * value2 && value1 < upperThresholdPercentage * value2; + } + +} diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/HostURLGenerator.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/HostURLGenerator.java new file mode 100644 index 00000000000..6f619141d2c --- /dev/null +++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/HostURLGenerator.java @@ -0,0 +1,23 @@ +package com.yahoo.vespa.hosted.node.verification.spec; + +import java.net.MalformedURLException; +import java.net.URL; + +/** + * Created by olaa on 14/07/2017. + * Makes the URL used to retrieve the JSON from the node repository with information about the node's spec. + */ +public class HostURLGenerator { + + private static final String NODE_HOSTNAME_PREFIX = "/nodes/v2/node/"; + + protected URL generateNodeInfoUrl(String configServerHostName) throws MalformedURLException { + String nodeHostName = getEnvironmentVariable("HOSTNAME"); + return new URL(configServerHostName + NODE_HOSTNAME_PREFIX + nodeHostName); + } + + protected String getEnvironmentVariable(String variableName) { + return System.getenv(variableName); + } + +} diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/README.md b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/README.md new file mode 100644 index 00000000000..b250223841f --- /dev/null +++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/README.md @@ -0,0 +1,2 @@ +# Spec Verification +Verification of node repo information diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/SpecVerifier.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/SpecVerifier.java new file mode 100644 index 00000000000..5f5690a16f6 --- /dev/null +++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/SpecVerifier.java @@ -0,0 +1,71 @@ +package com.yahoo.vespa.hosted.node.verification.spec; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.yahoo.vespa.hosted.node.verification.spec.noderepo.IPAddressVerifier; +import com.yahoo.vespa.hosted.node.verification.spec.noderepo.NodeGenerator; +import com.yahoo.vespa.hosted.node.verification.spec.noderepo.NodeInfoRetriever; +import com.yahoo.vespa.hosted.node.verification.spec.noderepo.NodeJsonModel; +import com.yahoo.vespa.hosted.node.verification.spec.retrievers.HardwareInfo; +import com.yahoo.vespa.hosted.node.verification.spec.retrievers.HardwareInfoRetriever; +import com.yahoo.vespa.hosted.node.verification.spec.yamasreport.YamasSpecReport; + +import java.net.MalformedURLException; +import java.net.URL; +import java.util.logging.Level; +import java.util.logging.Logger; + +/** + * Created by olaa on 14/07/2017. + * Creates two HardwareInfo objects, one with spec from node repository and one from spec retrieved at the node. + * Compares the objects and returns the result. + */ +public class SpecVerifier { + + private static final Logger logger = Logger.getLogger(SpecVerifier.class.getName()); + + public void verifySpec(String zoneHostName) { + URL nodeRepoUrl; + try { + HostURLGenerator hostURLGenerator = new HostURLGenerator(); + nodeRepoUrl = hostURLGenerator.generateNodeInfoUrl(zoneHostName); + } catch (MalformedURLException e) { + logger.log(Level.WARNING, "Failed to generate config server url", e); + return; + } + NodeJsonModel nodeJsonModel = NodeInfoRetriever.retrieve(nodeRepoUrl); + HardwareInfo node = NodeGenerator.convertJsonModel(nodeJsonModel); + HardwareInfo actualHardware = HardwareInfoRetriever.retrieve(); + YamasSpecReport yamasSpecReport = HardwareNodeComparator.compare(node, actualHardware); + IPAddressVerifier ipAddressVerifier = new IPAddressVerifier(); + ipAddressVerifier.reportFaultyIpAddresses(nodeJsonModel, yamasSpecReport); + + printResults(yamasSpecReport); + } + + private void printResults(YamasSpecReport yamasSpecReport) { + //TODO: Instead of println, report JSON to YAMAS + ObjectMapper om = new ObjectMapper(); + try { + System.out.println(om.writeValueAsString(yamasSpecReport)); + } catch (JsonProcessingException e) { + e.printStackTrace(); + } + } + + + public static void main(String[] args) { + /** + * When testing in docker container + * docker run --hostname 13305821.ostk.bm2.prod.gq1.yahoo.com --name 13305821.ostk.bm2.prod.gq1.yahoo.com [image] + */ + if (args.length != 1) { + throw new RuntimeException("Expected only 1 argument - config server zone url"); + } + + String zoneHostName = args[0]; + SpecVerifier specVerifier = new SpecVerifier(); + specVerifier.verifySpec(zoneHostName); + } + +} diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/noderepo/IPAddressVerifier.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/noderepo/IPAddressVerifier.java new file mode 100644 index 00000000000..b3b6950a54a --- /dev/null +++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/noderepo/IPAddressVerifier.java @@ -0,0 +1,101 @@ +package com.yahoo.vespa.hosted.node.verification.spec.noderepo; + +import com.yahoo.vespa.hosted.node.verification.spec.yamasreport.YamasSpecReport; + +import javax.naming.NamingEnumeration; +import javax.naming.NamingException; +import javax.naming.directory.Attribute; +import javax.naming.directory.Attributes; +import javax.naming.directory.DirContext; +import javax.naming.directory.InitialDirContext; +import java.util.ArrayList; +import java.util.Enumeration; +import java.util.Hashtable; +import java.util.logging.Level; +import java.util.logging.Logger; + +/** + * Created by olaa on 14/07/2017. + * Checks if all additional Ipv6 addresses has the same hostname as the main Ipv6 address. + */ + +public class IPAddressVerifier { + + private static final Logger logger = Logger.getLogger(IPAddressVerifier.class.getName()); + + public void reportFaultyIpAddresses(NodeJsonModel nodeJsonModel, YamasSpecReport yamasSpecReport) { + String[] faultyIpAddresses = getFaultyIpAddresses(nodeJsonModel.getIpv6Address(), nodeJsonModel.getAdditionalIpAddresses()); + if (faultyIpAddresses.length > 0) { + yamasSpecReport.setFaultyIpAddresses(faultyIpAddresses); + } + } + + protected String reverseLookUp(String ipAddress) throws NamingException { + Hashtable env = new Hashtable(); + env.put("java.naming.factory.initial", "com.sun.jndi.dns.DnsContextFactory"); + String ipAddressInLookupFormat = convertToLookupFormat(ipAddress); + String attributeName = ipAddress; + DirContext ctx = new InitialDirContext(env); + //98.138.253.109 + //Attributes attrs = ctx.getAttributes("1.0.6.f.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.2.9.7.7.0.c.0.4.9.9.8.0.2.0.0.1.ip6.arpa",new String[] {"PTR"}); + Attributes attrs = ctx.getAttributes(attributeName, new String[]{"PTR"}); + for (NamingEnumeration ae = attrs.getAll(); ae.hasMoreElements(); ) { + Attribute attr = (Attribute) ae.next(); + Enumeration vals = attr.getAll(); + if (vals.hasMoreElements()) { + return vals.nextElement().toString(); + } + } + ctx.close(); + return ""; + } + + protected String convertToLookupFormat(String ipAddress) { + StringBuilder newIpAddress = new StringBuilder(); + String doubleColonReplacement = "0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0."; + String domain = "ip6.arpa"; + String[] hextets = ipAddress.split(":"); + for (int i = hextets.length - 1; i >= 0; i--) { + String reversedHextet = new StringBuilder(hextets[i]).reverse().toString(); + if (reversedHextet.equals("")) { + newIpAddress.append(doubleColonReplacement); + continue; + } + String trailingZeroes = "0000"; + String paddedHextet = (reversedHextet + trailingZeroes).substring(0, trailingZeroes.length()); + String punctuatedHextet = paddedHextet.replaceAll(".(?=)", "$0."); + newIpAddress.append(punctuatedHextet); + } + newIpAddress.append(domain); + return newIpAddress.toString(); + } + + public String[] getFaultyIpAddresses(String ipAddress, String[] additionalIpAddresses) { + if (ipAddress == null || additionalIpAddresses == null || additionalIpAddresses.length == 0) + return new String[0]; + String realHostname; + ArrayList<String> faultyIpAddresses = new ArrayList<>(); + try { + realHostname = reverseLookUp(ipAddress); + } catch (NamingException e) { + logger.log(Level.WARNING, "Unable to look up host name of address " + ipAddress, e); + return new String[0]; + } + for (String additionalIpAddress : additionalIpAddresses) { + addIfFaultyIpAddress(realHostname, additionalIpAddress, faultyIpAddresses); + } + return faultyIpAddresses.stream().toArray(String[]::new); + } + + private void addIfFaultyIpAddress(String realHostname, String additionalIpAddress, ArrayList<String> faultyIpAddresses) { + try { + String additionalHostName = reverseLookUp(additionalIpAddress); + if (!realHostname.equals(additionalHostName)) { + faultyIpAddresses.add(additionalIpAddress); + } + } catch (NamingException e) { + logger.log(Level.WARNING, "Unable to retrieve hostname of additional address: " + additionalIpAddress, e); + } + } + +} diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/noderepo/NodeGenerator.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/noderepo/NodeGenerator.java new file mode 100644 index 00000000000..680cbbb132e --- /dev/null +++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/noderepo/NodeGenerator.java @@ -0,0 +1,23 @@ +package com.yahoo.vespa.hosted.node.verification.spec.noderepo; + +import com.yahoo.vespa.hosted.node.verification.spec.retrievers.HardwareInfo; + +/** + * Created by olaa on 07/07/2017. + * Converts a NodeJsonModel object to a HardwareInfo object. + */ +public class NodeGenerator { + + private static void addStandardSpecifications(HardwareInfo node) { + node.setIpv4Connectivity(true); + node.setIpv6Connectivity(true); + node.setInterfaceSpeedMbs(1000); + } + + public static HardwareInfo convertJsonModel(NodeJsonModel nodeJsonModel) { + HardwareInfo node = nodeJsonModel.copyToHardwareInfo(); + addStandardSpecifications(node); + return node; + } + +} diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/noderepo/NodeInfoRetriever.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/noderepo/NodeInfoRetriever.java new file mode 100644 index 00000000000..9297ad873be --- /dev/null +++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/noderepo/NodeInfoRetriever.java @@ -0,0 +1,31 @@ +package com.yahoo.vespa.hosted.node.verification.spec.noderepo; + +import com.fasterxml.jackson.databind.ObjectMapper; + +import java.io.IOException; +import java.net.URL; +import java.util.logging.Level; +import java.util.logging.Logger; + +/** + * Created by olaa on 04/07/2017. + * Parse JSON from node repository and stores information as a NodeJsonModel object. + */ +public class NodeInfoRetriever { + + private static final Logger logger = Logger.getLogger(NodeInfoRetriever.class.getName()); + + public static NodeJsonModel retrieve(URL url) { + NodeJsonModel nodeJsonModel; + ObjectMapper objectMapper = new ObjectMapper(); + try { + nodeJsonModel = objectMapper.readValue(url, NodeJsonModel.class); + } catch (IOException e) { + logger.log(Level.WARNING, "Failed to parse JSON", e); + return null; + } + return nodeJsonModel; + } + +} + diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/noderepo/NodeJsonModel.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/noderepo/NodeJsonModel.java new file mode 100644 index 00000000000..f5bbf39e876 --- /dev/null +++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/noderepo/NodeJsonModel.java @@ -0,0 +1,51 @@ +package com.yahoo.vespa.hosted.node.verification.spec.noderepo; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.yahoo.vespa.hosted.node.verification.spec.retrievers.HardwareInfo; + +/** + * Created by olaa on 05/07/2017. + * Object with the information node repositories has about the node. + */ + +@JsonIgnoreProperties(ignoreUnknown = true) +public class NodeJsonModel { + + @JsonProperty("minDiskAvailableGb") + private double minDiskAvailableGb; + @JsonProperty("minMainMemoryAvailableGb") + private double minMainMemoryAvailableGb; + @JsonProperty("minCpuCores") + private double minCpuCores; + @JsonProperty("fastDisk") + private boolean fastDisk; + @JsonProperty("ipAddresses") + private String[] ipAddresses; + @JsonProperty("additionalIpAddresses") + private String[] additionalIpAddresses; + + public String[] getAdditionalIpAddresses() { + return additionalIpAddresses; + } + + public HardwareInfo copyToHardwareInfo() { + HardwareInfo hardwareInfo = new HardwareInfo(); + hardwareInfo.setMinMainMemoryAvailableGb(this.minMainMemoryAvailableGb); + hardwareInfo.setMinDiskAvailableGb(this.minDiskAvailableGb); + hardwareInfo.setMinCpuCores((int) Math.round(this.minCpuCores)); + hardwareInfo.setFastDisk(this.fastDisk); + return hardwareInfo; + } + + public String getIpv6Address() { + String ipv6Regex = "^((?:[0-9A-Fa-f]{1,4}))((?::[0-9A-Fa-f]{1,4}))*::((?:[0-9A-Fa-f]{1,4}))((?::[0-9A-Fa-f]{1,4}))*|((?:[0-9A-Fa-f]{1,4}))((?::[0-9A-Fa-f]{1,4})){7}$"; + for (String ipAddress : ipAddresses) { + if (ipAddress.matches(ipv6Regex)) { + return ipAddress; + } + } + return null; + } + +} diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/CPURetriever.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/CPURetriever.java new file mode 100644 index 00000000000..e903bc4183f --- /dev/null +++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/CPURetriever.java @@ -0,0 +1,58 @@ +package com.yahoo.vespa.hosted.node.verification.spec.retrievers; + +import com.yahoo.vespa.hosted.node.verification.commons.CommandExecutor; +import com.yahoo.vespa.hosted.node.verification.commons.OutputParser; +import com.yahoo.vespa.hosted.node.verification.commons.ParseInstructions; +import com.yahoo.vespa.hosted.node.verification.commons.ParseResult; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.logging.Level; +import java.util.logging.Logger; + +/** + * Created by olaa on 30/06/2017. + */ +public class CPURetriever implements HardwareRetriever { + + private static final String CPU_INFO_COMMAND = "cat /proc/cpuinfo"; + private static final String SEARCH_WORD = "cpu MHz"; + private static final String REGEX_SPLIT = "\\s+:\\s"; + private static final int SEARCH_ELEMENT_INDEX = 0; + private static final int RETURN_ELEMENT_INDEX = 1; + private static final Logger logger = Logger.getLogger(CPURetriever.class.getName()); + private final HardwareInfo hardwareInfo; + private final CommandExecutor commandExecutor; + + public CPURetriever(HardwareInfo hardwareInfo, CommandExecutor commandExecutor) { + this.hardwareInfo = hardwareInfo; + this.commandExecutor = commandExecutor; + } + + public void updateInfo() { + try { + ArrayList<String> commandOutput = commandExecutor.executeCommand(CPU_INFO_COMMAND); + ArrayList<ParseResult> parseResults = parseCPUInfoFile(commandOutput); + setCpuCores(parseResults); + } catch (IOException e) { + logger.log(Level.WARNING, "Failed to retrieve CPU info", e); + } + } + + protected ArrayList<ParseResult> parseCPUInfoFile(ArrayList<String> commandOutput) { + ArrayList<String> searchWords = new ArrayList<>(Arrays.asList(SEARCH_WORD)); + ParseInstructions parseInstructions = new ParseInstructions(SEARCH_ELEMENT_INDEX, RETURN_ELEMENT_INDEX, REGEX_SPLIT, searchWords); + ArrayList<ParseResult> parseResults = OutputParser.parseOutput(parseInstructions, commandOutput); + return parseResults; + } + + protected void setCpuCores(ArrayList<ParseResult> parseResults) { + hardwareInfo.setMinCpuCores(countCpuCores(parseResults)); + } + + protected int countCpuCores(ArrayList<ParseResult> parseResults) { + return parseResults.size(); + } + +} diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/DiskRetriever.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/DiskRetriever.java new file mode 100644 index 00000000000..f8cb169ef5e --- /dev/null +++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/DiskRetriever.java @@ -0,0 +1,96 @@ +package com.yahoo.vespa.hosted.node.verification.spec.retrievers; + +import com.yahoo.vespa.hosted.node.verification.commons.CommandExecutor; +import com.yahoo.vespa.hosted.node.verification.commons.OutputParser; +import com.yahoo.vespa.hosted.node.verification.commons.ParseInstructions; +import com.yahoo.vespa.hosted.node.verification.commons.ParseResult; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.logging.Level; +import java.util.logging.Logger; + +/** + * Created by olaa on 30/06/2017. + */ +public class DiskRetriever implements HardwareRetriever { + private static final String DISK_CHECK_TYPE = "lsblk -d -o name,rota"; + private static final String DISK_CHECK_SIZE = "df -BG /"; + private static final String DISK_NAME = "sda"; + private static final String DISK_TYPE_REGEX_SPLIT = "\\s+"; + private static final int DISK_TYPE_SEARCH_ELEMENT_INDEX = 0; + private static final int DISK_TYPE_RETURN_ELEMENT_INDEX = 1; + private static final String DISK_SIZE_SEARCH_WORD = ".*\\d+.*"; + private static final String DISK_SIZE_REGEX_SPLIT = "\\s+"; + private static final int DISK_SIZE_SEARCH_ELEMENT_INDEX = 3; + private static final int DISK_SIZE_RETURN_ELEMENT_INDEX = 1; + private static final Logger logger = Logger.getLogger(DiskRetriever.class.getName()); + private final HardwareInfo hardwareInfo; + private final CommandExecutor commandExecutor; + + + public DiskRetriever(HardwareInfo hardwareInfo, CommandExecutor commandExecutor) { + this.hardwareInfo = hardwareInfo; + this.commandExecutor = commandExecutor; + } + + public void updateInfo() { + try { + updateDiskType(); + updateDiskSize(); + } catch (IOException e) { + logger.log(Level.WARNING, "Failed to retrieve disk info", e); + } + } + + protected void updateDiskType() throws IOException { + ArrayList<String> commandOutput = commandExecutor.executeCommand(DISK_CHECK_TYPE); + ParseResult parseResult = parseDiskType(commandOutput); + setDiskType(parseResult); + } + + protected void updateDiskSize() throws IOException { + ArrayList<String> commandOutput = commandExecutor.executeCommand(DISK_CHECK_SIZE); + ParseResult parseResult = parseDiskSize(commandOutput); + setDiskSize(parseResult); + } + + protected ParseResult parseDiskType(ArrayList<String> commandOutput) { + ArrayList<String> searchWords = new ArrayList<>(Arrays.asList(DISK_NAME)); + ParseInstructions parseInstructions = new ParseInstructions(DISK_TYPE_SEARCH_ELEMENT_INDEX, DISK_TYPE_RETURN_ELEMENT_INDEX, DISK_TYPE_REGEX_SPLIT, searchWords); + return OutputParser.parseSingleOutput(parseInstructions, commandOutput); + } + + protected void setDiskType(ParseResult parseResult) { + if (!parseResult.getSearchWord().equals(DISK_NAME)) { + return; + } + String fastDiskEnum = "0"; + String nonFastDiskEnum = "1"; + Boolean fastdisk = null; + if (parseResult.getValue().equals(fastDiskEnum)) { + fastdisk = true; + } else if (parseResult.getValue().equals(nonFastDiskEnum)) { + fastdisk = false; + } + hardwareInfo.setFastDisk(fastdisk); + } + + protected void setDiskSize(ParseResult parseResult) { + try { + String sizeValue = parseResult.getValue().replaceAll("[^\\d.]", ""); + double diskSize = Double.parseDouble(sizeValue); + hardwareInfo.setMinDiskAvailableGb(diskSize); + } catch (NumberFormatException | NullPointerException e) { + return; + } + } + + protected ParseResult parseDiskSize(ArrayList<String> commandOutput) { + ArrayList<String> searchWords = new ArrayList<>(Arrays.asList(DISK_SIZE_SEARCH_WORD)); + ParseInstructions parseInstructions = new ParseInstructions(DISK_SIZE_SEARCH_ELEMENT_INDEX, DISK_SIZE_RETURN_ELEMENT_INDEX, DISK_SIZE_REGEX_SPLIT, searchWords); + return OutputParser.parseSingleOutput(parseInstructions, commandOutput); + } + +}
\ No newline at end of file diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/HardwareInfo.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/HardwareInfo.java new file mode 100644 index 00000000000..3252c6ae8dc --- /dev/null +++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/HardwareInfo.java @@ -0,0 +1,75 @@ +package com.yahoo.vespa.hosted.node.verification.spec.retrievers; + +/** + * Created by olaa on 04/07/2017. + * All information the different retrievers retrieve is stored as a HardwareInfo object. + */ + +public class HardwareInfo { + + private double minDiskAvailableGb; + private double minMainMemoryAvailableGb; + private int minCpuCores; + private Boolean fastDisk; + private Boolean ipv4Connectivity; + private Boolean ipv6Connectivity; + private double interfaceSpeedMbs; + + + public double getInterfaceSpeedMbs() { + return interfaceSpeedMbs; + } + + public void setInterfaceSpeedMbs(double interfaceSpeedMbs) { + this.interfaceSpeedMbs = interfaceSpeedMbs; + } + + public double getMinDiskAvailableGb() { + return minDiskAvailableGb; + } + + public void setMinDiskAvailableGb(double minDiskAvailableGb) { + this.minDiskAvailableGb = minDiskAvailableGb; + } + + public Boolean getIpv6Connectivity() { + return ipv6Connectivity; + } + + public void setIpv6Connectivity(Boolean ipv6Connectivity) { + this.ipv6Connectivity = ipv6Connectivity; + } + + public Boolean getIpv4Connectivity() { + return ipv4Connectivity; + } + + public void setIpv4Connectivity(Boolean ipv4Connectivity) { + this.ipv4Connectivity = ipv4Connectivity; + } + + public double getMinMainMemoryAvailableGb() { + return minMainMemoryAvailableGb; + } + + public void setMinMainMemoryAvailableGb(double minMainMemoryAvailableGb) { + this.minMainMemoryAvailableGb = minMainMemoryAvailableGb; + } + + public void setFastDisk(Boolean fastDisk) { + this.fastDisk = fastDisk; + } + + public Boolean getFastDisk() { + return fastDisk; + } + + public int getMinCpuCores() { + return minCpuCores; + } + + public void setMinCpuCores(int minCpuCores) { + this.minCpuCores = minCpuCores; + } + +} diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/HardwareInfoRetriever.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/HardwareInfoRetriever.java new file mode 100644 index 00000000000..0ea0d91ac19 --- /dev/null +++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/HardwareInfoRetriever.java @@ -0,0 +1,28 @@ +package com.yahoo.vespa.hosted.node.verification.spec.retrievers; + +import com.yahoo.vespa.hosted.node.verification.commons.CommandExecutor; + +import java.util.ArrayList; + +/** + * Created by olaa on 30/06/2017. + * Makes a HardwareInfo object and calls all the retrievers for this object. + */ +public class HardwareInfoRetriever { + + public static HardwareInfo retrieve() { + HardwareInfo hardwareInfo = new HardwareInfo(); + CommandExecutor commandExecutor = new CommandExecutor(); + ArrayList<HardwareRetriever> infoList = new ArrayList<>(); + infoList.add(new CPURetriever(hardwareInfo, commandExecutor)); + infoList.add(new MemoryRetriever(hardwareInfo, commandExecutor)); + infoList.add(new DiskRetriever(hardwareInfo, commandExecutor)); + infoList.add(new NetRetriever(hardwareInfo, commandExecutor)); + + for (HardwareRetriever hardwareInfoType : infoList) { + hardwareInfoType.updateInfo(); + } + return hardwareInfo; + } + +} diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/HardwareRetriever.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/HardwareRetriever.java new file mode 100644 index 00000000000..2e3852a1d0f --- /dev/null +++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/HardwareRetriever.java @@ -0,0 +1,10 @@ +package com.yahoo.vespa.hosted.node.verification.spec.retrievers; + +/** + * Created by olaa on 30/06/2017. + */ +public interface HardwareRetriever { + + void updateInfo(); + +} diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/MemoryRetriever.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/MemoryRetriever.java new file mode 100644 index 00000000000..3cb2b1059ce --- /dev/null +++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/MemoryRetriever.java @@ -0,0 +1,63 @@ +package com.yahoo.vespa.hosted.node.verification.spec.retrievers; + +import com.yahoo.vespa.hosted.node.verification.commons.CommandExecutor; +import com.yahoo.vespa.hosted.node.verification.commons.OutputParser; +import com.yahoo.vespa.hosted.node.verification.commons.ParseInstructions; +import com.yahoo.vespa.hosted.node.verification.commons.ParseResult; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.logging.Level; +import java.util.logging.Logger; + +/** + * Created by olaa on 30/06/2017. + */ +public class MemoryRetriever implements HardwareRetriever { + + private static final String MEMORY_INFO_COMMAND = "cat /proc/meminfo"; + private static final String searchWord = "MemTotal"; + private static final String regexSplit = ":\\s"; + private static final int searchElementIndex = 0; + private static final int returnElementIndex = 1; + private static final Logger logger = Logger.getLogger(MemoryRetriever.class.getName()); + private final HardwareInfo hardwareInfo; + private final CommandExecutor commandExecutor; + + public MemoryRetriever(HardwareInfo hardwareInfo, CommandExecutor commandExecutor) { + this.hardwareInfo = hardwareInfo; + this.commandExecutor = commandExecutor; + } + + + public void updateInfo() { + try { + ArrayList<String> commandOutput = commandExecutor.executeCommand(MEMORY_INFO_COMMAND); + ParseResult parseResult = parseMemInfoFile(commandOutput); + updateMemoryInfo(parseResult); + } catch (IOException e) { + logger.log(Level.WARNING, "Failed to retrieve memory info", e); + } + } + + protected ParseResult parseMemInfoFile(ArrayList<String> commandOutput) { + ArrayList<String> searchWords = new ArrayList<>(Arrays.asList(searchWord)); + ParseInstructions parseInstructions = new ParseInstructions(searchElementIndex, returnElementIndex, regexSplit, searchWords); + ParseResult parseResult = OutputParser.parseSingleOutput(parseInstructions, commandOutput); + return parseResult; + } + + protected void updateMemoryInfo(ParseResult parseResult) { + double memory = convertKBToGB(parseResult.getValue()); + hardwareInfo.setMinMainMemoryAvailableGb(memory); + } + + protected double convertKBToGB(String totMem) { + String[] split = totMem.split(" "); + double value = Double.parseDouble(split[0]); + double kiloToGiga = 1000000.0; + return value / kiloToGiga; + } + +} diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/NetRetriever.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/NetRetriever.java new file mode 100644 index 00000000000..0d793bf416e --- /dev/null +++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/NetRetriever.java @@ -0,0 +1,112 @@ +package com.yahoo.vespa.hosted.node.verification.spec.retrievers; + +import com.yahoo.vespa.hosted.node.verification.commons.CommandExecutor; +import com.yahoo.vespa.hosted.node.verification.commons.OutputParser; +import com.yahoo.vespa.hosted.node.verification.commons.ParseInstructions; +import com.yahoo.vespa.hosted.node.verification.commons.ParseResult; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.logging.Level; +import java.util.logging.Logger; + +/** + * Created by olaa on 30/06/2017. + */ +public class NetRetriever implements HardwareRetriever { + private static final String NET_FIND_INTERFACE = "/sbin/ifconfig"; + private static final String NET_CHECK_INTERFACE_SPEED = "/sbin/ethtool"; + private static final String SEARCH_WORD_INTERFACE_IP4 = "inet"; + private static final String SEARCH_WORD_INTERFACE_IPV6 = "inet6"; + private static final String SEARCH_WORD_INTERFACE_NAME = "eth."; + private static final String SEARCH_WORD_INTERFACE_SPEED = "Speed"; + private static final String INTERFACE_NAME_REGEX_SPLIT = "\\s+"; + private static final String INTERFACE_NAME_SKIP_WORD = "lo"; + private static final String INTERFACE_NAME_SKIP_UNTIL_WORD = ""; + private static final int INTERFACE_NAME_SEARCH_ELEMENT_INDEX = 0; + private static final int INTERFACE_NAME_RETURN_ELEMENT_INDEX = 0; + private static final String INTERFACE_SPEED_REGEX_SPLIT = ":"; + private static final int INTERFACE_SPEED_SEARCH_ELEMENT_INDEX = 0; + private static final int INTERFACE_SPEED_RETURN_ELEMENT_INDEX = 1; + private static final Logger logger = Logger.getLogger(NetRetriever.class.getName()); + private final HardwareInfo hardwareInfo; + private final CommandExecutor commandExecutor; + + + public NetRetriever(HardwareInfo hardwareInfo, CommandExecutor commandExecutor) { + this.hardwareInfo = hardwareInfo; + this.commandExecutor = commandExecutor; + } + + public void updateInfo() { + try { + ArrayList<ParseResult> parseResults = findInterface(); + findInterfaceSpeed(parseResults); + updateHardwareInfoWithNet(parseResults); + } catch (IOException e) { + logger.log(Level.WARNING, "Failed to retrieve net info", e); + } + } + + protected ArrayList<ParseResult> findInterface() throws IOException { + ArrayList<String> commandOutput = commandExecutor.executeCommand(NET_FIND_INTERFACE); + ArrayList<ParseResult> parseResults = parseNetInterface(commandOutput); + return parseResults; + } + + protected void findInterfaceSpeed(ArrayList<ParseResult> parseResults) throws IOException { + String interfaceName = findInterfaceName(parseResults); + String command = NET_CHECK_INTERFACE_SPEED + " " + interfaceName; + ArrayList<String> commandOutput = commandExecutor.executeCommand(command); + parseResults.add(parseInterfaceSpeed(commandOutput)); + } + + protected ArrayList<ParseResult> parseNetInterface(ArrayList<String> commandOutput) { + ArrayList<String> searchWords = new ArrayList<>(Arrays.asList(SEARCH_WORD_INTERFACE_IP4, SEARCH_WORD_INTERFACE_IPV6, SEARCH_WORD_INTERFACE_NAME)); + ParseInstructions parseInstructions = new ParseInstructions(INTERFACE_NAME_SEARCH_ELEMENT_INDEX, INTERFACE_NAME_RETURN_ELEMENT_INDEX, INTERFACE_NAME_REGEX_SPLIT, searchWords); + parseInstructions.setSkipWord(INTERFACE_NAME_SKIP_WORD); + parseInstructions.setSkipUntilKeyword(INTERFACE_NAME_SKIP_UNTIL_WORD); + ArrayList<ParseResult> parseResults = OutputParser.parseOutPutWithSkips(parseInstructions, commandOutput); + return parseResults; + } + + protected ParseResult parseInterfaceSpeed(ArrayList<String> commandOutput) { + ArrayList<String> searchWords = new ArrayList<>(Arrays.asList(SEARCH_WORD_INTERFACE_SPEED)); + ParseInstructions parseInstructions = new ParseInstructions(INTERFACE_SPEED_SEARCH_ELEMENT_INDEX, INTERFACE_SPEED_RETURN_ELEMENT_INDEX, INTERFACE_SPEED_REGEX_SPLIT, searchWords); + ParseResult parseResult = OutputParser.parseSingleOutput(parseInstructions, commandOutput); + return parseResult; + } + + protected String findInterfaceName(ArrayList<ParseResult> parseResults) { + for (ParseResult parseResult : parseResults) { + if (!parseResult.getSearchWord().matches(SEARCH_WORD_INTERFACE_NAME)) continue; + return parseResult.getValue(); + } + return ""; + } + + protected void updateHardwareInfoWithNet(ArrayList<ParseResult> parseResults) { + hardwareInfo.setIpv6Connectivity(false); + hardwareInfo.setIpv4Connectivity(false); + for (ParseResult parseResult : parseResults) { + switch (parseResult.getSearchWord()) { + case SEARCH_WORD_INTERFACE_IP4: + hardwareInfo.setIpv4Connectivity(true); + break; + case SEARCH_WORD_INTERFACE_IPV6: + hardwareInfo.setIpv6Connectivity(true); + break; + case SEARCH_WORD_INTERFACE_SPEED: + String speedValue = parseResult.getValue().replaceAll("[^\\d.]", ""); + double speed = Double.parseDouble(speedValue); + hardwareInfo.setInterfaceSpeedMbs(speed); + break; + default: + if (parseResult.getSearchWord().matches(SEARCH_WORD_INTERFACE_NAME)) break; + throw new RuntimeException("Invalid ParseResult search word"); + } + } + } + +} diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/yamasreport/SpecReportDimensions.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/yamasreport/SpecReportDimensions.java new file mode 100644 index 00000000000..abf2f93023f --- /dev/null +++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/yamasreport/SpecReportDimensions.java @@ -0,0 +1,82 @@ +package com.yahoo.vespa.hosted.node.verification.spec.yamasreport; + +import com.fasterxml.jackson.annotation.JsonProperty; + +/** + * Created by olaa on 12/07/2017. + */ +public class SpecReportDimensions { + + + @JsonProperty + private boolean memoryMatch; + @JsonProperty + private boolean cpuCoresMatch; + @JsonProperty + private boolean fastDiskMatch; + @JsonProperty + private boolean netInterfaceSpeedMatch; + @JsonProperty + private boolean diskAvailableMatch; + @JsonProperty + private boolean ipv4Match; + @JsonProperty + private boolean ipv6Match; + + public boolean isNetInterfaceSpeedMatch() { + return netInterfaceSpeedMatch; + } + + public void setNetInterfaceSpeedMatch(boolean netInterfaceSpeedMatch) { + this.netInterfaceSpeedMatch = netInterfaceSpeedMatch; + } + + public boolean isMemoryMatch() { + return memoryMatch; + } + + public void setMemoryMatch(boolean memoryMatch) { + this.memoryMatch = memoryMatch; + } + + public boolean isCpuCoresMatch() { + return cpuCoresMatch; + } + + public void setCpuCoresMatch(boolean cpuCoresMatch) { + this.cpuCoresMatch = cpuCoresMatch; + } + + public boolean isFastDiskMatch() { + return fastDiskMatch; + } + + public void setFastDiskMatch(boolean fastDiskMatch) { + this.fastDiskMatch = fastDiskMatch; + } + + public boolean isDiskAvailableMatch() { + return diskAvailableMatch; + } + + public void setDiskAvailableMatch(boolean diskAvailableMatch) { + this.diskAvailableMatch = diskAvailableMatch; + } + + public boolean isIpv4Match() { + return ipv4Match; + } + + public void setIpv4Match(boolean ipv4Match) { + this.ipv4Match = ipv4Match; + } + + public boolean isIpv6Match() { + return ipv6Match; + } + + public void setIpv6Match(boolean ipv6Match) { + this.ipv6Match = ipv6Match; + } + +} diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/yamasreport/SpecReportMetrics.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/yamasreport/SpecReportMetrics.java new file mode 100644 index 00000000000..44332dd6aad --- /dev/null +++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/yamasreport/SpecReportMetrics.java @@ -0,0 +1,89 @@ +package com.yahoo.vespa.hosted.node.verification.spec.yamasreport; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; + +/** + * Created by olaa on 12/07/2017. + */ +@JsonInclude(JsonInclude.Include.NON_NULL) +public class SpecReportMetrics { + + @JsonProperty + private boolean match; + @JsonProperty + private Double expectedMemoryAvailable; + @JsonProperty + private Double actualMemoryAvailable; + @JsonProperty + private Boolean expectedFastDisk; + @JsonProperty + private Boolean actualFastDisk; + @JsonProperty + private Double expectedDiskSpaceAvailable; + @JsonProperty + private Double actualDiskSpaceAvailable; + @JsonProperty + private Double expectedInterfaceSpeed; + @JsonProperty + private Double actualInterfaceSpeed; + @JsonProperty + private Integer expectedcpuCores; + @JsonProperty + private Integer actualcpuCores; + @JsonProperty + private String[] faultyIpAddresses; + + public void setMatch(boolean match) { + this.match = match; + } + + public boolean isMatch() { + return this.match; + } + + public void setExpectedMemoryAvailable(Double expectedMemoryAvailable) { + this.expectedMemoryAvailable = expectedMemoryAvailable; + } + + public void setActualMemoryAvailable(Double actualMemoryAvailable) { + this.actualMemoryAvailable = actualMemoryAvailable; + } + + public void setExpectedDiskType(Boolean expectedFastDisk) { + this.expectedFastDisk = expectedFastDisk; + } + + public void setActualDiskType(Boolean actualFastDisk) { + this.actualFastDisk = actualFastDisk; + } + + public void setExpectedDiskSpaceAvailable(Double expectedDiskSpaceAvailable) { + this.expectedDiskSpaceAvailable = expectedDiskSpaceAvailable; + } + + public void setActualDiskSpaceAvailable(Double actualDiskSpaceAvailable) { + this.actualDiskSpaceAvailable = actualDiskSpaceAvailable; + } + + public void setExpectedcpuCores(int expectedcpuCores) { + this.expectedcpuCores = expectedcpuCores; + } + + public void setActualcpuCores(int actualcpuCores) { + this.actualcpuCores = actualcpuCores; + } + + public void setExpectedInterfaceSpeed(Double expectedInterfaceSpeed) { + this.expectedInterfaceSpeed = expectedInterfaceSpeed; + } + + public void setActualInterfaceSpeed(Double actualInterfaceSpeed) { + this.actualInterfaceSpeed = actualInterfaceSpeed; + } + + public void setFaultyIpAddresses(String[] faultyIpAddresses) { + this.faultyIpAddresses = faultyIpAddresses; + } + +} diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/yamasreport/YamasSpecReport.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/yamasreport/YamasSpecReport.java new file mode 100644 index 00000000000..c9c2857bbea --- /dev/null +++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/yamasreport/YamasSpecReport.java @@ -0,0 +1,77 @@ +package com.yahoo.vespa.hosted.node.verification.spec.yamasreport; + +import com.fasterxml.jackson.annotation.JsonAnyGetter; +import com.fasterxml.jackson.annotation.JsonAnySetter; +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.util.HashMap; +import java.util.Map; + +/** + * Created by olaa on 12/07/2017. + */ +public class YamasSpecReport { + + @JsonProperty + private long timeStamp; + @JsonProperty + private SpecReportDimensions dimensions; + @JsonProperty + private SpecReportMetrics metrics; + @JsonProperty + private JsonObjectWrapper routing; + + public YamasSpecReport() { + this.timeStamp = System.currentTimeMillis() / 1000L; + setRouting(); + } + + public void setDimensions(SpecReportDimensions dimensions) { + this.dimensions = dimensions; + } + + public SpecReportDimensions getDimensions() { + return this.dimensions; + } + + public void setMetrics(SpecReportMetrics metrics) { + this.metrics = metrics; + } + + public void setFaultyIpAddresses(String[] faultyIpAddresses) { + this.metrics.setFaultyIpAddresses(faultyIpAddresses); + } + + public SpecReportMetrics getMetrics() { + return this.metrics; + } + + public long getTimeStamp() { + return this.timeStamp; + } + + private void setRouting() { + JsonObjectWrapper wrap = new JsonObjectWrapper("namespace", new String[]{"Vespa"}); + routing = new JsonObjectWrapper("yamas", wrap); + } + + class JsonObjectWrapper<T> { + + private Map<String, T> wrappedObjects = new HashMap<String, T>(); + + public JsonObjectWrapper(String name, T wrappedObject) { + this.wrappedObjects.put(name, wrappedObject); + } + + @JsonAnyGetter + public Map<String, T> any() { + return wrappedObjects; + } + + @JsonAnySetter + public void set(String name, T value) { + wrappedObjects.put(name, value); + } + } + +} diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/commons/CommandExecutorTest.java b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/commons/CommandExecutorTest.java new file mode 100644 index 00000000000..ffdf5e0a124 --- /dev/null +++ b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/commons/CommandExecutorTest.java @@ -0,0 +1,33 @@ +package com.yahoo.vespa.hosted.node.verification.commons; + +import org.junit.Before; +import org.junit.Test; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import static java.util.Arrays.asList; +import static org.junit.Assert.assertEquals; + +/** + * Created by sgrostad on 12/07/2017. + */ +public class CommandExecutorTest { + + private CommandExecutor commandExecutor; + + @Before + public void setup() { + commandExecutor = new CommandExecutor(); + } + + @Test + public void test_if_executeAString_reads_testReadFile_correct() throws IOException { + String command = "cat src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/resources/testReadFile.txt"; + ArrayList<String> commandOutput = commandExecutor.executeCommand(command); + List<String> expectedOutput = asList("This test file tests apache commons exec", "Second line"); + assertEquals(expectedOutput, commandOutput); + } + +}
\ No newline at end of file diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/commons/OutputParserTest.java b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/commons/OutputParserTest.java new file mode 100644 index 00000000000..056ee2cc5c4 --- /dev/null +++ b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/commons/OutputParserTest.java @@ -0,0 +1,109 @@ +package com.yahoo.vespa.hosted.node.verification.commons; + +import com.yahoo.vespa.hosted.node.verification.mock.MockCommandExecutor; +import org.junit.Before; +import org.junit.Test; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; + +import static org.junit.Assert.assertEquals; + +/** + * Created by sgrostad on 21/07/2017. + */ +public class OutputParserTest { + + private static final String RETURN_VALUE = "#returnValue#"; + private static final String OUTPUT_WITH_MATCH_1 = "This; Should be; a match; when; Parsing ; " + RETURN_VALUE; + private static final String OUTPUT_WITHOUT_MATCH = "This; is; not a; match"; + private static final String OUTPUT_WITH_MATCH_2 = "But; thiS will-also; be; a match:; this ; " + RETURN_VALUE; + private static final String SEARCH_WORD_1 = "Parsing"; + private static final String SEARCH_WORD_2 = "this"; + private static final String REGEX_SEARCH_WORD = ".*S.*"; + private static final String PARSE_OUTPUT_WITH_SKIPS_TEST_FILE = "src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/parseOutputWithSkipsTest"; + private ArrayList<String> commandOutput; + private ArrayList<String> searchWords; + + @Before + public void setup() { + commandOutput = new ArrayList<>(Arrays.asList(OUTPUT_WITH_MATCH_1, OUTPUT_WITHOUT_MATCH, OUTPUT_WITH_MATCH_2)); + } + + @Test + public void parseOutput_searching_for_two_normal_words() { + searchWords = new ArrayList<>(Arrays.asList(SEARCH_WORD_1, SEARCH_WORD_2)); + ParseInstructions parseInstructions = new ParseInstructions(6, 8, " ", searchWords); + ArrayList<ParseResult> parseResults = OutputParser.parseOutput(parseInstructions, commandOutput); + ParseResult expectedParseResult1 = new ParseResult(SEARCH_WORD_1, RETURN_VALUE); + ParseResult expectedParseResult2 = new ParseResult(SEARCH_WORD_2, RETURN_VALUE); + assertEquals(expectedParseResult1, parseResults.get(0)); + assertEquals(expectedParseResult2, parseResults.get(1)); + } + + @Test + public void parseOutput_searching_for_two_normal_words_with_semicolon_as_line_split() { + searchWords = new ArrayList<>(Arrays.asList(SEARCH_WORD_1, SEARCH_WORD_2)); + ParseInstructions parseInstructions = new ParseInstructions(4, 5, ";", searchWords); + ArrayList<ParseResult> parseResults = OutputParser.parseOutput(parseInstructions, commandOutput); + ParseResult expectedParseResult1 = new ParseResult(SEARCH_WORD_1, RETURN_VALUE); + ParseResult expectedParseResult2 = new ParseResult(SEARCH_WORD_2, RETURN_VALUE); + assertEquals(expectedParseResult1, parseResults.get(0)); + assertEquals(expectedParseResult2, parseResults.get(1)); + } + + @Test + public void parseOutput_searching_for_word_containing_capital_s() { + searchWords = new ArrayList<>(Arrays.asList(REGEX_SEARCH_WORD)); + ParseInstructions parseInstructions = new ParseInstructions(1, 8, " ", searchWords); + ArrayList<ParseResult> parseResults = OutputParser.parseOutput(parseInstructions, commandOutput); + ParseResult expectedParseResult1 = new ParseResult("Should", RETURN_VALUE); + ParseResult expectedParseResult2 = new ParseResult("thiS", RETURN_VALUE); + assertEquals(expectedParseResult1, parseResults.get(0)); + assertEquals(expectedParseResult2, parseResults.get(1)); + } + + @Test + public void parseSingleOutput_should_return_first_match() { + searchWords = new ArrayList<>(Arrays.asList(SEARCH_WORD_1)); + ParseInstructions parseInstructions = new ParseInstructions(6, 8, " ", searchWords); + ParseResult parseResult = OutputParser.parseSingleOutput(parseInstructions, commandOutput); + ParseResult expectedParseResult = new ParseResult(SEARCH_WORD_1, RETURN_VALUE); + assertEquals(expectedParseResult, parseResult); + } + + @Test + public void parseSingleOutput_should_return_invalid_parseResult() { + searchWords = new ArrayList<>(Arrays.asList("No match")); + ParseInstructions parseInstructions = new ParseInstructions(6, 8, " ", searchWords); + ParseResult parseResult = OutputParser.parseSingleOutput(parseInstructions, commandOutput); + ParseResult expectedParseResult = new ParseResult("invalid", "invalid"); + assertEquals(expectedParseResult, parseResult); + } + + @Test + public void parseOutputWithSkips_should_return_two_matches() throws IOException { + searchWords = new ArrayList<>(Arrays.asList(SEARCH_WORD_1)); + ParseInstructions parseInstructions = new ParseInstructions(1, 7, " ", searchWords); + parseInstructions.setSkipWord("SkipFromKeyword"); + parseInstructions.setSkipUntilKeyword("skipUntilKeyword"); + ArrayList<String> commandSkipOutput = MockCommandExecutor.readFromFile(PARSE_OUTPUT_WITH_SKIPS_TEST_FILE); + ArrayList<ParseResult> parseResults = OutputParser.parseOutPutWithSkips(parseInstructions, commandSkipOutput); + ParseResult expectedParseResult = new ParseResult(SEARCH_WORD_1, RETURN_VALUE); + assertEquals(expectedParseResult, parseResults.get(0)); + assertEquals(expectedParseResult, parseResults.get(1)); + } + + @Test + public void skipToIndex_should_return_correct_index() throws IOException { + searchWords = new ArrayList<>(Arrays.asList(SEARCH_WORD_1)); + ParseInstructions parseInstructions = new ParseInstructions(0, 0, " ", searchWords); + parseInstructions.setSkipUntilKeyword("skipUntilKeyword"); + ArrayList<String> commandSkipOutput = MockCommandExecutor.readFromFile(PARSE_OUTPUT_WITH_SKIPS_TEST_FILE); + int indexReturned = OutputParser.skipToIndex(3, parseInstructions, commandSkipOutput); + int expectedReturnIndex = 10; + assertEquals(expectedReturnIndex, indexReturned); + } + +}
\ No newline at end of file diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/CPUBenchmarkTest.java b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/CPUBenchmarkTest.java new file mode 100644 index 00000000000..7fab0d62619 --- /dev/null +++ b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/CPUBenchmarkTest.java @@ -0,0 +1,114 @@ +package com.yahoo.vespa.hosted.node.verification.hardware.benchmarks; + + +import com.yahoo.vespa.hosted.node.verification.commons.ParseResult; +import com.yahoo.vespa.hosted.node.verification.mock.MockCommandExecutor; +import org.junit.Before; +import org.junit.Test; + +import java.io.IOException; +import java.util.ArrayList; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.assertEquals; + +/** + * Created by sgrostad on 11/07/2017. + */ +public class CPUBenchmarkTest { + + private static final String cpuEuropeanDelimiters = "src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/resources/cpuCyclesWithDotsTimeWithCommaTest.txt"; + private static final String cpuAlternativeDelimiters = "src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/resources/cpuCyclesWithCommasTimeWithDotTest.txt"; + private static final String cpuWrongOutput = "src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/resources/cpuWrongOutputTest.txt"; + private HardwareResults hardwareResults; + private MockCommandExecutor commandExecutor; + private CPUBenchmark cpu; + private static final double DELTA = 0.1; + + @Before + public void setup() { + commandExecutor = new MockCommandExecutor(); + hardwareResults = new HardwareResults(); + cpu = new CPUBenchmark(hardwareResults, commandExecutor); + } + + @Test + public void doBenchmark_find_correct_cpuCyclesPerSec() { + String mockCommand = "cat " + cpuAlternativeDelimiters; + commandExecutor.addCommand(mockCommand); + cpu.doBenchmark(); + double result = hardwareResults.getCpuCyclesPerSec(); + double expected = 2.1576482291815062; + assertEquals(expected, result, DELTA); + } + + @Test + public void doBenchmark_wrong_output_stores_frequency_of_zero() { + String mockCommand = "cat " + cpuWrongOutput; + commandExecutor.addCommand(mockCommand); + cpu.doBenchmark(); + double result = hardwareResults.getCpuCyclesPerSec(); + double expected = 0; + assertEquals(expected, result, DELTA); + } + + @Test + public void parseCpuCyclesPerSec_return_correct_ArrayList() throws IOException { + ArrayList<String> mockCommandOutput = commandExecutor.readFromFile(cpuEuropeanDelimiters); + ArrayList<ParseResult> parseResults = cpu.parseCpuCyclesPerSec(mockCommandOutput); + ParseResult expectedParseCyclesResult = new ParseResult("cycles", "2.066.201.729"); + ParseResult expectedParseSecondsResult = new ParseResult("seconds", "0,957617512"); + assertEquals(expectedParseCyclesResult, parseResults.get(0)); + assertEquals(expectedParseSecondsResult, parseResults.get(1)); + } + + @Test + public void test_if_setCpuCyclesPerSec_reads_output_correctly() throws IOException { + ArrayList<ParseResult> parseResults = new ArrayList<>(); + parseResults.add(new ParseResult("cycles", "2.066.201.729")); + parseResults.add(new ParseResult("seconds", "0,957617512")); + cpu.setCpuCyclesPerSec(parseResults); + double expectedCpuCyclesPerSec = 2.1576482291815062; + assertEquals(expectedCpuCyclesPerSec, hardwareResults.getCpuCyclesPerSec(), DELTA); + } + + @Test + public void test_if_makeCyclesDouble_converts_European_and_alternative_delimiters_correctly() { + String toBeConvertedEuropean = "2.066.201.729"; + String toBEConvertedAlternative = "2,066,201,729"; + double expectedCycles = 2066201729; + assertEquals(expectedCycles, cpu.makeCyclesDouble(toBeConvertedEuropean), DELTA); + assertEquals(expectedCycles, cpu.makeCyclesDouble(toBEConvertedAlternative), DELTA); + } + + @Test + public void test_if_makeSecondsDouble_converts_European_and_alternative_delimiters_correctly() { + String toBeConvertedEuropean = "0,957617512"; + String toBEConvertedAlternative = "0.957617512"; + double expectedSeconds = 0.957617512; + assertEquals(expectedSeconds, cpu.makeSecondsDouble(toBeConvertedEuropean), DELTA); + assertEquals(expectedSeconds, cpu.makeSecondsDouble(toBEConvertedAlternative), DELTA); + } + + @Test + public void test_if_checkIfNumber_returns_true() { + String number = "125.5"; + assertTrue(cpu.checkIfNumber(number)); + } + + @Test + public void test_if_checkIfNumber_returns_false() { + String notANumber = "125.5a"; + assertFalse(cpu.checkIfNumber(notANumber)); + } + + @Test + public void test_if_convertToGHz_converts_correctly() { + double cycles = 2066201729; + double seconds = 0.957617512; + double expectedGHz = 2.1576482291815062; + assertEquals(expectedGHz, cpu.convertToGHz(cycles, seconds), DELTA); + } + +}
\ No newline at end of file diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/DiskBenchmarkTest.java b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/DiskBenchmarkTest.java new file mode 100644 index 00000000000..9249c4e9876 --- /dev/null +++ b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/DiskBenchmarkTest.java @@ -0,0 +1,131 @@ +package com.yahoo.vespa.hosted.node.verification.hardware.benchmarks; + +import com.yahoo.vespa.hosted.node.verification.commons.ParseResult; +import com.yahoo.vespa.hosted.node.verification.mock.MockCommandExecutor; +import org.junit.Before; +import org.junit.Test; + +import java.util.ArrayList; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.assertEquals; + +/** + * Created by sgrostad on 12/07/2017. + */ +public class DiskBenchmarkTest { + + private DiskBenchmark diskBenchmark; + private HardwareResults hardwareResults; + private MockCommandExecutor commandExecutor; + private static final String VALID_OUTPUT_FILE = "src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/resources/diskBenchmarkValidOutput"; + private static final String INVALID_OUTPUT_FILE = "src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/resources/diskBenchmarkInvalidOutput"; + private static final double DELTA = 0.1; + + @Before + public void setup() { + commandExecutor = new MockCommandExecutor(); + hardwareResults = new HardwareResults(); + diskBenchmark = new DiskBenchmark(hardwareResults, commandExecutor); + } + + @Test + public void doBenchmark_should_store_diskSpeed_when_valid_output() { + String mockCommand = "cat " + VALID_OUTPUT_FILE; + commandExecutor.addCommand(mockCommand); + diskBenchmark.doBenchmark(); + double expectedSpeed = 243; + double actualSpeed = hardwareResults.getDiskSpeedMbs(); + assertEquals(expectedSpeed, actualSpeed, DELTA); + } + + @Test + public void doBenchmark_should_store_diskSpeed_as_zero_when_invalid_output() { + String mockCommand = "cat " + INVALID_OUTPUT_FILE; + commandExecutor.addCommand(mockCommand); + diskBenchmark.doBenchmark(); + double expectedSpeed = 0; + double actualSpeed = hardwareResults.getDiskSpeedMbs(); + assertEquals(expectedSpeed, actualSpeed, DELTA); + } + + + @Test + public void parseDiskSpeed_valid_input() throws Exception { + ArrayList<String> mockCommandOutput = commandExecutor.readFromFile(VALID_OUTPUT_FILE); + ParseResult parseResult = diskBenchmark.parseDiskSpeed(mockCommandOutput); + ParseResult expectedParseResult = new ParseResult("MB/s", "243"); + assertEquals(expectedParseResult, parseResult); + } + + @Test + public void parseDiskSpeed_invalid_input() throws Exception { + ArrayList<String> mockCommandOutput = commandExecutor.readFromFile(INVALID_OUTPUT_FILE); + ParseResult parseResult = diskBenchmark.parseDiskSpeed(mockCommandOutput); + ParseResult expectedParseResult = new ParseResult("invalid", "invalid"); + assertEquals(expectedParseResult, parseResult); + } + + @Test + public void setDiskSpeed_valid_input() { + ParseResult parseResult = new ParseResult("MB/s", "243"); + diskBenchmark.setDiskSpeed(parseResult); + double expectedDiskSpeed = 243; + assertEquals(expectedDiskSpeed, hardwareResults.getDiskSpeedMbs(), DELTA); + } + + @Test + public void setDiskSpeed_invalid_input() { + ParseResult parseResult = new ParseResult("invalid", "invalid"); + diskBenchmark.setDiskSpeed(parseResult); + double expectedDiskSpeed = 0; + assertEquals(expectedDiskSpeed, hardwareResults.getDiskSpeedMbs(), DELTA); + } + + @Test + public void getDiskSpeedInMBs_for_KBs_MBs_and_GBs() { + ParseResult KBsParseResult = new ParseResult("kB/s", "243000"); + ParseResult MBsParseResult = new ParseResult("MB/s", "243"); + ParseResult GBsParseResult = new ParseResult("GB/s", "0.243"); + double expectedMBs = 243; + assertEquals(expectedMBs, diskBenchmark.getDiskSpeedInMBs(KBsParseResult), DELTA); + assertEquals(expectedMBs, diskBenchmark.getDiskSpeedInMBs(MBsParseResult), DELTA); + assertEquals(expectedMBs, diskBenchmark.getDiskSpeedInMBs(GBsParseResult), DELTA); + } + + @Test + public void ckeckSpeedValidity_should_return_true_for_valid_format() { + String speed = "123"; + assertTrue(diskBenchmark.checkSpeedValidity(speed)); + speed = "30000"; + assertTrue(diskBenchmark.checkSpeedValidity(speed)); + speed = "6"; + assertTrue(diskBenchmark.checkSpeedValidity(speed)); + } + + @Test + public void ckeckSpeedValidity_should_return_false_for_valid_format() { + String speed = "124 GHz"; + assertFalse(diskBenchmark.checkSpeedValidity(speed)); + speed = null; + assertFalse(diskBenchmark.checkSpeedValidity(speed)); + speed = "This should return false as well"; + assertFalse(diskBenchmark.checkSpeedValidity(speed)); + } + + @Test + public void convertToMbs_should_return_properly_converted_disk_speeds() { + String speed = "1234"; + double factor = 1000; + double expectedSpeed = 1234000; + assertEquals(expectedSpeed, diskBenchmark.convertToMBs(speed, factor), DELTA); + factor = 1 / 1000.0; + expectedSpeed = 1.234; + assertEquals(expectedSpeed, diskBenchmark.convertToMBs(speed, factor), DELTA); + factor = 1; + expectedSpeed = 1234; + assertEquals(expectedSpeed, diskBenchmark.convertToMBs(speed, factor), DELTA); + } + +}
\ No newline at end of file diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/MemoryBenchmarkTest.java b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/MemoryBenchmarkTest.java new file mode 100644 index 00000000000..0f6d515a645 --- /dev/null +++ b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/MemoryBenchmarkTest.java @@ -0,0 +1,114 @@ +package com.yahoo.vespa.hosted.node.verification.hardware.benchmarks; + +import com.yahoo.vespa.hosted.node.verification.commons.ParseResult; +import com.yahoo.vespa.hosted.node.verification.mock.MockCommandExecutor; +import org.junit.Before; +import org.junit.Test; + +import java.util.ArrayList; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; + +/** + * Created by olaa on 14/07/2017. + */ +public class MemoryBenchmarkTest { + + private MemoryBenchmark memoryBenchmark; + private HardwareResults hardwareResults; + private MockCommandExecutor commandExecutor; + private static final double DELTA = 0.1; + + @Before + public void setup() { + hardwareResults = new HardwareResults(); + commandExecutor = new MockCommandExecutor(); + memoryBenchmark = new MemoryBenchmark(hardwareResults, commandExecutor); + } + + @Test + public void doBenchMark_should_update_read_and_write_memory_speed() { + commandExecutor.addDummyCommand(); + commandExecutor.addDummyCommand(); + commandExecutor.addCommand("cat src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/resources/validMemoryWriteSpeed"); + commandExecutor.addCommand("cat src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/resources/validMemoryReadSpeed"); + commandExecutor.addDummyCommand(); + commandExecutor.addDummyCommand(); + memoryBenchmark.doBenchmark(); + double expectedReadSpeed = 5.9; + double expectedWriteSpeed = 3.4; + assertEquals(expectedReadSpeed, hardwareResults.getMemoryReadSpeedGBs(), DELTA); + assertEquals(expectedWriteSpeed, hardwareResults.getMemoryWriteSpeedGBs(), DELTA); + } + + @Test + public void parseMemorySpeed_valid_output() throws Exception { + Double expectedSpeed = 12.1; + String mockOutput = "This is a test \n the memory speed to be found is " + expectedSpeed + " GB/s"; + ArrayList<String> mockCommandOutput = commandExecutor.outputFromString(mockOutput); + ParseResult parseResult = memoryBenchmark.parseMemorySpeed(mockCommandOutput); + ParseResult expectedParseResult = new ParseResult("GB/s", expectedSpeed.toString()); + assertEquals(expectedParseResult, parseResult); + } + + @Test + public void parseMemorySpeed_invalid_output() throws Exception { + ArrayList<String> mockCommandOutput = commandExecutor.outputFromString(""); + ParseResult parseResult = memoryBenchmark.parseMemorySpeed(mockCommandOutput); + ParseResult expectedParseResult = new ParseResult("invalid", "invalid"); + assertEquals(expectedParseResult, parseResult); + mockCommandOutput = commandExecutor.outputFromString("Exit status 1"); + parseResult = memoryBenchmark.parseMemorySpeed(mockCommandOutput); + assertEquals(expectedParseResult, parseResult); + } + + @Test + public void memoryReadSpeed_valid_input_should_update_hardwareResults() { + Double expectedMemoryReadSpeed = 12.1; + memoryBenchmark.updateMemoryReadSpeed(expectedMemoryReadSpeed.toString()); + assertEquals(expectedMemoryReadSpeed, hardwareResults.getMemoryReadSpeedGBs(), DELTA); + } + + @Test + public void memoryReadSpeed_invalid_input_should_not_update_hardwareResults() { + memoryBenchmark.updateMemoryReadSpeed("Invalid speed"); + assertNull(hardwareResults.getMemoryReadSpeedGBs()); + } + + @Test + public void memoryWriteSpeed_valid_input_should_update_hardwareResults() { + Double expectedMemoryWriteSpeed = 3.8; + memoryBenchmark.updateMemoryWriteSpeed(expectedMemoryWriteSpeed.toString()); + assertEquals(expectedMemoryWriteSpeed, hardwareResults.getMemoryWriteSpeedGBs(), DELTA); + } + + @Test + public void memoryWriteSpeed_invalid_input_should_not_update_hardwareResults() { + memoryBenchmark.updateMemoryWriteSpeed("Invalid speed"); + assertNull(hardwareResults.getMemoryWriteSpeedGBs()); + } + + @Test + public void isValidMemory_should_return_true_when_parameter_is_number() { + String benchmarkOutput = "6.32"; + boolean validMemory = memoryBenchmark.isValidMemory(benchmarkOutput); + assertTrue(validMemory); + } + + @Test + public void isValidMemory_should_return_false_when_parameter_is_not_number() { + String benchmarkOutput = ""; + boolean validMemory = memoryBenchmark.isValidMemory(benchmarkOutput); + assertFalse(validMemory); + benchmarkOutput = null; + validMemory = memoryBenchmark.isValidMemory(benchmarkOutput); + assertFalse(validMemory); + benchmarkOutput = "Exit status 127"; + validMemory = memoryBenchmark.isValidMemory(benchmarkOutput); + assertFalse(validMemory); + } + +}
\ No newline at end of file diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/NetBenchmarkTest.java b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/NetBenchmarkTest.java new file mode 100644 index 00000000000..c737baa2b0b --- /dev/null +++ b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/NetBenchmarkTest.java @@ -0,0 +1,97 @@ +package com.yahoo.vespa.hosted.node.verification.hardware.benchmarks; + +import com.yahoo.vespa.hosted.node.verification.commons.ParseResult; +import com.yahoo.vespa.hosted.node.verification.mock.MockCommandExecutor; +import org.junit.Before; +import org.junit.Test; + +import java.io.IOException; +import java.util.ArrayList; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.assertEquals; + +/** + * Created by olaa on 11/07/2017. + */ +public class NetBenchmarkTest { + + private HardwareResults hardwareResults; + private NetBenchmark netBenchmark; + private MockCommandExecutor commandExecutor; + private static String VALID_PING_RESPONSE = "src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/resources/validpingresponse"; + private static String INVALID_PING_RESPONSE = "src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/resources/invalidpingresponse"; + private static String CRAZY_PING_RESPONSE = "src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/resources/crazypingresponse"; + + @Before + public void setup() { + hardwareResults = new HardwareResults(); + commandExecutor = new MockCommandExecutor(); + netBenchmark = new NetBenchmark(hardwareResults, commandExecutor); + } + + @Test + public void doBenchmark_should_update_hardwareResults_to_true() { + String mockCommand = "cat " + VALID_PING_RESPONSE; + commandExecutor.addCommand(mockCommand); + netBenchmark.doBenchmark(); + assertTrue(hardwareResults.isIpv6Connectivity()); + } + + @Test + public void doBenchmark_should_update_hardwareResults_to_false_1() { + String mockCommand = "cat " + INVALID_PING_RESPONSE; + commandExecutor.addCommand(mockCommand); + netBenchmark.doBenchmark(); + assertFalse(hardwareResults.isIpv6Connectivity()); + } + + @Test + public void doBenchmark_should_update_hardwareResults_to_false_2() { + String mockCommand = "cat " + CRAZY_PING_RESPONSE; + commandExecutor.addCommand(mockCommand); + netBenchmark.doBenchmark(); + assertFalse(hardwareResults.isIpv6Connectivity()); + } + + @Test + public void parsePingResponse_valid_ping_response_should_return_ipv6_connectivity() throws IOException { + String command = "src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/resources/validpingresponse"; + ArrayList<String> mockCommandOutput = commandExecutor.readFromFile(command); + ParseResult parseResult = netBenchmark.parsePingResponse(mockCommandOutput); + String expectedPing = "0%"; + assertEquals(expectedPing, parseResult.getValue()); + } + + @Test + public void parsePingResponse_invalid_ping_response_should_return_invalid_ParseResult() throws IOException { + String command = "src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/resources/crazypingresponse"; + ArrayList<String> mockCommandOutput = commandExecutor.readFromFile(command); + ParseResult parseResult = netBenchmark.parsePingResponse(mockCommandOutput); + ParseResult expectedParseResult = new ParseResult("invalid", "invalid"); + assertEquals(expectedParseResult, parseResult); + } + + @Test + public void setIpv6Connectivity_valid_ping_response_should_return_ipv6_connectivity() { + ParseResult parseResult = new ParseResult("loss,", "0%"); + netBenchmark.setIpv6Connectivity(parseResult); + assertTrue(hardwareResults.isIpv6Connectivity()); + } + + @Test + public void setIpv6Connectivity_invalid_ping_response_should_return_no_ipv6_connectivity_1() { + ParseResult parseResult = new ParseResult("loss,", "100%"); + netBenchmark.setIpv6Connectivity(parseResult); + assertFalse(hardwareResults.isIpv6Connectivity()); + } + + @Test + public void setIpv6Connectivity_invalid_ping_response_should_return_no_ipv6_connectivity_2() { + ParseResult parseResult = new ParseResult("loss,", "invalid"); + netBenchmark.setIpv6Connectivity(parseResult); + assertFalse(hardwareResults.isIpv6Connectivity()); + } + +}
\ No newline at end of file diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/resources/cpuCyclesWithCommasTimeWithDotTest.txt b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/resources/cpuCyclesWithCommasTimeWithDotTest.txt new file mode 100644 index 00000000000..08782ef065f --- /dev/null +++ b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/resources/cpuCyclesWithCommasTimeWithDotTest.txt @@ -0,0 +1,2 @@ + 2,066,201,729 cycles + 0.957617512 seconds time elapsed diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/resources/cpuCyclesWithDotsTimeWithCommaTest.txt b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/resources/cpuCyclesWithDotsTimeWithCommaTest.txt new file mode 100644 index 00000000000..fd58bfaf8c7 --- /dev/null +++ b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/resources/cpuCyclesWithDotsTimeWithCommaTest.txt @@ -0,0 +1,2 @@ + 2.066.201.729 cycles + 0,957617512 seconds time elapsed diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/resources/cpuWrongOutputTest.txt b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/resources/cpuWrongOutputTest.txt new file mode 100644 index 00000000000..61a4c8fb2a2 --- /dev/null +++ b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/resources/cpuWrongOutputTest.txt @@ -0,0 +1,2 @@ + Dummy 2.066.201.729 cycles + 0,957617512 seconds time elapsed diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/resources/crazypingresponse b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/resources/crazypingresponse new file mode 100644 index 00000000000..d111ac34ce1 --- /dev/null +++ b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/resources/crazypingresponse @@ -0,0 +1 @@ +this is no pingresponse ?!!? 34234
\ No newline at end of file diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/resources/diskBenchmarkInvalidOutput b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/resources/diskBenchmarkInvalidOutput new file mode 100644 index 00000000000..4b3d2c6edfb --- /dev/null +++ b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/resources/diskBenchmarkInvalidOutput @@ -0,0 +1,5 @@ +dd: bs: illegal numeric value + +real 0m0.074s +user 0m0.002s +sys 0m0.058s
\ No newline at end of file diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/resources/diskBenchmarkValidOutput b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/resources/diskBenchmarkValidOutput new file mode 100644 index 00000000000..26778d9dd9a --- /dev/null +++ b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/resources/diskBenchmarkValidOutput @@ -0,0 +1,5 @@ +243 MB/s + +real 0m1.448s +user 0m0.000s +sys 0m0.260s
\ No newline at end of file diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/resources/invalidpingresponse b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/resources/invalidpingresponse new file mode 100644 index 00000000000..0e6a67d3d89 --- /dev/null +++ b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/resources/invalidpingresponse @@ -0,0 +1 @@ +6 packets transmitted, 6 received, 100% packet loss, time 5002ms diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/resources/testReadFile.txt b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/resources/testReadFile.txt new file mode 100644 index 00000000000..28858c30a70 --- /dev/null +++ b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/resources/testReadFile.txt @@ -0,0 +1,2 @@ +This test file tests apache commons exec +Second line
\ No newline at end of file diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/resources/validMemoryReadSpeed b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/resources/validMemoryReadSpeed new file mode 100644 index 00000000000..26657cea9e0 --- /dev/null +++ b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/resources/validMemoryReadSpeed @@ -0,0 +1,3 @@ +512+0 records in +512+0 records out +536870912 bytes (537 MB) copied, 0.0904486 s, 5.9 GB/s
\ No newline at end of file diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/resources/validMemoryWriteSpeed b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/resources/validMemoryWriteSpeed new file mode 100644 index 00000000000..ab4ccc986f8 --- /dev/null +++ b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/resources/validMemoryWriteSpeed @@ -0,0 +1,3 @@ +512+0 records in +512+0 records out +536870912 bytes (537 MB) copied, 0.158612 s, 3.4 GB/s
\ No newline at end of file diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/resources/validpingresponse b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/resources/validpingresponse new file mode 100644 index 00000000000..f6ef0559571 --- /dev/null +++ b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/resources/validpingresponse @@ -0,0 +1 @@ +6 packets transmitted, 6 received, 0% packet loss, time 5002ms diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/mock/MockCommandExecutor.java b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/mock/MockCommandExecutor.java new file mode 100644 index 00000000000..5b0da7d1948 --- /dev/null +++ b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/mock/MockCommandExecutor.java @@ -0,0 +1,48 @@ +package com.yahoo.vespa.hosted.node.verification.mock; + +import com.yahoo.vespa.hosted.node.verification.commons.CommandExecutor; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Arrays; + +/** + * Created by olaa on 17/07/2017. + */ +public class MockCommandExecutor extends CommandExecutor { + + private ArrayList<String> mockCommands; + private int counter; + public static final String DUMMY_COMMAND = "DUMMY"; + + public MockCommandExecutor() { + mockCommands = new ArrayList<>(); + counter = 0; + } + + @Override + public ArrayList<String> executeCommand(String command) throws IOException { + String mockCommand = mockCommands.get(counter++); + if (mockCommand.equals(DUMMY_COMMAND)) return null; + return super.executeCommand(mockCommand); + } + + public void addCommand(String command) { + mockCommands.add(command); + } + + public void addDummyCommand() { + mockCommands.add(DUMMY_COMMAND); + } + + public static ArrayList<String> readFromFile(String filepath) throws IOException { + return new ArrayList<>(Arrays.asList(new String(Files.readAllBytes(Paths.get(filepath))).split("\n"))); + } + + public ArrayList<String> outputFromString(String output) { + return new ArrayList<>(Arrays.asList(output.split("\n"))); + } + +} diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/HardwareNodeComparatorTest.java b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/HardwareNodeComparatorTest.java new file mode 100644 index 00000000000..0a06ac05c4b --- /dev/null +++ b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/HardwareNodeComparatorTest.java @@ -0,0 +1,45 @@ +package com.yahoo.vespa.hosted.node.verification.spec; + +import com.yahoo.vespa.hosted.node.verification.spec.retrievers.HardwareInfo; +import org.junit.Before; +import org.junit.Test; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +/** + * Created by olaa on 07/07/2017. + */ +public class HardwareNodeComparatorTest { + + private HardwareInfo actualHardware; + private HardwareInfo nodeInfo; + + @Before + public void setup() { + actualHardware = new HardwareInfo(); + nodeInfo = new HardwareInfo(); + actualHardware.setMinCpuCores(24); + nodeInfo.setMinCpuCores(24); + actualHardware.setMinMainMemoryAvailableGb(16); + nodeInfo.setMinMainMemoryAvailableGb(16); + nodeInfo.setInterfaceSpeedMbs(10000); + actualHardware.setInterfaceSpeedMbs(10000); + actualHardware.setMinDiskAvailableGb(500); + nodeInfo.setMinDiskAvailableGb(500); + } + + @Test + public void compare_should_be_equal() { + assertTrue(HardwareNodeComparator.compare(nodeInfo, actualHardware).getMetrics().isMatch()); + + } + + @Test + public void compare_different_amount_of_cores_should_be_false() { + actualHardware.setMinCpuCores(4); + nodeInfo.setMinCpuCores(1); + assertFalse(HardwareNodeComparator.compare(nodeInfo, actualHardware).getMetrics().isMatch()); + } + +}
\ No newline at end of file diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/HostURLGeneratorTest.java b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/HostURLGeneratorTest.java new file mode 100644 index 00000000000..78d30dbc836 --- /dev/null +++ b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/HostURLGeneratorTest.java @@ -0,0 +1,29 @@ +package com.yahoo.vespa.hosted.node.verification.spec; + +import org.junit.Test; + +import java.net.URL; + +import static org.junit.Assert.assertEquals; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.when; + +/** + * Created by olaa on 14/07/2017. + */ +public class HostURLGeneratorTest { + + @Test + public void generateNodeInfoUrl_test_if_url_is_formatted_correctly() throws Exception { + String zoneHostName = "http://cfg1.prod.us-west-1.vespahosted.gq1.yahoo.com:4080"; + String midUrl = "/nodes/v2/node/"; + String nodeHostName = "13305821.ostk.bm2.prod.gq1.yahoo.com"; + HostURLGenerator hostURLGenerator = spy(new HostURLGenerator()); + when(hostURLGenerator.getEnvironmentVariable("HOSTNAME")).thenReturn(nodeHostName); + URL url = hostURLGenerator.generateNodeInfoUrl(zoneHostName); + String expectedUrl = zoneHostName + midUrl + nodeHostName; + String actualUrl = url.toString(); + assertEquals(expectedUrl, actualUrl); + } + +}
\ No newline at end of file diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/noderepo/IPAddressVerifierTest.java b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/noderepo/IPAddressVerifierTest.java new file mode 100644 index 00000000000..36e4c7af03d --- /dev/null +++ b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/noderepo/IPAddressVerifierTest.java @@ -0,0 +1,50 @@ +package com.yahoo.vespa.hosted.node.verification.spec.noderepo; + +import org.junit.Before; +import org.junit.Test; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.spy; + +public class IPAddressVerifierTest { + + private IPAddressVerifier ipAddressVerifier = spy(new IPAddressVerifier()); + private String ipAddress; + private String additionalIp1; + private String additionalIp2; + private String additionalIp3; + private String[] additionalIpAddresses; + + @Before + public void setup() { + ipAddress = "2001:4998:c:2977::1060"; + additionalIp1 = "2001:4998:c:2977::106f"; + additionalIp2 = "2001:4998:c:2977::106a"; + additionalIp3 = "2001:4998:c:2977::106c"; + additionalIpAddresses = new String[]{additionalIp1, additionalIp2, additionalIp3}; + } + + @Test + public void verifyAdditionalIpAddress_should_add_IP_address_when_different_hostname() throws Exception { + String realHostName = "www.yahoo.com"; + String wrongHostName = "www.nrk.no"; + doReturn(realHostName).when(ipAddressVerifier).reverseLookUp(ipAddress); + doReturn(realHostName).when(ipAddressVerifier).reverseLookUp(additionalIp1); + doReturn(realHostName).when(ipAddressVerifier).reverseLookUp(additionalIp2); + doReturn(wrongHostName).when(ipAddressVerifier).reverseLookUp(additionalIp3); + String[] faultyIpAddresses = ipAddressVerifier.getFaultyIpAddresses(ipAddress, additionalIpAddresses); + String[] expectedFaultyIpAddresses = new String[]{additionalIp3}; + assertArrayEquals(expectedFaultyIpAddresses, faultyIpAddresses); + } + + @Test + public void convertToLookupFormat_should_return_properly_converted_ipv6_address() { + String ipv6Address = "2001:db8::567:89ab"; + String actualConvertedAddress = ipAddressVerifier.convertToLookupFormat(ipv6Address); + String expectedConvertedAddress = "b.a.9.8.7.6.5.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.8.b.d.0.1.0.0.2.ip6.arpa"; + assertEquals(expectedConvertedAddress, actualConvertedAddress); + } + +}
\ No newline at end of file diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/noderepo/NodeGeneratorTest.java b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/noderepo/NodeGeneratorTest.java new file mode 100644 index 00000000000..9ccf48f8494 --- /dev/null +++ b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/noderepo/NodeGeneratorTest.java @@ -0,0 +1,38 @@ +package com.yahoo.vespa.hosted.node.verification.spec.noderepo; + +import com.yahoo.vespa.hosted.node.verification.spec.retrievers.HardwareInfo; +import org.junit.Test; + +import java.io.File; +import java.net.URL; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.assertEquals; + +/** + * Created by olaa on 07/07/2017. + */ +public class NodeGeneratorTest { + + private static final double DELTA = 0.1; + + @Test + public void convertJsonModel_should_return_correct_HardwareInfo() throws Exception { + URL url = new File("src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/nodeInfoTest.json").toURI().toURL(); + NodeJsonModel nodeJsonModel = NodeInfoRetriever.retrieve(url); + HardwareInfo hardwareInfo = NodeGenerator.convertJsonModel(nodeJsonModel); + double expectedMinDiskAvailable = 500.0; + double expectedMinMainMemoryAvailable = 24.0; + double expectedMinCpuCores = 24.0; + double expectedInterfaceSpeedMbs = 1000; + assertEquals(expectedMinDiskAvailable, hardwareInfo.getMinDiskAvailableGb(), DELTA); + assertEquals(expectedMinMainMemoryAvailable, hardwareInfo.getMinMainMemoryAvailableGb(), DELTA); + assertEquals(expectedMinCpuCores, hardwareInfo.getMinCpuCores(), DELTA); + assertTrue(hardwareInfo.getIpv4Connectivity()); + assertTrue(hardwareInfo.getIpv6Connectivity()); + assertEquals(expectedInterfaceSpeedMbs, hardwareInfo.getInterfaceSpeedMbs(), DELTA); + assertFalse(hardwareInfo.getFastDisk()); + } + +}
\ No newline at end of file diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/DiskInfoTest b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/DiskInfoTest new file mode 100644 index 00000000000..80c8b713e84 --- /dev/null +++ b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/DiskInfoTest @@ -0,0 +1,2 @@ +NAME ROTA +vda 1
\ No newline at end of file diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/DiskTypeInvalid b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/DiskTypeInvalid new file mode 100644 index 00000000000..bd0824b2eba --- /dev/null +++ b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/DiskTypeInvalid @@ -0,0 +1,2 @@ +Name Rota +sda z
\ No newline at end of file diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/DiskTypeNonFastDisk b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/DiskTypeNonFastDisk new file mode 100644 index 00000000000..efe71b963c1 --- /dev/null +++ b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/DiskTypeNonFastDisk @@ -0,0 +1,2 @@ +Name Rota +sda 1
\ No newline at end of file diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/DisktypeFastDisk b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/DisktypeFastDisk new file mode 100644 index 00000000000..3b0eed77569 --- /dev/null +++ b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/DisktypeFastDisk @@ -0,0 +1,2 @@ +Name Rota +sda 0
\ No newline at end of file diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/cpuinfoTest b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/cpuinfoTest new file mode 100644 index 00000000000..8e1b0236a6c --- /dev/null +++ b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/cpuinfoTest @@ -0,0 +1,104 @@ +processor : 0 +vendor_id : GenuineIntel +cpu family : 6 +model : 70 +model name : Intel(R) Core(TM) i7-4870HQ CPU @ 2.50GHz +stepping : 1 +cpu MHz : 2493.821 +cache size : 6144 KB +physical id : 0 +siblings : 1 +core id : 0 +cpu cores : 1 +apicid : 0 +initial apicid : 0 +fpu : yes +fpu_exception : yes +cpuid level : 13 +wp : yes +flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ss ht pbe syscall nx pdpe1gb lm constant_tsc rep_good nopl xtopology nonstop_tsc eagerfpu pni pclmulqdq dtes64 ds_cpl ssse3 sdbg fma cx16 xtpr pcid sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm abm fsgsbase bmi1 avx2 bmi2 erms xsaveopt arat +bugs : +bogomips : 4987.64 +clflush size : 64 +cache_alignment : 64 +address sizes : 39 bits physical, 48 bits virtual +power management: + +processor : 1 +vendor_id : GenuineIntel +cpu family : 6 +model : 70 +model name : Intel(R) Core(TM) i7-4870HQ CPU @ 2.50GHz +stepping : 1 +cpu MHz : 2493.821 +cache size : 6144 KB +physical id : 1 +siblings : 1 +core id : 0 +cpu cores : 1 +apicid : 1 +initial apicid : 1 +fpu : yes +fpu_exception : yes +cpuid level : 13 +wp : yes +flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ss ht pbe syscall nx pdpe1gb lm constant_tsc rep_good nopl xtopology nonstop_tsc eagerfpu pni pclmulqdq dtes64 ds_cpl ssse3 sdbg fma cx16 xtpr pcid sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm abm fsgsbase bmi1 avx2 bmi2 erms xsaveopt arat +bugs : +bogomips : 5094.26 +clflush size : 64 +cache_alignment : 64 +address sizes : 39 bits physical, 48 bits virtual +power management: + +processor : 2 +vendor_id : GenuineIntel +cpu family : 6 +model : 70 +model name : Intel(R) Core(TM) i7-4870HQ CPU @ 2.50GHz +stepping : 1 +cpu MHz : 2493.821 +cache size : 6144 KB +physical id : 2 +siblings : 1 +core id : 0 +cpu cores : 1 +apicid : 2 +initial apicid : 2 +fpu : yes +fpu_exception : yes +cpuid level : 13 +wp : yes +flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ss ht pbe syscall nx pdpe1gb lm constant_tsc rep_good nopl xtopology nonstop_tsc eagerfpu pni pclmulqdq dtes64 ds_cpl ssse3 sdbg fma cx16 xtpr pcid sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm abm fsgsbase bmi1 avx2 bmi2 erms xsaveopt arat +bugs : +bogomips : 4390.19 +clflush size : 64 +cache_alignment : 64 +address sizes : 39 bits physical, 48 bits virtual +power management: + +processor : 3 +vendor_id : GenuineIntel +cpu family : 6 +model : 70 +model name : Intel(R) Core(TM) i7-4870HQ CPU @ 2.50GHz +stepping : 1 +cpu MHz : 2493.821 +cache size : 6144 KB +physical id : 3 +siblings : 1 +core id : 0 +cpu cores : 1 +apicid : 3 +initial apicid : 3 +fpu : yes +fpu_exception : yes +cpuid level : 13 +wp : yes +flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ss ht pbe syscall nx pdpe1gb lm constant_tsc rep_good nopl xtopology nonstop_tsc eagerfpu pni pclmulqdq dtes64 ds_cpl ssse3 sdbg fma cx16 xtpr pcid sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm abm fsgsbase bmi1 avx2 bmi2 erms xsaveopt arat +bugs : +bogomips : 5448.95 +clflush size : 64 +cache_alignment : 64 +address sizes : 39 bits physical, 48 bits virtual +power management: + diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/eth0 b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/eth0 new file mode 100755 index 00000000000..7b2d15a44c1 --- /dev/null +++ b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/eth0 @@ -0,0 +1,15 @@ + Supported ports: [ ] + Supported link modes: Not reported + Supported pause frame use: No + Supports auto-negotiation: No + Advertised link modes: Not reported + Advertised pause frame use: No + Advertised auto-negotiation: No + Speed: 1000Mb/s + Duplex: Full + Port: Twisted Pair + PHYAD: 0 + Transceiver: internal + Auto-negotiation: off + MDI-X: Unknown + Link detected: yes
\ No newline at end of file diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/executeTestFile.txt b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/executeTestFile.txt new file mode 100644 index 00000000000..8d4cafee461 --- /dev/null +++ b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/executeTestFile.txt @@ -0,0 +1,4 @@ +This file test +if executeCommand +reads +this file properly.
\ No newline at end of file diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/filesize b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/filesize new file mode 100644 index 00000000000..24aeccfdb0f --- /dev/null +++ b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/filesize @@ -0,0 +1,2 @@ +Filesystem 1G-blocks Used Available Use% Mounted on +overlay 63G 17G 44G 28% /
\ No newline at end of file diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/ifconfig b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/ifconfig new file mode 100755 index 00000000000..70ef5ea8cdc --- /dev/null +++ b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/ifconfig @@ -0,0 +1,16 @@ +eth0 Link encap:Ethernet HWaddr 02:42:AC:11:00:03 + inet addr:172.17.0.3 Bcast:0.0.0.0 Mask:255.255.0.0 + inet6 addr: fe80::a00:27ff:fe70:e3f5/64 Scope:Link + UP BROADCAST RUNNING MULTICAST MTU:1500 Metric:1 + RX packets:52908 errors:0 dropped:0 overruns:0 frame:0 + TX packets:29527 errors:0 dropped:0 overruns:0 carrier:0 + collisions:0 txqueuelen:0 + RX bytes:75003406 (71.5 MiB) TX bytes:1839045 (1.7 MiB) + +lo Link encap:Local Loopback + inet addr:127.0.0.1 Mask:255.0.0.0 + UP LOOPBACK RUNNING MTU:65536 Metric:1 + RX packets:2 errors:0 dropped:0 overruns:0 frame:0 + TX packets:2 errors:0 dropped:0 overruns:0 carrier:0 + collisions:0 txqueuelen:1 + RX bytes:100 (100.0 b) TX bytes:100 (100.0 b) diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/ifconfigNoIpv6 b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/ifconfigNoIpv6 new file mode 100755 index 00000000000..1e21b9e1195 --- /dev/null +++ b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/ifconfigNoIpv6 @@ -0,0 +1,15 @@ +lo Link encap:Local Loopback + inet addr:127.0.0.1 Mask:255.0.0.0 + UP LOOPBACK RUNNING MTU:65536 Metric:1 + RX packets:2 errors:0 dropped:0 overruns:0 frame:0 + TX packets:2 errors:0 dropped:0 overruns:0 carrier:0 + collisions:0 txqueuelen:1 + RX bytes:100 (100.0 b) TX bytes:100 (100.0 b) + +eth0 Link encap:Ethernet HWaddr 02:42:AC:11:00:03 + inet addr:172.17.0.3 Bcast:0.0.0.0 Mask:255.255.0.0 + UP BROADCAST RUNNING MULTICAST MTU:1500 Metric:1 + RX packets:52908 errors:0 dropped:0 overruns:0 frame:0 + TX packets:29527 errors:0 dropped:0 overruns:0 carrier:0 + collisions:0 txqueuelen:0 + RX bytes:75003406 (71.5 MiB) TX bytes:1839045 (1.7 MiB)
\ No newline at end of file diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/invalidFilesize b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/invalidFilesize new file mode 100644 index 00000000000..3a0c1be8ef7 --- /dev/null +++ b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/invalidFilesize @@ -0,0 +1,2 @@ +This should +not 123 work
\ No newline at end of file diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/meminfoTest b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/meminfoTest new file mode 100644 index 00000000000..df2282d9528 --- /dev/null +++ b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/meminfoTest @@ -0,0 +1,45 @@ +MemTotal: 4042128 kB +MemFree: 3584108 kB +MemAvailable: 3562648 kB +Buffers: 9568 kB +Cached: 315836 kB +SwapCached: 0 kB +Active: 176620 kB +Inactive: 202120 kB +Active(anon): 113320 kB +Inactive(anon): 105560 kB +Active(file): 63300 kB +Inactive(file): 96560 kB +Unevictable: 0 kB +Mlocked: 0 kB +SwapTotal: 1048572 kB +SwapFree: 1048572 kB +Dirty: 0 kB +Writeback: 0 kB +AnonPages: 53348 kB +Mapped: 49140 kB +Shmem: 165548 kB +Slab: 54504 kB +SReclaimable: 38372 kB +SUnreclaim: 16132 kB +KernelStack: 3552 kB +PageTables: 1368 kB +NFS_Unstable: 0 kB +Bounce: 0 kB +WritebackTmp: 0 kB +CommitLimit: 3069636 kB +Committed_AS: 810504 kB +VmallocTotal: 34359738367 kB +VmallocUsed: 0 kB +VmallocChunk: 0 kB +AnonHugePages: 0 kB +ShmemHugePages: 0 kB +ShmemPmdMapped: 0 kB +HugePages_Total: 0 +HugePages_Free: 0 +HugePages_Rsvd: 0 +HugePages_Surp: 0 +Hugepagesize: 2048 kB +DirectMap4k: 24576 kB +DirectMap2M: 2072576 kB +DirectMap1G: 4194304 kB diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/nodeInfoTest.json b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/nodeInfoTest.json new file mode 100644 index 00000000000..d0779af5feb --- /dev/null +++ b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/nodeInfoTest.json @@ -0,0 +1,48 @@ +{ + "url": "https://api.vespa.corp.yahoo.com:4443/zone/v2/prod/eu-west-1/nodes/v2/node/20004731.ostk.bm1.prod.ir2.yahoo.com", + "id": "20004731.ostk.bm1.prod.ir2.yahoo.com", + "state": "ready", + "type": "tenant", + "hostname": "20004731.ostk.bm1.prod.ir2.yahoo.com", + "openStackId": "77df66c5-3cbc-4905-b0e9-f790bddd13bf", + "fastDisk": false, + "flavor": "C-2B/24/500", + "canonicalFlavor": "C-2B/24/500", + "minDiskAvailableGb": 500.0, + "minMainMemoryAvailableGb": 24.0, + "description": "BARE_METAL with 24.0 CPUs, 24.0 Gb memory and 500.0 Gb disk", + "minCpuCores": 24.0, + "cost": 66, + "environment": "BARE_METAL", + "rebootGeneration": 10, + "currentRebootGeneration": 10, + "vespaVersion": "6.120.30", + "currentDockerImage": "docker-registry.ops.yahoo.com:4443/vespa/ci:6.120.30", + "hostedVersion": "6.120.30", + "convergedStateVersion": "6.120.30", + "failCount": 0, + "hardwareFailure": false, + "wantToRetire": false, + "wantToDeprovision": false, + "history": [ + { + "event": "readied", + "at": 1489609585827, + "agent": "system" + }, + { + "event": "rebooted", + "at": 1498309261481, + "agent": "system" + }, + { + "event": "requested", + "at": 1499332178235, + "agent": "system" + } + ], + "ipAddresses": [ + "10.200.66.16" + ], + "additionalIpAddresses": [] +}
\ No newline at end of file diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/parseOutputWithSkipsTest b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/parseOutputWithSkipsTest new file mode 100644 index 00000000000..460b3c54b46 --- /dev/null +++ b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/parseOutputWithSkipsTest @@ -0,0 +1,16 @@ +This will be read, and this will be read +But it will not be matched +But NOW we are going to skip! +SkipFromKeyword +here we skip +and we skip +when Parsing this we get a match... +No we don't! Because we are skipping! +we continue skipping +And soon we stop to skip +NOW! +skipUntilKeyword +Still no matches +When Parsing this we get a match #returnValue# +Not this, +but Parsing this gives a match with #returnValue#
\ No newline at end of file diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/CPURetrieverTest.java b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/CPURetrieverTest.java new file mode 100644 index 00000000000..36b886de194 --- /dev/null +++ b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/CPURetrieverTest.java @@ -0,0 +1,70 @@ +package com.yahoo.vespa.hosted.node.verification.spec.retrievers; + +import com.yahoo.vespa.hosted.node.verification.commons.ParseResult; +import com.yahoo.vespa.hosted.node.verification.mock.MockCommandExecutor; +import org.junit.Before; +import org.junit.Test; + +import java.io.IOException; +import java.util.ArrayList; + +import static org.junit.Assert.assertEquals; + +/** + * Created by olaa on 03/07/2017. + */ +public class CPURetrieverTest { + + private static final String FILENAME = "src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/cpuinfoTest"; + private HardwareInfo hardwareInfo; + private MockCommandExecutor commandExecutor; + private CPURetriever cpu; + private static final double DELTA = 0.1; + + @Before + public void setup() { + hardwareInfo = new HardwareInfo(); + commandExecutor = new MockCommandExecutor(); + cpu = new CPURetriever(hardwareInfo, commandExecutor); + } + + @Test + public void updateInfo_should_write_numOfCpuCores_to_hardware_info() throws Exception { + commandExecutor.addCommand("cat " + FILENAME); + cpu.updateInfo(); + double expectedAmountOfCores = 4; + assertEquals(expectedAmountOfCores, hardwareInfo.getMinCpuCores(), DELTA); + } + + @Test + public void parseCPUInfoFile_should_return_valid_ArrayList() throws IOException { + ArrayList<String> commandOutput = MockCommandExecutor.readFromFile(FILENAME); + ArrayList<ParseResult> ParseResults = cpu.parseCPUInfoFile(commandOutput); + String expectedSearchWord = "cpu MHz"; + String expectedValue = "2493.821"; + + assertEquals(expectedSearchWord, ParseResults.get(0).getSearchWord()); + assertEquals(expectedValue, ParseResults.get(0).getValue()); + + assertEquals(expectedSearchWord, ParseResults.get(1).getSearchWord()); + assertEquals(expectedValue, ParseResults.get(1).getValue()); + + assertEquals(expectedSearchWord, ParseResults.get(2).getSearchWord()); + assertEquals(expectedValue, ParseResults.get(2).getValue()); + + assertEquals(expectedSearchWord, ParseResults.get(3).getSearchWord()); + assertEquals(expectedValue, ParseResults.get(3).getValue()); + } + + @Test + public void setCpuCores_counts_cores_correctly() { + ArrayList<ParseResult> parseResults = new ArrayList<>(); + parseResults.add(new ParseResult("cpu MHz", "2000")); + parseResults.add(new ParseResult("cpu MHz", "2000")); + parseResults.add(new ParseResult("cpu MHz", "2000")); + cpu.setCpuCores(parseResults); + int expectedCpuCores = 3; + assertEquals(expectedCpuCores, hardwareInfo.getMinCpuCores()); + } + +} diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/DiskRetrieverTest.java b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/DiskRetrieverTest.java new file mode 100644 index 00000000000..cd5c16c43d2 --- /dev/null +++ b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/DiskRetrieverTest.java @@ -0,0 +1,95 @@ +package com.yahoo.vespa.hosted.node.verification.spec.retrievers; + +import com.yahoo.vespa.hosted.node.verification.commons.ParseResult; +import com.yahoo.vespa.hosted.node.verification.mock.MockCommandExecutor; +import org.junit.Before; +import org.junit.Test; + +import java.io.IOException; +import java.util.ArrayList; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +/** + * Created by olaa on 06/07/2017. + */ +public class DiskRetrieverTest { + + private MockCommandExecutor commandExecutor; + private HardwareInfo hardwareInfo; + private DiskRetriever diskRetriever; + private static String CAT_RESOURCE_PATH = "cat src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/"; + private static final double DELTA = 0.1; + + @Before + public void setup() { + hardwareInfo = new HardwareInfo(); + commandExecutor = new MockCommandExecutor(); + diskRetriever = new DiskRetriever(hardwareInfo, commandExecutor); + } + + @Test + public void updateInfo_should_store_diskType_and_diskSize_in_hardware_info() { + commandExecutor.addCommand(CAT_RESOURCE_PATH + "DiskTypeFastDisk"); + commandExecutor.addCommand(CAT_RESOURCE_PATH + "filesize"); + diskRetriever.updateInfo(); + assertTrue(hardwareInfo.getFastDisk()); + double expectedSize = 63D; + assertEquals(expectedSize, hardwareInfo.getMinDiskAvailableGb(), DELTA); + } + + @Test + public void updateDiskType__should_store_diskType_in_hardwareInfo() throws IOException { + commandExecutor.addCommand(CAT_RESOURCE_PATH + "DiskTypeFastDisk"); + diskRetriever.updateDiskType(); + assertTrue(hardwareInfo.getFastDisk()); + } + + @Test + public void updateDiskSize__should_store_diskSize_in_hardwareInfo() throws IOException { + commandExecutor.addCommand(CAT_RESOURCE_PATH + "filesize"); + diskRetriever.updateDiskSize(); + double expectedSize = 63D; + assertEquals(expectedSize, hardwareInfo.getMinDiskAvailableGb(), DELTA); + } + + @Test + public void parseDiskType_should_find_fast_disk() throws Exception { + diskRetriever = new DiskRetriever(hardwareInfo, commandExecutor); + ArrayList<String> mockOutput = commandExecutor.outputFromString("Name Rota \nsda 0"); + ParseResult parseResult = diskRetriever.parseDiskType(mockOutput); + ParseResult expectedParseResult = new ParseResult("sda", "0"); + assertEquals(expectedParseResult, parseResult); + } + + @Test + public void parseDiskType_should_not_find_fast_disk() throws Exception { + ArrayList<String> mockOutput = commandExecutor.outputFromString("Name Rota \nsda 1"); + ParseResult parseResult = diskRetriever.parseDiskType(mockOutput); + ParseResult expectedParseResult = new ParseResult("sda", "1"); + assertEquals(expectedParseResult, parseResult); + } + + @Test + public void parseDiskType_with_invalid_output_stream_should_not_find_disk_type() throws Exception { + ArrayList<String> mockOutput = commandExecutor.outputFromString("Name Rota \nsda x"); + ParseResult parseResult = diskRetriever.parseDiskType(mockOutput); + ParseResult expectedParseResult = new ParseResult("sda", "x"); + assertEquals(expectedParseResult, parseResult); + mockOutput = commandExecutor.outputFromString("Name Rota"); + parseResult = diskRetriever.parseDiskType(mockOutput); + expectedParseResult = new ParseResult("invalid", "invalid"); + assertEquals(expectedParseResult, parseResult); + } + + @Test + public void parseDiskSize_should_find_size_from_file_and_insert_into_parseResult() throws Exception { + String filepath = "src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/filesize"; + ArrayList<String> mockOutput = MockCommandExecutor.readFromFile(filepath); + ParseResult parseResult = diskRetriever.parseDiskSize(mockOutput); + ParseResult expectedParseResult = new ParseResult("44G", "63G"); + assertEquals(expectedParseResult, parseResult); + } + +}
\ No newline at end of file diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/MemoryRetrieverTest.java b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/MemoryRetrieverTest.java new file mode 100644 index 00000000000..aee18fd8301 --- /dev/null +++ b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/MemoryRetrieverTest.java @@ -0,0 +1,62 @@ +package com.yahoo.vespa.hosted.node.verification.spec.retrievers; + +import com.yahoo.vespa.hosted.node.verification.commons.ParseResult; +import com.yahoo.vespa.hosted.node.verification.mock.MockCommandExecutor; +import org.junit.Before; +import org.junit.Test; + +import java.io.IOException; +import java.util.ArrayList; + +import static org.junit.Assert.assertEquals; + +/** + * Created by sgrostad on 06/07/2017. + */ +public class MemoryRetrieverTest { + + private static final String FILENAME = "src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/meminfoTest"; + private HardwareInfo hardwareInfo; + private MockCommandExecutor commandExecutor; + private MemoryRetriever memory; + private final double DELTA = 0.1; + + @Before + public void setup() { + hardwareInfo = new HardwareInfo(); + commandExecutor = new MockCommandExecutor(); + memory = new MemoryRetriever(hardwareInfo, commandExecutor); + } + + @Test + public void updateInfo_should_set_memory_available_in_hardwareInfo() throws IOException { + commandExecutor.addCommand("cat " + FILENAME); + memory.updateInfo(); + double expectedMemory = 4.042128; + assertEquals(expectedMemory, hardwareInfo.getMinMainMemoryAvailableGb(), DELTA); + } + + @Test + public void parseMemInfoFile_should_return_valid_parseResult() throws IOException { + ArrayList<String> commandOutput = MockCommandExecutor.readFromFile(FILENAME); + ParseResult parseResult = memory.parseMemInfoFile(commandOutput); + ParseResult expectedParseResult = new ParseResult("MemTotal", "4042128 kB"); + assertEquals(expectedParseResult, parseResult); + } + + @Test + public void updateMemoryInfo_valid_input() { + ParseResult testParseResult = new ParseResult("MemTotal", "4042128"); + memory.updateMemoryInfo(testParseResult); + double expectedMemory = 4.042128; + assertEquals(expectedMemory, hardwareInfo.getMinMainMemoryAvailableGb(), DELTA); + } + + @Test + public void convertToGB_valid_input() { + String testTotMem = "4042128"; + double expectedTotMem = 4.042128; + assertEquals(expectedTotMem, memory.convertKBToGB(testTotMem), DELTA); + } + +}
\ No newline at end of file diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/NetRetrieverTest.java b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/NetRetrieverTest.java new file mode 100644 index 00000000000..8f02917237e --- /dev/null +++ b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/NetRetrieverTest.java @@ -0,0 +1,128 @@ +package com.yahoo.vespa.hosted.node.verification.spec.retrievers; + +import com.yahoo.vespa.hosted.node.verification.commons.ParseResult; +import com.yahoo.vespa.hosted.node.verification.mock.MockCommandExecutor; +import org.junit.Before; +import org.junit.Test; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +/** + * Created by sgrostad on 07/07/2017. + */ +public class NetRetrieverTest { + + private static final String NET_FIND_INTERFACE = "src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/ifconfig"; + private static final String NET_CHECK_INTERFACE_SPEED = "src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/"; + private HardwareInfo hardwareInfo; + private MockCommandExecutor commandExecutor; + private NetRetriever net; + private ArrayList<ParseResult> parseResults; + private static final double DELTA = 0.1; + + @Before + public void setup() { + hardwareInfo = new HardwareInfo(); + commandExecutor = new MockCommandExecutor(); + net = new NetRetriever(hardwareInfo, commandExecutor); + parseResults = new ArrayList<>(); + } + + @Test + public void updateInfo_should_store_ipv4_ipv6_connectivity_and_interface_speed() { + commandExecutor.addCommand("cat " + NET_FIND_INTERFACE); + commandExecutor.addCommand("cat " + NET_CHECK_INTERFACE_SPEED + "eth0"); + net.updateInfo(); + assertTrue(hardwareInfo.getIpv4Connectivity()); + assertTrue(hardwareInfo.getIpv6Connectivity()); + double expectedInterfaceSpeed = 1000; + assertEquals(expectedInterfaceSpeed, hardwareInfo.getInterfaceSpeedMbs(), DELTA); + } + + @Test + public void findInterface_valid_input() throws IOException { + commandExecutor.addCommand("cat " + NET_FIND_INTERFACE); + parseResults = net.findInterface(); + ParseResult expectedParseResult = new ParseResult("eth0", "eth0"); + assertEquals(expectedParseResult, parseResults.get(0)); + } + + @Test + public void findInterfaceSpeed_valid_input() throws IOException { + commandExecutor.addCommand("cat " + NET_FIND_INTERFACE); + commandExecutor.addCommand("cat " + NET_CHECK_INTERFACE_SPEED + "eth0"); + parseResults = net.findInterface(); + net.findInterfaceSpeed(parseResults); + ParseResult expectedParseResults = new ParseResult("Speed", "1000Mb/s"); + assertEquals(expectedParseResults, parseResults.get(3)); + } + + @Test + public void parseNetInterface_get_ipv_from_ifconfig_testFile() throws IOException { + ArrayList<String> mockOutput = MockCommandExecutor.readFromFile(NET_FIND_INTERFACE); + parseResults = net.parseNetInterface(mockOutput); + net.updateHardwareInfoWithNet(parseResults); + assertTrue(hardwareInfo.getIpv4Connectivity()); + assertTrue(hardwareInfo.getIpv6Connectivity()); + } + + @Test + public void parseNetInterface_get_ipv_from_ifconfigNotIpv6_testFile() throws IOException { + ArrayList<String> mockOutput = MockCommandExecutor.readFromFile(NET_FIND_INTERFACE + "NoIpv6"); + parseResults = net.parseNetInterface(mockOutput); + ArrayList<ParseResult> expextedParseResults = new ArrayList<>(Arrays.asList( + new ParseResult("eth0", "eth0"), + new ParseResult("inet", "inet"))); + assertEquals(expextedParseResults, parseResults); + } + + @Test + public void parseNetInterface_get_interfaceName_from_ifconfig_testFile() throws IOException { + ArrayList<String> mockOutput = MockCommandExecutor.readFromFile(NET_FIND_INTERFACE); + parseResults = net.parseNetInterface(mockOutput); + String interfaceName = net.findInterfaceName(parseResults); + String expectedInterfaceName = "eth0"; + assertEquals(expectedInterfaceName, interfaceName); + } + + @Test + public void parseInterfaceSpeed_get_interfaceSpeed_from_eth0_testFile() throws IOException { + ArrayList<String> mockOutput = MockCommandExecutor.readFromFile("src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/eth0"); + ParseResult parseResult = net.parseInterfaceSpeed(mockOutput); + ParseResult expectedParseResult = new ParseResult("Speed", "1000Mb/s"); + assertEquals(expectedParseResult, parseResult); + } + + @Test + public void findInterfaceName_should_return_interface_name() { + parseResults.add(new ParseResult("eth0", "eth0")); + String expectedInterfaceName = "eth0"; + assertEquals(expectedInterfaceName, net.findInterfaceName(parseResults)); + } + + @Test + public void findInterfaceName_should_return_empty_interface_name() { + parseResults.add(new ParseResult("et", "et0")); + String expectedInterfaceName = ""; + assertEquals(expectedInterfaceName, net.findInterfaceName(parseResults)); + } + + @Test + public void updateHardwareinfoWithNet_valid_input() { + parseResults.add(new ParseResult("eth0", "eth0")); + parseResults.add(new ParseResult("inet", "inet")); + parseResults.add(new ParseResult("inet6", "inet6")); + parseResults.add(new ParseResult("Speed", "1000Mb/s")); + net.updateHardwareInfoWithNet(parseResults); + double expectedInterfaceSpeed = 1000; + assertEquals(expectedInterfaceSpeed, hardwareInfo.getInterfaceSpeedMbs(), DELTA); + assertTrue(hardwareInfo.getIpv4Connectivity()); + assertTrue(hardwareInfo.getIpv6Connectivity()); + } + +}
\ No newline at end of file diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/yamasreport/YamasSpecReportTest.java b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/yamasreport/YamasSpecReportTest.java new file mode 100644 index 00000000000..2131d09c0dd --- /dev/null +++ b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/yamasreport/YamasSpecReportTest.java @@ -0,0 +1,54 @@ +package com.yahoo.vespa.hosted.node.verification.spec.yamasreport; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.junit.Before; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + +/** + * Created by olaa on 12/07/2017. + */ +public class YamasSpecReportTest { + + SpecReportDimensions specReportDimensions; + SpecReportMetrics specReportMetrics; + + @Before + public void setup() { + specReportDimensions = new SpecReportDimensions(); + specReportMetrics = new SpecReportMetrics(); + specReportDimensions.setCpuCoresMatch(true); + specReportDimensions.setDiskAvailableMatch(true); + specReportDimensions.setIpv4Match(true); + specReportDimensions.setIpv6Match(true); + specReportDimensions.setMemoryMatch(true); + specReportDimensions.setNetInterfaceSpeedMatch(true); + specReportDimensions.setFastDiskMatch(true); + specReportMetrics.setActualInterfaceSpeed(100D); + specReportMetrics.setExpectedInterfaceSpeed(100D); + specReportMetrics.setActualDiskSpaceAvailable(500D); + specReportMetrics.setExpectedDiskSpaceAvailable(500D); + specReportMetrics.setActualDiskType(true); + specReportMetrics.setExpectedDiskType(true); + specReportMetrics.setActualMemoryAvailable(123D); + specReportMetrics.setExpectedMemoryAvailable(123D); + specReportMetrics.setActualcpuCores(4); + specReportMetrics.setExpectedcpuCores(4); + specReportMetrics.setMatch(true); + } + + @Test + public void Json_is_in_wanted_format() throws Exception { + YamasSpecReport yamasSpecReport = new YamasSpecReport(); + yamasSpecReport.setMetrics(specReportMetrics); + yamasSpecReport.setDimensions(specReportDimensions); + yamasSpecReport.setMetrics(specReportMetrics); + long time = yamasSpecReport.getTimeStamp(); + String expectedJson = "{\"timeStamp\":" + time + ",\"dimensions\":{\"memoryMatch\":true,\"cpuCoresMatch\":true,\"fastDiskMatch\":true,\"netInterfaceSpeedMatch\":true,\"diskAvailableMatch\":true,\"ipv4Match\":true,\"ipv6Match\":true},\"metrics\":{\"match\":true,\"expectedMemoryAvailable\":123.0,\"actualMemoryAvailable\":123.0,\"expectedFastDisk\":true,\"actualFastDisk\":true,\"expectedDiskSpaceAvailable\":500.0,\"actualDiskSpaceAvailable\":500.0,\"expectedInterfaceSpeed\":100.0,\"actualInterfaceSpeed\":100.0,\"expectedcpuCores\":4,\"actualcpuCores\":4},\"routing\":{\"yamas\":{\"namespace\":[\"Vespa\"]}}}"; + ObjectMapper om = new ObjectMapper(); + String json = om.writeValueAsString(yamasSpecReport); + assertEquals(expectedJson, json); + } + +}
\ No newline at end of file |