From 45d8bdd375bd744c3d4d143c9c9d0f5d9d62219d Mon Sep 17 00:00:00 2001 From: Sindre Grøstad <30319420+sgrostad@users.noreply.github.com> Date: Mon, 31 Jul 2017 11:36:29 +0200 Subject: Summer interns/node verification (#3025) * Fixed wrong case in file name * Interfacespeed string to double converting moved out to new function * Changed names and refactored SpecVerifier * Changed disk type from boolean to enum * Made HostURLGenerator tests and renamed one variable in specVerifier * Added test for SpecVerifier * added tests in some spec classes and a check for Ipv6 when retrieving from nodeRepo * Added tests for verifySpec * Hardwareverifier changed name to HardwareBenchmarker and som refactoring in this class * More tests. Made methods in SpecVerifier static * Fixed CommandExecutor * added configserver URL retriver * Fixed resource path names * Moved pinging to spec verification. Deleted Client and Server classes from hardware verification. Created TerminationController * Added tests for buildNodeInfoUrl * Calls static function without object instance instance in HostURLGeneratorTest * umount is now sudo * Changed BenchmarkResults attributes to primitives * Added more tests * added tests for HardwareBenckmarker and TerminationController * Change of name * Reformatted code * Removed done TODOs * Changed parameters of Diskbenchmark --- .../node/verification/commons/CommandExecutor.java | 4 +- .../verification/hardware/HardwareBenchmarker.java | 56 +++++++++++ .../verification/hardware/HardwareVerifier.java | 47 --------- .../hardware/TerminationController.java | 48 +++++++++ .../hardware/benchmarks/BenchmarkResults.java | 47 +++++++++ .../hardware/benchmarks/CPUBenchmark.java | 8 +- .../hardware/benchmarks/DiskBenchmark.java | 10 +- .../hardware/benchmarks/HardwareResults.java | 56 ----------- .../hardware/benchmarks/MemoryBenchmark.java | 12 +-- .../hardware/benchmarks/NetBenchmark.java | 60 ------------ .../node/verification/hardware/net/Client.java | 46 --------- .../node/verification/hardware/net/Server.java | 41 -------- .../yamasreport/HardwareReportMetrics.java | 31 ++++-- .../hardware/yamasreport/YamasHardwareReport.java | 13 ++- .../verification/spec/HardwareNodeComparator.java | 30 +++--- .../node/verification/spec/HostURLGenerator.java | 56 ++++++++++- .../node/verification/spec/SpecVerifier.java | 61 +++++------- .../spec/noderepo/IPAddressVerifier.java | 6 +- .../verification/spec/noderepo/NodeGenerator.java | 23 ----- .../spec/noderepo/NodeInfoRetriever.java | 31 ------ .../spec/noderepo/NodeJsonConverter.java | 29 ++++++ .../verification/spec/noderepo/NodeJsonModel.java | 51 ---------- .../spec/noderepo/NodeRepoInfoRetriever.java | 35 +++++++ .../spec/noderepo/NodeRepoJsonModel.java | 71 ++++++++++++++ .../spec/retrievers/DiskRetriever.java | 16 +-- .../verification/spec/retrievers/HardwareInfo.java | 43 +++++--- .../spec/retrievers/HardwareInfoRetriever.java | 3 +- .../verification/spec/retrievers/NetRetriever.java | 49 +++++++-- .../spec/yamasreport/SpecReportDimensions.java | 10 +- .../spec/yamasreport/SpecReportMetrics.java | 20 ++-- .../hardware/HardwareBenchmarkerTest.java | 38 +++++++ .../hardware/TerminationControllerTest.java | 61 ++++++++++++ .../hardware/benchmarks/CPUBenchmarkTest.java | 14 +-- .../hardware/benchmarks/DiskBenchmarkTest.java | 16 +-- .../hardware/benchmarks/MemoryBenchmarkTest.java | 21 ++-- .../hardware/benchmarks/NetBenchmarkTest.java | 97 ------------------ .../hardware/resources/crazypingresponse | 1 - .../hardware/resources/invalidpingresponse | 1 - .../hardware/resources/validpingresponse | 1 - .../yamasreport/YamasHardwareReportTest.java | 31 ++++++ .../spec/HardwareNodeComparatorTest.java | 9 ++ .../verification/spec/HostURLGeneratorTest.java | 66 +++++++++++-- .../node/verification/spec/SpecVerifierTest.java | 109 +++++++++++++++++++++ .../spec/noderepo/IPAddressVerifierTest.java | 7 ++ .../spec/noderepo/NodeGeneratorTest.java | 38 ------- .../spec/noderepo/NodeJsonConverterTest.java | 40 ++++++++ .../spec/noderepo/NodeRepoInfoRetrieverTest.java | 60 ++++++++++++ .../verification/spec/resources/DiskTypeFastDisk | 2 + .../verification/spec/resources/DisktypeFastDisk | 2 - .../spec/resources/hostURLGeneratorExceptionTest | 3 + .../spec/resources/hostURLGeneratorTest | 1 + .../spec/resources/invalidpingresponse | 1 + .../verification/spec/resources/nodeHostNameOutput | 1 + .../spec/resources/nodes/v2/node/nodeRepo.json | 81 +++++++++++++++ .../spec/resources/pingresponse-all-packets-lost | 1 + .../verification/spec/resources/validpingresponse | 1 + .../spec/retrievers/DiskRetrieverTest.java | 17 ++-- .../spec/retrievers/HardwareInfoRetrieverTest.java | 58 +++++++++++ .../spec/retrievers/NetRetrieverTest.java | 74 +++++++++++--- .../spec/yamasreport/YamasSpecReportTest.java | 9 +- 60 files changed, 1190 insertions(+), 684 deletions(-) create mode 100644 node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/HardwareBenchmarker.java delete mode 100644 node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/HardwareVerifier.java create mode 100644 node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/TerminationController.java create mode 100644 node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/BenchmarkResults.java delete mode 100644 node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/HardwareResults.java delete mode 100644 node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/NetBenchmark.java delete mode 100644 node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/net/Client.java delete mode 100644 node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/net/Server.java delete mode 100644 node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/noderepo/NodeGenerator.java delete mode 100644 node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/noderepo/NodeInfoRetriever.java create mode 100644 node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/noderepo/NodeJsonConverter.java delete mode 100644 node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/noderepo/NodeJsonModel.java create mode 100644 node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/noderepo/NodeRepoInfoRetriever.java create mode 100644 node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/noderepo/NodeRepoJsonModel.java create mode 100644 node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/HardwareBenchmarkerTest.java create mode 100644 node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/TerminationControllerTest.java delete mode 100644 node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/NetBenchmarkTest.java delete mode 100644 node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/resources/crazypingresponse delete mode 100644 node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/resources/invalidpingresponse delete mode 100644 node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/resources/validpingresponse create mode 100644 node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/yamasreport/YamasHardwareReportTest.java create mode 100644 node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/SpecVerifierTest.java delete mode 100644 node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/noderepo/NodeGeneratorTest.java create mode 100644 node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/noderepo/NodeJsonConverterTest.java create mode 100644 node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/noderepo/NodeRepoInfoRetrieverTest.java create mode 100644 node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/DiskTypeFastDisk delete mode 100644 node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/DisktypeFastDisk create mode 100644 node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/hostURLGeneratorExceptionTest create mode 100644 node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/hostURLGeneratorTest create mode 100644 node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/invalidpingresponse create mode 100644 node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/nodeHostNameOutput create mode 100644 node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/nodes/v2/node/nodeRepo.json create mode 100644 node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/pingresponse-all-packets-lost create mode 100644 node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/validpingresponse create mode 100644 node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/HardwareInfoRetrieverTest.java (limited to 'node-maintainer/src') diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/CommandExecutor.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/CommandExecutor.java index c03e90d298a..c047200f7c4 100644 --- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/CommandExecutor.java +++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/CommandExecutor.java @@ -25,7 +25,9 @@ public class CommandExecutor { } private void writeToOutputStream(ByteArrayOutputStream outputStream, String command) throws IOException { - CommandLine cmdLine = CommandLine.parse(command); + CommandLine cmdLine = new CommandLine("/bin/bash"); + cmdLine.addArgument("-c", false); + cmdLine.addArgument(command, false); DefaultExecutor executor = new DefaultExecutor(); PumpStreamHandler streamHandler = new PumpStreamHandler(outputStream); executor.setStreamHandler(streamHandler); diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/HardwareBenchmarker.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/HardwareBenchmarker.java new file mode 100644 index 00000000000..cd4bcdfd758 --- /dev/null +++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/HardwareBenchmarker.java @@ -0,0 +1,56 @@ +package com.yahoo.vespa.hosted.node.verification.hardware; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.yahoo.vespa.hosted.node.verification.commons.CommandExecutor; +import com.yahoo.vespa.hosted.node.verification.hardware.benchmarks.Benchmark; +import com.yahoo.vespa.hosted.node.verification.hardware.benchmarks.BenchmarkResults; +import com.yahoo.vespa.hosted.node.verification.hardware.benchmarks.CPUBenchmark; +import com.yahoo.vespa.hosted.node.verification.hardware.benchmarks.DiskBenchmark; +import com.yahoo.vespa.hosted.node.verification.hardware.benchmarks.MemoryBenchmark; +import com.yahoo.vespa.hosted.node.verification.hardware.yamasreport.YamasHardwareReport; + +import java.util.ArrayList; +import java.util.Arrays; + +/** + * Benchmarks different hardware components and creates report + */ +public class HardwareBenchmarker { + + public static boolean hardwareBenchmarks(CommandExecutor commandExecutor) { + BenchmarkResults benchmarkResults = new BenchmarkResults(); + ArrayList benchmarks = new ArrayList<>(Arrays.asList( + new DiskBenchmark(benchmarkResults, commandExecutor), + new CPUBenchmark(benchmarkResults, commandExecutor), + new MemoryBenchmark(benchmarkResults, commandExecutor))); + for (Benchmark benchmark : benchmarks) { + benchmark.doBenchmark(); + } + + YamasHardwareReport yamasHardwareReport = makeYamasHardwareReport(benchmarkResults); + printBenchmarkResults(yamasHardwareReport); + TerminationController.terminateIfInvalidBenchmarkResults(benchmarkResults); + return true; + } + + protected static YamasHardwareReport makeYamasHardwareReport(BenchmarkResults benchmarkResults) { + YamasHardwareReport yamasHardwareReport = new YamasHardwareReport(); + yamasHardwareReport.createReportFromBenchmarkResults(benchmarkResults); + return yamasHardwareReport; + } + + private static void printBenchmarkResults(YamasHardwareReport yamasHardwareReport) { + ObjectMapper om = new ObjectMapper(); + try { + System.out.println(om.writeValueAsString(yamasHardwareReport)); + } catch (Exception e) { + e.printStackTrace(); + } + } + + public static void main(String[] args) { + CommandExecutor commandExecutor = new CommandExecutor(); + HardwareBenchmarker.hardwareBenchmarks(commandExecutor); + } + +} diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/HardwareVerifier.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/HardwareVerifier.java deleted file mode 100644 index 61d26b322f4..00000000000 --- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/HardwareVerifier.java +++ /dev/null @@ -1,47 +0,0 @@ -package com.yahoo.vespa.hosted.node.verification.hardware; - -import com.fasterxml.jackson.databind.ObjectMapper; -import com.yahoo.vespa.hosted.node.verification.commons.CommandExecutor; -import com.yahoo.vespa.hosted.node.verification.hardware.benchmarks.Benchmark; -import com.yahoo.vespa.hosted.node.verification.hardware.benchmarks.CPUBenchmark; -import com.yahoo.vespa.hosted.node.verification.hardware.benchmarks.DiskBenchmark; -import com.yahoo.vespa.hosted.node.verification.hardware.benchmarks.HardwareResults; -import com.yahoo.vespa.hosted.node.verification.hardware.benchmarks.MemoryBenchmark; -import com.yahoo.vespa.hosted.node.verification.hardware.benchmarks.NetBenchmark; -import com.yahoo.vespa.hosted.node.verification.hardware.yamasreport.YamasHardwareReport; - -import java.util.ArrayList; -import java.util.Arrays; - -/** - * Benchmarks different hardware components and creates report - */ -public class HardwareVerifier { - - public static void verifyHardware() { - HardwareResults hardwareResults = new HardwareResults(); - CommandExecutor commandExecutor = new CommandExecutor(); - ArrayList benchmarks = new ArrayList<>(Arrays.asList( - new DiskBenchmark(hardwareResults, commandExecutor), - new CPUBenchmark(hardwareResults, commandExecutor), - new MemoryBenchmark(hardwareResults, commandExecutor), - new NetBenchmark(hardwareResults, commandExecutor))); - - for (Benchmark benchmark : benchmarks) { - benchmark.doBenchmark(); - } - YamasHardwareReport yamasHardwareReport = new YamasHardwareReport(); - yamasHardwareReport.createFromHardwareResults(hardwareResults); - ObjectMapper om = new ObjectMapper(); - try { - System.out.println(om.writeValueAsString(yamasHardwareReport)); - } catch (Exception e) { - e.printStackTrace(); - } - } - - public static void main(String[] args) { - HardwareVerifier.verifyHardware(); - } - -} diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/TerminationController.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/TerminationController.java new file mode 100644 index 00000000000..fe75cdf88a2 --- /dev/null +++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/TerminationController.java @@ -0,0 +1,48 @@ +package com.yahoo.vespa.hosted.node.verification.hardware; + +import com.yahoo.vespa.hosted.node.verification.hardware.benchmarks.BenchmarkResults; + +import java.util.logging.Level; +import java.util.logging.Logger; + +public class TerminationController { + + private static final Logger logger = Logger.getLogger(TerminationController.class.getName()); + + private static final double CPU_FREQUENCY_LOWER_LIMIT = 0.5; + private static final double MEMORY_WRITE_SPEED_LOWER_LIMIT = 1D; + private static final double MEMORY_READ_SPEED_LOWER_LIMIT = 1D; + private static final double DISK_SPEED_LOWER_LIMIT = 50D; + + public static void terminateIfInvalidBenchmarkResults(BenchmarkResults benchmarkResults) { + if (!isBenchmarkResultsValid(benchmarkResults)) { + System.exit(1); + } + } + + public static boolean isBenchmarkResultsValid(BenchmarkResults benchmarkResults) { + boolean validResults = true; + + if (benchmarkResults.getCpuCyclesPerSec() < CPU_FREQUENCY_LOWER_LIMIT) { + logger.log(Level.WARNING, "CPU frequency below accepted value. Value: " + benchmarkResults.getCpuCyclesPerSec() + " GHz"); + validResults = false; + } + + if (benchmarkResults.getMemoryWriteSpeedGBs() < MEMORY_WRITE_SPEED_LOWER_LIMIT) { + logger.log(Level.WARNING, "Memory write speed below accepted value. Value: " + benchmarkResults.getMemoryWriteSpeedGBs() + " GB/s"); + validResults = false; + } + + if (benchmarkResults.getMemoryReadSpeedGBs() < MEMORY_READ_SPEED_LOWER_LIMIT) { + logger.log(Level.WARNING, "Memory read speed below accepted value. Value: " + benchmarkResults.getMemoryReadSpeedGBs() + " GB/s"); + validResults = false; + } + + if (benchmarkResults.getDiskSpeedMbs() < DISK_SPEED_LOWER_LIMIT) { + logger.log(Level.WARNING, "Disk speed below accepted value. Value: " + benchmarkResults.getDiskSpeedMbs() + " MB/s"); + validResults = false; + } + return validResults; + } + +} diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/BenchmarkResults.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/BenchmarkResults.java new file mode 100644 index 00000000000..590857b81d6 --- /dev/null +++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/BenchmarkResults.java @@ -0,0 +1,47 @@ +package com.yahoo.vespa.hosted.node.verification.hardware.benchmarks; + +/** + * Created by sgrostad on 11/07/2017. + * Stores results from benchmarks + */ +public class BenchmarkResults { + + private double cpuCyclesPerSec; + private double diskSpeedMbs; + private double memoryWriteSpeedGBs; + private double memoryReadSpeedGBs; + + + public double getMemoryWriteSpeedGBs() { + return memoryWriteSpeedGBs; + } + + public void setMemoryWriteSpeedGBs(double memoryWriteSpeedGBs) { + this.memoryWriteSpeedGBs = memoryWriteSpeedGBs; + } + + public double getMemoryReadSpeedGBs() { + return memoryReadSpeedGBs; + } + + public void setMemoryReadSpeedGBs(double memoryReadSpeedGBs) { + this.memoryReadSpeedGBs = memoryReadSpeedGBs; + } + + public double getCpuCyclesPerSec() { + return cpuCyclesPerSec; + } + + public void setCpuCyclesPerSec(double cpuCycles) { + this.cpuCyclesPerSec = cpuCycles; + } + + public double getDiskSpeedMbs() { + return diskSpeedMbs; + } + + public void setDiskSpeedMbs(double diskSpeedMbs) { + this.diskSpeedMbs = diskSpeedMbs; + } + +} diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/CPUBenchmark.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/CPUBenchmark.java index 836b8213ffb..169fb11447b 100644 --- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/CPUBenchmark.java +++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/CPUBenchmark.java @@ -23,12 +23,12 @@ public class CPUBenchmark implements Benchmark { private static final int SEARCH_ELEMENT_INDEX = 1; private static final int RETURN_ELEMENT_INDEX = 0; private static final Logger logger = Logger.getLogger(CPUBenchmark.class.getName()); - private final HardwareResults hardwareResults; + private final BenchmarkResults benchmarkResults; private final CommandExecutor commandExecutor; - public CPUBenchmark(HardwareResults hardwareResults, CommandExecutor commandExecutor) { - this.hardwareResults = hardwareResults; + public CPUBenchmark(BenchmarkResults benchmarkResults, CommandExecutor commandExecutor) { + this.benchmarkResults = benchmarkResults; this.commandExecutor = commandExecutor; } @@ -52,7 +52,7 @@ public class CPUBenchmark implements Benchmark { protected void setCpuCyclesPerSec(ArrayList parseResults) { double cpuCyclesPerSec = getCyclesPerSecond(parseResults); if (cpuCyclesPerSec > 0) { - hardwareResults.setCpuCyclesPerSec(cpuCyclesPerSec); + benchmarkResults.setCpuCyclesPerSec(cpuCyclesPerSec); } } diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/DiskBenchmark.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/DiskBenchmark.java index 8b5502cd415..d2a83bf94c5 100644 --- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/DiskBenchmark.java +++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/DiskBenchmark.java @@ -17,7 +17,7 @@ import java.util.logging.Logger; */ public class DiskBenchmark implements Benchmark { - private static final String DISK_BENCHMARK_COMMAND = "time (dd if=/dev/zero of=/tmp/tempfile bs=16k count=16k > /dev/null; sync; rm /tmp/tempfile) 2>&1 | grep bytes | awk '{ print $8 \" \" $9 }'"; + private static final String DISK_BENCHMARK_COMMAND = "time (dd if=/dev/zero of=/tmp/tempfile bs=16G count=1 > /dev/null; sync; rm /tmp/tempfile) 2>&1 | grep bytes | awk '{ print $8 \" \" $9 }'"; private static final String KILO_BYTE_SEARCH_WORD = "kB/s"; private static final String MEGA_BYTE_SEARCH_WORD = "MB/s"; private static final String GIGA_BYTE_SEARCH_WORD = "GB/s"; @@ -25,11 +25,11 @@ public class DiskBenchmark implements Benchmark { private static final int SEARCH_ELEMENT_INDEX = 1; private static final int RETURN_ELEMENT_INDEX = 0; private static final Logger logger = Logger.getLogger(DiskBenchmark.class.getName()); - private final HardwareResults hardwareResults; + private final BenchmarkResults benchmarkResults; private final CommandExecutor commandExecutor; - public DiskBenchmark(HardwareResults hardwareResults, CommandExecutor commandExecutor) { - this.hardwareResults = hardwareResults; + public DiskBenchmark(BenchmarkResults benchmarkResults, CommandExecutor commandExecutor) { + this.benchmarkResults = benchmarkResults; this.commandExecutor = commandExecutor; } @@ -51,7 +51,7 @@ public class DiskBenchmark implements Benchmark { protected void setDiskSpeed(ParseResult parseResult) { double diskSpeedMBs = getDiskSpeedInMBs(parseResult); - hardwareResults.setDiskSpeedMbs(diskSpeedMBs); + benchmarkResults.setDiskSpeedMbs(diskSpeedMBs); } protected double getDiskSpeedInMBs(ParseResult parseResult) { diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/HardwareResults.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/HardwareResults.java deleted file mode 100644 index 6d1cb01fdbb..00000000000 --- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/HardwareResults.java +++ /dev/null @@ -1,56 +0,0 @@ -package com.yahoo.vespa.hosted.node.verification.hardware.benchmarks; - -/** - * Created by sgrostad on 11/07/2017. - * Stores results from benchmarks - */ -public class HardwareResults { - - private double cpuCyclesPerSec; - private double diskSpeedMbs; - private boolean ipv6Connectivity; - private Double memoryWriteSpeedGBs; - private Double memoryReadSpeedGBs; - - - public Double getMemoryWriteSpeedGBs() { - return memoryWriteSpeedGBs; - } - - public void setMemoryWriteSpeedGBs(Double memoryWriteSpeedGBs) { - this.memoryWriteSpeedGBs = memoryWriteSpeedGBs; - } - - public Double getMemoryReadSpeedGBs() { - return memoryReadSpeedGBs; - } - - public void setMemoryReadSpeedGBs(Double memoryReadSpeedGBs) { - this.memoryReadSpeedGBs = memoryReadSpeedGBs; - } - - public double getCpuCyclesPerSec() { - return cpuCyclesPerSec; - } - - public void setCpuCyclesPerSec(double cpuCycles) { - this.cpuCyclesPerSec = cpuCycles; - } - - public double getDiskSpeedMbs() { - return diskSpeedMbs; - } - - public void setDiskSpeedMbs(double diskSpeedMbs) { - this.diskSpeedMbs = diskSpeedMbs; - } - - public boolean isIpv6Connectivity() { - return ipv6Connectivity; - } - - public void setIpv6Connectivity(boolean ipv6Connectivity) { - this.ipv6Connectivity = ipv6Connectivity; - } - -} diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/MemoryBenchmark.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/MemoryBenchmark.java index 898c2071b92..3ef5796e032 100644 --- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/MemoryBenchmark.java +++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/MemoryBenchmark.java @@ -18,7 +18,7 @@ public class MemoryBenchmark implements Benchmark { private static final String MEM_BENCHMARK_CREATE_FOLDER = "mkdir -p RAM_test"; private static final String MEM_BENCHMARK_MOUNT_TMPFS = "sudo mount tmpfs -t tmpfs RAM_test/"; - private static final String MEM_BENCHMARK_UNMOUNT_TMPFS = "umount RAM_test"; + private static final String MEM_BENCHMARK_UNMOUNT_TMPFS = "sudo umount RAM_test"; private static final String MEM_BENCHMARK_DELETE_FOLDER = "rm -rf RAM_test"; private static final String MEM_BENCHMARK_WRITE_SPEED = "dd if=/dev/zero of=RAM_test/data_tmp bs=1M count=512"; private static final String MEM_BENCHMARK_READ_SPEED = "dd if=RAM_test/data_tmp of=/dev/null bs=1M count=512"; @@ -27,11 +27,11 @@ public class MemoryBenchmark implements Benchmark { private static final int SEARCH_ELEMENT_INDEX = 8; private static final int RETURN_ELEMENT_INDEX = 7; private static final Logger logger = Logger.getLogger(MemoryBenchmark.class.getName()); - private final HardwareResults hardwareResults; + private final BenchmarkResults benchmarkResults; private final CommandExecutor commandExecutor; - public MemoryBenchmark(HardwareResults hardwareResults, CommandExecutor commandExecutor) { - this.hardwareResults = hardwareResults; + public MemoryBenchmark(BenchmarkResults benchmarkResults, CommandExecutor commandExecutor) { + this.benchmarkResults = benchmarkResults; this.commandExecutor = commandExecutor; } @@ -78,13 +78,13 @@ public class MemoryBenchmark implements Benchmark { protected void updateMemoryWriteSpeed(String memorySpeed) { if (!isValidMemory(memorySpeed)) return; double memoryWriteSpeedGbs = Double.parseDouble(memorySpeed); - hardwareResults.setMemoryWriteSpeedGBs(memoryWriteSpeedGbs); + benchmarkResults.setMemoryWriteSpeedGBs(memoryWriteSpeedGbs); } protected void updateMemoryReadSpeed(String memorySpeed) { if (!isValidMemory(memorySpeed)) return; double memoryReadSpeedGbs = Double.parseDouble(memorySpeed); - hardwareResults.setMemoryReadSpeedGBs(memoryReadSpeedGbs); + benchmarkResults.setMemoryReadSpeedGBs(memoryReadSpeedGbs); } protected boolean isValidMemory(String benchmarkOutput) { diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/NetBenchmark.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/NetBenchmark.java deleted file mode 100644 index 9133cb47690..00000000000 --- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/NetBenchmark.java +++ /dev/null @@ -1,60 +0,0 @@ -package com.yahoo.vespa.hosted.node.verification.hardware.benchmarks; - -import com.yahoo.vespa.hosted.node.verification.commons.CommandExecutor; -import com.yahoo.vespa.hosted.node.verification.commons.OutputParser; -import com.yahoo.vespa.hosted.node.verification.commons.ParseInstructions; -import com.yahoo.vespa.hosted.node.verification.commons.ParseResult; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.logging.Level; -import java.util.logging.Logger; - -/** - * Created by sgrostad on 11/07/2017. - */ -public class NetBenchmark implements Benchmark { - - private static final String NET_BENCHMARK_COMMAND = "ping6 -c 10 www.yahoo.com | grep transmitted"; - private static final String PING_SEARCH_WORD = "loss,"; - private static final String SPLIT_REGEX_STRING = "\\s+"; - private static final int SEARCH_ELEMENT_INDEX = 7; - private static final int RETURN_ELEMENT_INDEX = 5; - private static final Logger logger = Logger.getLogger(NetBenchmark.class.getName()); - private final HardwareResults hardwareResults; - private final CommandExecutor commandExecutor; - - public NetBenchmark(HardwareResults hardwareResults, CommandExecutor commandExecutor) { - this.hardwareResults = hardwareResults; - this.commandExecutor = commandExecutor; - } - - public void doBenchmark() { - try { - ArrayList commandOutput = commandExecutor.executeCommand(NET_BENCHMARK_COMMAND); - ParseResult parseResult = parsePingResponse(commandOutput); - setIpv6Connectivity(parseResult); - } catch (IOException e) { - logger.log(Level.WARNING, "Failed to perform net benchmark", e); - } - } - - protected ParseResult parsePingResponse(ArrayList commandOutput) { - ArrayList searchWords = new ArrayList<>(Arrays.asList(PING_SEARCH_WORD)); - ParseInstructions parseInstructions = new ParseInstructions(SEARCH_ELEMENT_INDEX, RETURN_ELEMENT_INDEX, SPLIT_REGEX_STRING, searchWords); - return OutputParser.parseSingleOutput(parseInstructions, commandOutput); - - } - - protected void setIpv6Connectivity(ParseResult parseResult) { - if (parseResult.getSearchWord().equals(PING_SEARCH_WORD)) { - String pingResponse = parseResult.getValue(); - String packetLoss = pingResponse.replaceAll("[^\\d.]", ""); - if (packetLoss.equals("")) return; - if (Double.parseDouble(packetLoss) > 99) return; - hardwareResults.setIpv6Connectivity(true); - } - } - -} diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/net/Client.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/net/Client.java deleted file mode 100644 index 26c97e0707b..00000000000 --- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/net/Client.java +++ /dev/null @@ -1,46 +0,0 @@ -package com.yahoo.vespa.hosted.node.verification.hardware.net; - -import java.io.DataOutputStream; -import java.io.File; -import java.io.FileInputStream; -import java.io.IOException; -import java.net.Socket; - -/** - * Created by olaa on 19/07/2017. - * Sends file to server, for checking connection speed. - * Not used, can be deleted - */ -public class Client { - - public void sendFile(Socket socket, File file) throws IOException { - try (FileInputStream inputStream = new FileInputStream(file); - DataOutputStream outputStream = new DataOutputStream(socket.getOutputStream())) { - int fileSize = (int) file.length(); - byte[] buffer = new byte[fileSize]; - outputStream.writeUTF(file.getName()); - int receivedBytesCount; - while ((receivedBytesCount = inputStream.read(buffer)) != -1) { - outputStream.write(buffer, 0, receivedBytesCount); - } - } catch (IOException e) { - e.printStackTrace(); - } finally { - socket.close(); - } - } - - public static void main(String[] args) { - Client client = new Client(); - File file = new File("src/test/resources/testReadFile.txt"); - double start = System.currentTimeMillis() / 1000.0; - try { - client.sendFile(new Socket("localhost", 10000), file); - } catch (IOException e) { - e.printStackTrace(); - } - double finish = System.currentTimeMillis() / 1000.0; - System.out.println(((double) file.length() / (finish - start)) + " B/s"); - } - -} diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/net/Server.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/net/Server.java deleted file mode 100644 index 96d363f8da2..00000000000 --- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/net/Server.java +++ /dev/null @@ -1,41 +0,0 @@ -package com.yahoo.vespa.hosted.node.verification.hardware.net; - -import java.io.DataInputStream; -import java.io.FileOutputStream; -import java.io.IOException; -import java.net.ServerSocket; -import java.net.Socket; - -/** - * Created by olaa on 19/07/2017. - * Receives file from client. - * Not used, can be deleted - */ -public class Server { - - public void serve(int portNumber) throws IOException { - ServerSocket server; - Socket client; - server = new ServerSocket(portNumber); - client = server.accept(); - DataInputStream dataInputStream = new DataInputStream(client.getInputStream()); - DataInputStream inputStream = new DataInputStream(client.getInputStream()); - String fileName = dataInputStream.readUTF(); - FileOutputStream fileOutputStream = new FileOutputStream("./" + fileName); - byte[] buffer = new byte[65535]; - int currentLength; - while ((currentLength = inputStream.read(buffer)) != -1) { - fileOutputStream.write(buffer, 0, currentLength); - } - } - - public static void main(String[] args) { - Server server = new Server(); - try { - server.serve(10000); - } catch (IOException e) { - e.printStackTrace(); - } - } - -} diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/yamasreport/HardwareReportMetrics.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/yamasreport/HardwareReportMetrics.java index 6e4d868b710..e0c1c704174 100644 --- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/yamasreport/HardwareReportMetrics.java +++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/yamasreport/HardwareReportMetrics.java @@ -8,17 +8,15 @@ import com.fasterxml.jackson.annotation.JsonProperty; public class HardwareReportMetrics { @JsonProperty - private Double cpuCyclesPerSec; + private double cpuCyclesPerSec; @JsonProperty - private Double diskSpeedMbs; + private double diskSpeedMbs; @JsonProperty - private Boolean ipv6Connectivity; + private double memoryWriteSpeedGBs; @JsonProperty - private Double memoryWriteSpeedGBs; - @JsonProperty - private Double memoryReadSpeedGBs; + private double memoryReadSpeedGBs; - public void setCpuCyclesPerSec(Double cpuCyclesPerSec) { + public void setCpuCyclesPerSec(double cpuCyclesPerSec) { this.cpuCyclesPerSec = cpuCyclesPerSec; } @@ -26,9 +24,6 @@ public class HardwareReportMetrics { this.diskSpeedMbs = diskSpeedMbs != null ? diskSpeedMbs : -1; } - public void setIpv6Connectivity(Boolean ipv6Connectivity) { - this.ipv6Connectivity = ipv6Connectivity; - } public void setMemoryWriteSpeedGBs(Double memoryWriteSpeedGBs) { this.memoryWriteSpeedGBs = memoryWriteSpeedGBs != null ? memoryWriteSpeedGBs : -1; @@ -38,4 +33,20 @@ public class HardwareReportMetrics { this.memoryReadSpeedGBs = memoryReadSpeedGBs != null ? memoryReadSpeedGBs : -1; } + public Double getCpuCyclesPerSec() { + return cpuCyclesPerSec; + } + + public double getDiskSpeedMbs() { + return diskSpeedMbs; + } + + public double getMemoryWriteSpeedGBs() { + return memoryWriteSpeedGBs; + } + + public double getMemoryReadSpeedGBs() { + return memoryReadSpeedGBs; + } + } diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/yamasreport/YamasHardwareReport.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/yamasreport/YamasHardwareReport.java index 517e06af9b1..c76edd39b3a 100644 --- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/yamasreport/YamasHardwareReport.java +++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/yamasreport/YamasHardwareReport.java @@ -3,7 +3,7 @@ package com.yahoo.vespa.hosted.node.verification.hardware.yamasreport; import com.fasterxml.jackson.annotation.JsonAnyGetter; import com.fasterxml.jackson.annotation.JsonAnySetter; import com.fasterxml.jackson.annotation.JsonProperty; -import com.yahoo.vespa.hosted.node.verification.hardware.benchmarks.HardwareResults; +import com.yahoo.vespa.hosted.node.verification.hardware.benchmarks.BenchmarkResults; import java.util.HashMap; import java.util.Map; @@ -49,14 +49,13 @@ public class YamasHardwareReport { routing = new JsonObjectWrapper<>("yamas", wrap); } - public void createFromHardwareResults(HardwareResults hardwareResults) { + public void createReportFromBenchmarkResults(BenchmarkResults benchmarkResults) { metrics = new HardwareReportMetrics(); dimensions = new HardwareReportDimensions(); - metrics.setCpuCyclesPerSec(hardwareResults.getCpuCyclesPerSec()); - metrics.setDiskSpeedMbs(hardwareResults.getDiskSpeedMbs()); - metrics.setIpv6Connectivity(hardwareResults.isIpv6Connectivity()); - metrics.setMemoryWriteSpeedGBs(hardwareResults.getMemoryWriteSpeedGBs()); - metrics.setMemoryReadSpeedGBs(hardwareResults.getMemoryReadSpeedGBs()); + metrics.setCpuCyclesPerSec(benchmarkResults.getCpuCyclesPerSec()); + metrics.setDiskSpeedMbs(benchmarkResults.getDiskSpeedMbs()); + metrics.setMemoryWriteSpeedGBs(benchmarkResults.getMemoryWriteSpeedGBs()); + metrics.setMemoryReadSpeedGBs(benchmarkResults.getMemoryReadSpeedGBs()); } class JsonObjectWrapper { diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/HardwareNodeComparator.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/HardwareNodeComparator.java index adec38c2ea4..a61ba11ff22 100644 --- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/HardwareNodeComparator.java +++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/HardwareNodeComparator.java @@ -1,6 +1,7 @@ package com.yahoo.vespa.hosted.node.verification.spec; import com.yahoo.vespa.hosted.node.verification.spec.retrievers.HardwareInfo; +import com.yahoo.vespa.hosted.node.verification.spec.retrievers.HardwareInfo.DiskType; import com.yahoo.vespa.hosted.node.verification.spec.yamasreport.SpecReportDimensions; import com.yahoo.vespa.hosted.node.verification.spec.yamasreport.SpecReportMetrics; import com.yahoo.vespa.hosted.node.verification.spec.yamasreport.YamasSpecReport; @@ -11,22 +12,22 @@ import com.yahoo.vespa.hosted.node.verification.spec.yamasreport.YamasSpecReport */ public class HardwareNodeComparator { - public static YamasSpecReport compare(HardwareInfo node, HardwareInfo actualHardware) { + public static YamasSpecReport compare(HardwareInfo nodeRepoHardwareInfo, HardwareInfo actualHardware) { Boolean equalHardware = true; YamasSpecReport yamasSpecReport = new YamasSpecReport(); SpecReportDimensions specReportDimensions = new SpecReportDimensions(); SpecReportMetrics specReportMetrics = new SpecReportMetrics(); - if (node == null || actualHardware == null) { + if (nodeRepoHardwareInfo == null || actualHardware == null) { return yamasSpecReport; } - setReportMetrics(node, actualHardware, specReportMetrics); + setReportMetrics(nodeRepoHardwareInfo, actualHardware, specReportMetrics); - equalHardware &= compareMemory(node, actualHardware, specReportDimensions); - equalHardware &= compareCPU(node, actualHardware, specReportDimensions); - equalHardware &= compareNetInterface(node, actualHardware, specReportDimensions); - equalHardware &= compareDisk(node, actualHardware, specReportDimensions); + equalHardware &= compareMemory(nodeRepoHardwareInfo, actualHardware, specReportDimensions); + equalHardware &= compareCPU(nodeRepoHardwareInfo, actualHardware, specReportDimensions); + equalHardware &= compareNetInterface(nodeRepoHardwareInfo, actualHardware, specReportDimensions); + equalHardware &= compareDisk(nodeRepoHardwareInfo, actualHardware, specReportDimensions); specReportMetrics.setMatch(equalHardware); yamasSpecReport.setDimensions(specReportDimensions); @@ -62,8 +63,8 @@ public class HardwareNodeComparator { } private static void setDiskTypeMetrics(HardwareInfo node, HardwareInfo actualHardware, SpecReportMetrics specReportMetrics) { - Boolean expectedFastDisk = node.getFastDisk(); - Boolean actualFastDisk = actualHardware.getFastDisk(); + DiskType expectedFastDisk = node.getDiskType(); + DiskType actualFastDisk = actualHardware.getDiskType(); if (expectedFastDisk != null && actualFastDisk != null && expectedFastDisk != actualFastDisk) { specReportMetrics.setExpectedDiskType(expectedFastDisk); specReportMetrics.setActualDiskType(actualFastDisk); @@ -86,6 +87,9 @@ public class HardwareNodeComparator { specReportMetrics.setExpectedInterfaceSpeed(expectedInterfaceSpeed); specReportMetrics.setActualInterfaceSpeed(actualInterfaceSpeed); } + if (!actualHardware.isIpv6Connection()) { + specReportMetrics.setActualIpv6Connection(false); + } } private static boolean compareCPU(HardwareInfo node, HardwareInfo actualHardware, SpecReportDimensions specReportDimensions) { @@ -102,8 +106,8 @@ public class HardwareNodeComparator { private static boolean compareNetInterface(HardwareInfo node, HardwareInfo actualHardware, SpecReportDimensions specReportDimensions) { boolean equalNetInterfaceSpeed = insideThreshold(node.getInterfaceSpeedMbs(), actualHardware.getInterfaceSpeedMbs()); - boolean equalIpv6 = node.getIpv6Connectivity() == actualHardware.getIpv6Connectivity(); - boolean equalIpv4 = node.getIpv4Connectivity() == actualHardware.getIpv4Connectivity(); + boolean equalIpv6 = node.getIpv6Interface() == actualHardware.getIpv6Interface(); + boolean equalIpv4 = node.getIpv4Interface() == actualHardware.getIpv4Interface(); specReportDimensions.setNetInterfaceSpeedMatch(equalNetInterfaceSpeed); specReportDimensions.setIpv6Match(equalIpv6); specReportDimensions.setIpv4Match(equalIpv4); @@ -112,9 +116,9 @@ public class HardwareNodeComparator { } private static boolean compareDisk(HardwareInfo node, HardwareInfo actualHardware, SpecReportDimensions specReportDimensions) { - boolean equalDiskType = node.getFastDisk() == actualHardware.getFastDisk(); + boolean equalDiskType = node.getDiskType() == actualHardware.getDiskType(); boolean equalDiskSize = insideThreshold(node.getMinDiskAvailableGb(), actualHardware.getMinDiskAvailableGb()); - specReportDimensions.setFastDiskMatch(equalDiskType); + specReportDimensions.setDiskTypeMatch(equalDiskType); specReportDimensions.setDiskAvailableMatch(equalDiskSize); return equalDiskType && equalDiskSize; } diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/HostURLGenerator.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/HostURLGenerator.java index 6f619141d2c..2fb81d4db63 100644 --- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/HostURLGenerator.java +++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/HostURLGenerator.java @@ -1,7 +1,11 @@ package com.yahoo.vespa.hosted.node.verification.spec; +import com.yahoo.vespa.hosted.node.verification.commons.CommandExecutor; + +import java.io.IOException; import java.net.MalformedURLException; import java.net.URL; +import java.util.ArrayList; /** * Created by olaa on 14/07/2017. @@ -10,14 +14,56 @@ import java.net.URL; public class HostURLGenerator { private static final String NODE_HOSTNAME_PREFIX = "/nodes/v2/node/"; + private static final String PORT_NUMBER = ":4080"; + private static final String CONFIG_SERVER_HOST_NAME_COMMAND = "yinst set | grep cfg"; + private static final String HTTP = "http://"; + private static final String PARSE_OUT_HOSTNAMES_REGEX = "\\s+"; + private static final String PARSE_ALL_HOSTNAMES_REGEX = ","; + private static final String PROTOCOL_REGEX = "^(https?|file)://.*$"; + + public static ArrayList generateNodeInfoUrl(CommandExecutor commandExecutor) throws IOException { + String[] configServerHostNames = getConfigServerHostNames(commandExecutor); + String nodeHostName = generateNodeHostName(commandExecutor); + ArrayList nodeInfoUrls = new ArrayList<>(); + for (String configServerHostName : configServerHostNames) { + nodeInfoUrls.add(buildNodeInfoURL(configServerHostName, nodeHostName)); + } + return nodeInfoUrls; + } + + protected static URL buildNodeInfoURL(String configServerHostName, String nodeHostName) throws MalformedURLException { + if (configServerHostName.matches(PROTOCOL_REGEX)) { + return new URL(configServerHostName + NODE_HOSTNAME_PREFIX + nodeHostName); + } + return new URL(HTTP + configServerHostName + PORT_NUMBER + NODE_HOSTNAME_PREFIX + nodeHostName); + } + + protected static String generateNodeHostName(CommandExecutor commandExecutor) throws IOException { + String nodeHostName = getEnvironmentVariable(commandExecutor); + return nodeHostName; + } + + protected static String getEnvironmentVariable(CommandExecutor commandExecutor) throws IOException { + ArrayList output = commandExecutor.executeCommand("hostname"); + if (output.size() == 1) { + return output.get(0); + } + throw new IOException("Unexpected output from \"hostname\" command."); + } - protected URL generateNodeInfoUrl(String configServerHostName) throws MalformedURLException { - String nodeHostName = getEnvironmentVariable("HOSTNAME"); - return new URL(configServerHostName + NODE_HOSTNAME_PREFIX + nodeHostName); + protected static String[] getConfigServerHostNames(CommandExecutor commandExecutor) throws IOException { + ArrayList output = commandExecutor.executeCommand(CONFIG_SERVER_HOST_NAME_COMMAND); + if (output.size() != 1) + throw new IOException("Expected one line return from the command: " + CONFIG_SERVER_HOST_NAME_COMMAND); + String[] configServerHostNames = parseOutHostNames(output.get(0)); + return configServerHostNames; } - protected String getEnvironmentVariable(String variableName) { - return System.getenv(variableName); + private static String[] parseOutHostNames(String output) throws IOException { + String[] outputSplit = output.trim().split(PARSE_OUT_HOSTNAMES_REGEX); + if (outputSplit.length != 2) throw new IOException("Expected config server hsot names to have index 1"); + String[] configServerHostNames = outputSplit[1].split(PARSE_ALL_HOSTNAMES_REGEX); + return configServerHostNames; } } diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/SpecVerifier.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/SpecVerifier.java index 5f5690a16f6..08d5d1b1d74 100644 --- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/SpecVerifier.java +++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/SpecVerifier.java @@ -2,17 +2,18 @@ package com.yahoo.vespa.hosted.node.verification.spec; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; +import com.yahoo.vespa.hosted.node.verification.commons.CommandExecutor; import com.yahoo.vespa.hosted.node.verification.spec.noderepo.IPAddressVerifier; -import com.yahoo.vespa.hosted.node.verification.spec.noderepo.NodeGenerator; -import com.yahoo.vespa.hosted.node.verification.spec.noderepo.NodeInfoRetriever; -import com.yahoo.vespa.hosted.node.verification.spec.noderepo.NodeJsonModel; +import com.yahoo.vespa.hosted.node.verification.spec.noderepo.NodeJsonConverter; +import com.yahoo.vespa.hosted.node.verification.spec.noderepo.NodeRepoInfoRetriever; +import com.yahoo.vespa.hosted.node.verification.spec.noderepo.NodeRepoJsonModel; import com.yahoo.vespa.hosted.node.verification.spec.retrievers.HardwareInfo; import com.yahoo.vespa.hosted.node.verification.spec.retrievers.HardwareInfoRetriever; import com.yahoo.vespa.hosted.node.verification.spec.yamasreport.YamasSpecReport; -import java.net.MalformedURLException; +import java.io.IOException; import java.net.URL; -import java.util.logging.Level; +import java.util.ArrayList; import java.util.logging.Logger; /** @@ -24,26 +25,28 @@ public class SpecVerifier { private static final Logger logger = Logger.getLogger(SpecVerifier.class.getName()); - public void verifySpec(String zoneHostName) { - URL nodeRepoUrl; - try { - HostURLGenerator hostURLGenerator = new HostURLGenerator(); - nodeRepoUrl = hostURLGenerator.generateNodeInfoUrl(zoneHostName); - } catch (MalformedURLException e) { - logger.log(Level.WARNING, "Failed to generate config server url", e); - return; - } - NodeJsonModel nodeJsonModel = NodeInfoRetriever.retrieve(nodeRepoUrl); - HardwareInfo node = NodeGenerator.convertJsonModel(nodeJsonModel); - HardwareInfo actualHardware = HardwareInfoRetriever.retrieve(); - YamasSpecReport yamasSpecReport = HardwareNodeComparator.compare(node, actualHardware); + public static boolean verifySpec(CommandExecutor commandExecutor) throws IOException { + NodeRepoJsonModel nodeRepoJsonModel = getNodeRepositoryJSON(commandExecutor); + HardwareInfo actualHardware = HardwareInfoRetriever.retrieve(commandExecutor); + YamasSpecReport yamasSpecReport = makeYamasSpecReport(actualHardware, nodeRepoJsonModel); + printResults(yamasSpecReport); + return yamasSpecReport.getMetrics().isMatch(); + } + + protected static YamasSpecReport makeYamasSpecReport(HardwareInfo actualHardware, NodeRepoJsonModel nodeRepoJsonModel) { + YamasSpecReport yamasSpecReport = HardwareNodeComparator.compare(NodeJsonConverter.convertJsonModelToHardwareInfo(nodeRepoJsonModel), actualHardware); IPAddressVerifier ipAddressVerifier = new IPAddressVerifier(); - ipAddressVerifier.reportFaultyIpAddresses(nodeJsonModel, yamasSpecReport); + ipAddressVerifier.reportFaultyIpAddresses(nodeRepoJsonModel, yamasSpecReport); + return yamasSpecReport; + } - printResults(yamasSpecReport); + protected static NodeRepoJsonModel getNodeRepositoryJSON(CommandExecutor commandExecutor) throws IOException { + ArrayList nodeInfoUrls = HostURLGenerator.generateNodeInfoUrl(commandExecutor); + NodeRepoJsonModel nodeRepoJsonModel = NodeRepoInfoRetriever.retrieve(nodeInfoUrls); + return nodeRepoJsonModel; } - private void printResults(YamasSpecReport yamasSpecReport) { + private static void printResults(YamasSpecReport yamasSpecReport) { //TODO: Instead of println, report JSON to YAMAS ObjectMapper om = new ObjectMapper(); try { @@ -53,19 +56,9 @@ public class SpecVerifier { } } - - public static void main(String[] args) { - /** - * When testing in docker container - * docker run --hostname 13305821.ostk.bm2.prod.gq1.yahoo.com --name 13305821.ostk.bm2.prod.gq1.yahoo.com [image] - */ - if (args.length != 1) { - throw new RuntimeException("Expected only 1 argument - config server zone url"); - } - - String zoneHostName = args[0]; - SpecVerifier specVerifier = new SpecVerifier(); - specVerifier.verifySpec(zoneHostName); + public static void main(String[] args) throws IOException { + CommandExecutor commandExecutor = new CommandExecutor(); + SpecVerifier.verifySpec(commandExecutor); } } diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/noderepo/IPAddressVerifier.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/noderepo/IPAddressVerifier.java index 1e343ee4686..690a57ba242 100644 --- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/noderepo/IPAddressVerifier.java +++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/noderepo/IPAddressVerifier.java @@ -23,8 +23,8 @@ public class IPAddressVerifier { private static final Logger logger = Logger.getLogger(IPAddressVerifier.class.getName()); - public void reportFaultyIpAddresses(NodeJsonModel nodeJsonModel, YamasSpecReport yamasSpecReport) { - String[] faultyIpAddresses = getFaultyIpAddresses(nodeJsonModel.getIpv6Address(), nodeJsonModel.getAdditionalIpAddresses()); + public void reportFaultyIpAddresses(NodeRepoJsonModel nodeRepoJsonModel, YamasSpecReport yamasSpecReport) { + String[] faultyIpAddresses = getFaultyIpAddresses(nodeRepoJsonModel.getIpv6Address(), nodeRepoJsonModel.getAdditionalIpAddresses()); if (faultyIpAddresses.length > 0) { yamasSpecReport.setFaultyIpAddresses(faultyIpAddresses); } @@ -36,8 +36,6 @@ public class IPAddressVerifier { String ipAddressInLookupFormat = convertToLookupFormat(ipAddress); String attributeName = ipAddress; DirContext ctx = new InitialDirContext(env); - //98.138.253.109 - //Attributes attrs = ctx.getAttributes("1.0.6.f.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.2.9.7.7.0.c.0.4.9.9.8.0.2.0.0.1.ip6.arpa",new String[] {"PTR"}); Attributes attrs = ctx.getAttributes(attributeName, new String[]{"PTR"}); for (NamingEnumeration ae = attrs.getAll(); ae.hasMoreElements(); ) { Attribute attr = ae.next(); diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/noderepo/NodeGenerator.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/noderepo/NodeGenerator.java deleted file mode 100644 index 680cbbb132e..00000000000 --- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/noderepo/NodeGenerator.java +++ /dev/null @@ -1,23 +0,0 @@ -package com.yahoo.vespa.hosted.node.verification.spec.noderepo; - -import com.yahoo.vespa.hosted.node.verification.spec.retrievers.HardwareInfo; - -/** - * Created by olaa on 07/07/2017. - * Converts a NodeJsonModel object to a HardwareInfo object. - */ -public class NodeGenerator { - - private static void addStandardSpecifications(HardwareInfo node) { - node.setIpv4Connectivity(true); - node.setIpv6Connectivity(true); - node.setInterfaceSpeedMbs(1000); - } - - public static HardwareInfo convertJsonModel(NodeJsonModel nodeJsonModel) { - HardwareInfo node = nodeJsonModel.copyToHardwareInfo(); - addStandardSpecifications(node); - return node; - } - -} diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/noderepo/NodeInfoRetriever.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/noderepo/NodeInfoRetriever.java deleted file mode 100644 index 9297ad873be..00000000000 --- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/noderepo/NodeInfoRetriever.java +++ /dev/null @@ -1,31 +0,0 @@ -package com.yahoo.vespa.hosted.node.verification.spec.noderepo; - -import com.fasterxml.jackson.databind.ObjectMapper; - -import java.io.IOException; -import java.net.URL; -import java.util.logging.Level; -import java.util.logging.Logger; - -/** - * Created by olaa on 04/07/2017. - * Parse JSON from node repository and stores information as a NodeJsonModel object. - */ -public class NodeInfoRetriever { - - private static final Logger logger = Logger.getLogger(NodeInfoRetriever.class.getName()); - - public static NodeJsonModel retrieve(URL url) { - NodeJsonModel nodeJsonModel; - ObjectMapper objectMapper = new ObjectMapper(); - try { - nodeJsonModel = objectMapper.readValue(url, NodeJsonModel.class); - } catch (IOException e) { - logger.log(Level.WARNING, "Failed to parse JSON", e); - return null; - } - return nodeJsonModel; - } - -} - diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/noderepo/NodeJsonConverter.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/noderepo/NodeJsonConverter.java new file mode 100644 index 00000000000..dfd2bb011a2 --- /dev/null +++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/noderepo/NodeJsonConverter.java @@ -0,0 +1,29 @@ +package com.yahoo.vespa.hosted.node.verification.spec.noderepo; + +import com.yahoo.vespa.hosted.node.verification.spec.retrievers.HardwareInfo; + +/** + * Created by olaa on 07/07/2017. + * Converts a NodeRepoJsonModel object to a HardwareInfo object. + */ +public class NodeJsonConverter { + + private static void addStandardSpecifications(HardwareInfo nodeRepoHardwareInfo) { + nodeRepoHardwareInfo.setIpv4Interface(true); + nodeRepoHardwareInfo.setInterfaceSpeedMbs(1000); + } + + protected static void setIpv6AddressConnectivity(NodeRepoJsonModel nodeRepoJsonModel, HardwareInfo nodeRepoHardwareInfo) { + if (nodeRepoJsonModel.getIpv6Address() != null) { + nodeRepoHardwareInfo.setIpv6Interface(true); + } + } + + public static HardwareInfo convertJsonModelToHardwareInfo(NodeRepoJsonModel nodeRepoJsonModel) { + HardwareInfo nodeRepoHardwareInfo = nodeRepoJsonModel.copyToHardwareInfo(); + addStandardSpecifications(nodeRepoHardwareInfo); + setIpv6AddressConnectivity(nodeRepoJsonModel, nodeRepoHardwareInfo); + return nodeRepoHardwareInfo; + } + +} diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/noderepo/NodeJsonModel.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/noderepo/NodeJsonModel.java deleted file mode 100644 index f5bbf39e876..00000000000 --- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/noderepo/NodeJsonModel.java +++ /dev/null @@ -1,51 +0,0 @@ -package com.yahoo.vespa.hosted.node.verification.spec.noderepo; - -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.yahoo.vespa.hosted.node.verification.spec.retrievers.HardwareInfo; - -/** - * Created by olaa on 05/07/2017. - * Object with the information node repositories has about the node. - */ - -@JsonIgnoreProperties(ignoreUnknown = true) -public class NodeJsonModel { - - @JsonProperty("minDiskAvailableGb") - private double minDiskAvailableGb; - @JsonProperty("minMainMemoryAvailableGb") - private double minMainMemoryAvailableGb; - @JsonProperty("minCpuCores") - private double minCpuCores; - @JsonProperty("fastDisk") - private boolean fastDisk; - @JsonProperty("ipAddresses") - private String[] ipAddresses; - @JsonProperty("additionalIpAddresses") - private String[] additionalIpAddresses; - - public String[] getAdditionalIpAddresses() { - return additionalIpAddresses; - } - - public HardwareInfo copyToHardwareInfo() { - HardwareInfo hardwareInfo = new HardwareInfo(); - hardwareInfo.setMinMainMemoryAvailableGb(this.minMainMemoryAvailableGb); - hardwareInfo.setMinDiskAvailableGb(this.minDiskAvailableGb); - hardwareInfo.setMinCpuCores((int) Math.round(this.minCpuCores)); - hardwareInfo.setFastDisk(this.fastDisk); - return hardwareInfo; - } - - public String getIpv6Address() { - String ipv6Regex = "^((?:[0-9A-Fa-f]{1,4}))((?::[0-9A-Fa-f]{1,4}))*::((?:[0-9A-Fa-f]{1,4}))((?::[0-9A-Fa-f]{1,4}))*|((?:[0-9A-Fa-f]{1,4}))((?::[0-9A-Fa-f]{1,4})){7}$"; - for (String ipAddress : ipAddresses) { - if (ipAddress.matches(ipv6Regex)) { - return ipAddress; - } - } - return null; - } - -} diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/noderepo/NodeRepoInfoRetriever.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/noderepo/NodeRepoInfoRetriever.java new file mode 100644 index 00000000000..f3bc70310ce --- /dev/null +++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/noderepo/NodeRepoInfoRetriever.java @@ -0,0 +1,35 @@ +package com.yahoo.vespa.hosted.node.verification.spec.noderepo; + +import com.fasterxml.jackson.databind.ObjectMapper; + +import java.io.IOException; +import java.net.URL; +import java.util.ArrayList; +import java.util.logging.Level; +import java.util.logging.Logger; + +/** + * Created by olaa on 04/07/2017. + * Parse JSON from node repository and stores information as a NodeRepoJsonModel object. + */ +public class NodeRepoInfoRetriever { + + private static final Logger logger = Logger.getLogger(NodeRepoInfoRetriever.class.getName()); + + public static NodeRepoJsonModel retrieve(ArrayList nodeInfoUrls) throws IOException { + NodeRepoJsonModel nodeRepoJsonModel; + ObjectMapper objectMapper = new ObjectMapper(); + for (URL nodeInfoURL : nodeInfoUrls) { + try { + nodeRepoJsonModel = objectMapper.readValue(nodeInfoURL, NodeRepoJsonModel.class); + return nodeRepoJsonModel; + + } catch (IOException e) { + logger.log(Level.WARNING, "Failed to parse JSON from config server: " + nodeInfoURL.toString(), e); + } + } + throw new IOException("Failed to parse JSON from all possible config servers."); + } + +} + diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/noderepo/NodeRepoJsonModel.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/noderepo/NodeRepoJsonModel.java new file mode 100644 index 00000000000..f946a7343dc --- /dev/null +++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/noderepo/NodeRepoJsonModel.java @@ -0,0 +1,71 @@ +package com.yahoo.vespa.hosted.node.verification.spec.noderepo; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.yahoo.vespa.hosted.node.verification.spec.retrievers.HardwareInfo; +import com.yahoo.vespa.hosted.node.verification.spec.retrievers.HardwareInfo.DiskType; + +/** + * Created by olaa on 05/07/2017. + * Object with the information node repositories has about the node. + */ + +@JsonIgnoreProperties(ignoreUnknown = true) +public class NodeRepoJsonModel { + @JsonProperty("minDiskAvailableGb") + private double minDiskAvailableGb; + @JsonProperty("minMainMemoryAvailableGb") + private double minMainMemoryAvailableGb; + @JsonProperty("minCpuCores") + private double minCpuCores; + @JsonProperty("fastDisk") + private boolean fastDisk; + @JsonProperty("ipAddresses") + private String[] ipAddresses; + @JsonProperty("additionalIpAddresses") + private String[] additionalIpAddresses; + + public String[] getAdditionalIpAddresses() { + return additionalIpAddresses; + } + + public HardwareInfo copyToHardwareInfo() { + HardwareInfo hardwareInfo = new HardwareInfo(); + hardwareInfo.setMinMainMemoryAvailableGb(this.minMainMemoryAvailableGb); + hardwareInfo.setMinDiskAvailableGb(this.minDiskAvailableGb); + hardwareInfo.setMinCpuCores((int) Math.round(this.minCpuCores)); + hardwareInfo.setDiskType(this.fastDisk ? DiskType.FAST : DiskType.SLOW); + return hardwareInfo; + } + + public String getIpv6Address() { + String ipv6Regex = "^((?:[0-9A-Fa-f]{1,4}))((?::[0-9A-Fa-f]{1,4}))*::((?:[0-9A-Fa-f]{1,4}))((?::[0-9A-Fa-f]{1,4}))*|((?:[0-9A-Fa-f]{1,4}))((?::[0-9A-Fa-f]{1,4})){7}$"; + for (String ipAddress : ipAddresses) { + if (ipAddress.matches(ipv6Regex)) { + return ipAddress; + } + } + return null; + } + + public double getMinDiskAvailableGb() { + return minDiskAvailableGb; + } + + public double getMinMainMemoryAvailableGb() { + return minMainMemoryAvailableGb; + } + + public double getMinCpuCores() { + return minCpuCores; + } + + public boolean isFastDisk() { + return fastDisk; + } + + public String[] getIpAddresses() { + return ipAddresses; + } + +} diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/DiskRetriever.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/DiskRetriever.java index f8cb169ef5e..80c076efd5f 100644 --- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/DiskRetriever.java +++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/DiskRetriever.java @@ -4,6 +4,7 @@ import com.yahoo.vespa.hosted.node.verification.commons.CommandExecutor; import com.yahoo.vespa.hosted.node.verification.commons.OutputParser; import com.yahoo.vespa.hosted.node.verification.commons.ParseInstructions; import com.yahoo.vespa.hosted.node.verification.commons.ParseResult; +import com.yahoo.vespa.hosted.node.verification.spec.retrievers.HardwareInfo.DiskType; import java.io.IOException; import java.util.ArrayList; @@ -63,18 +64,17 @@ public class DiskRetriever implements HardwareRetriever { } protected void setDiskType(ParseResult parseResult) { + hardwareInfo.setDiskType(DiskType.UNKNOWN); if (!parseResult.getSearchWord().equals(DISK_NAME)) { return; } - String fastDiskEnum = "0"; - String nonFastDiskEnum = "1"; - Boolean fastdisk = null; - if (parseResult.getValue().equals(fastDiskEnum)) { - fastdisk = true; - } else if (parseResult.getValue().equals(nonFastDiskEnum)) { - fastdisk = false; + String fastDiskSymbol = "0"; + String nonFastDiskSymbol = "1"; + if (parseResult.getValue().equals(fastDiskSymbol)) { + hardwareInfo.setDiskType(DiskType.FAST); + } else if (parseResult.getValue().equals(nonFastDiskSymbol)) { + hardwareInfo.setDiskType(DiskType.SLOW); } - hardwareInfo.setFastDisk(fastdisk); } protected void setDiskSize(ParseResult parseResult) { diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/HardwareInfo.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/HardwareInfo.java index e369272b01f..5f534ad63af 100644 --- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/HardwareInfo.java +++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/HardwareInfo.java @@ -10,10 +10,11 @@ public class HardwareInfo { private double minDiskAvailableGb; private double minMainMemoryAvailableGb; private int minCpuCores; - private boolean fastDisk; - private boolean ipv4Connectivity; - private boolean ipv6Connectivity; + private boolean ipv4Interface; + private boolean ipv6Interface; + private boolean ipv6Connection; private double interfaceSpeedMbs; + private DiskType diskType; public double getInterfaceSpeedMbs() { @@ -32,20 +33,28 @@ public class HardwareInfo { this.minDiskAvailableGb = minDiskAvailableGb; } - public boolean getIpv6Connectivity() { - return ipv6Connectivity; + public boolean getIpv6Interface() { + return ipv6Interface; } - public void setIpv6Connectivity(boolean ipv6Connectivity) { - this.ipv6Connectivity = ipv6Connectivity; + public void setIpv6Interface(boolean ipv6Interface) { + this.ipv6Interface = ipv6Interface; } - public boolean getIpv4Connectivity() { - return ipv4Connectivity; + public boolean getIpv4Interface() { + return ipv4Interface; } - public void setIpv4Connectivity(boolean ipv4Connectivity) { - this.ipv4Connectivity = ipv4Connectivity; + public void setIpv4Interface(boolean ipv4Interface) { + this.ipv4Interface = ipv4Interface; + } + + public boolean isIpv6Connection() { + return ipv6Connection; + } + + public void setIpv6Connection(boolean ipv6Connection) { + this.ipv6Connection = ipv6Connection; } public double getMinMainMemoryAvailableGb() { @@ -56,12 +65,12 @@ public class HardwareInfo { this.minMainMemoryAvailableGb = minMainMemoryAvailableGb; } - public void setFastDisk(boolean fastDisk) { - this.fastDisk = fastDisk; + public void setDiskType(DiskType diskType) { + this.diskType = diskType; } - public boolean getFastDisk() { - return fastDisk; + public DiskType getDiskType() { + return diskType; } public int getMinCpuCores() { @@ -72,4 +81,8 @@ public class HardwareInfo { this.minCpuCores = minCpuCores; } + public enum DiskType {SLOW, FAST, UNKNOWN} + + ; + } diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/HardwareInfoRetriever.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/HardwareInfoRetriever.java index 0ea0d91ac19..77b514a56b3 100644 --- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/HardwareInfoRetriever.java +++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/HardwareInfoRetriever.java @@ -10,9 +10,8 @@ import java.util.ArrayList; */ public class HardwareInfoRetriever { - public static HardwareInfo retrieve() { + public static HardwareInfo retrieve(CommandExecutor commandExecutor) { HardwareInfo hardwareInfo = new HardwareInfo(); - CommandExecutor commandExecutor = new CommandExecutor(); ArrayList infoList = new ArrayList<>(); infoList.add(new CPURetriever(hardwareInfo, commandExecutor)); infoList.add(new MemoryRetriever(hardwareInfo, commandExecutor)); diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/NetRetriever.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/NetRetriever.java index 0d793bf416e..21c269119cb 100644 --- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/NetRetriever.java +++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/NetRetriever.java @@ -29,6 +29,11 @@ public class NetRetriever implements HardwareRetriever { private static final String INTERFACE_SPEED_REGEX_SPLIT = ":"; private static final int INTERFACE_SPEED_SEARCH_ELEMENT_INDEX = 0; private static final int INTERFACE_SPEED_RETURN_ELEMENT_INDEX = 1; + private static final String PING_NET_COMMAND = "ping6 -c 1 www.yahoo.com | grep transmitted"; + private static final String PING_SEARCH_WORD = "loss,"; + private static final String PING_SPLIT_REGEX_STRING = "\\s+"; + private static final int PING_SEARCH_ELEMENT_INDEX = 7; + private static final int PING_RETURN_ELEMENT_INDEX = 5; private static final Logger logger = Logger.getLogger(NetRetriever.class.getName()); private final HardwareInfo hardwareInfo; private final CommandExecutor commandExecutor; @@ -43,6 +48,7 @@ public class NetRetriever implements HardwareRetriever { try { ArrayList parseResults = findInterface(); findInterfaceSpeed(parseResults); + testPingResponse(parseResults); updateHardwareInfoWithNet(parseResults); } catch (IOException e) { logger.log(Level.WARNING, "Failed to retrieve net info", e); @@ -62,6 +68,11 @@ public class NetRetriever implements HardwareRetriever { parseResults.add(parseInterfaceSpeed(commandOutput)); } + protected void testPingResponse(ArrayList parseResults) throws IOException { + ArrayList commandOutput = commandExecutor.executeCommand(PING_NET_COMMAND); + parseResults.add(parsePingResponse(commandOutput)); + } + protected ArrayList parseNetInterface(ArrayList commandOutput) { ArrayList searchWords = new ArrayList<>(Arrays.asList(SEARCH_WORD_INTERFACE_IP4, SEARCH_WORD_INTERFACE_IPV6, SEARCH_WORD_INTERFACE_NAME)); ParseInstructions parseInstructions = new ParseInstructions(INTERFACE_NAME_SEARCH_ELEMENT_INDEX, INTERFACE_NAME_RETURN_ELEMENT_INDEX, INTERFACE_NAME_REGEX_SPLIT, searchWords); @@ -78,6 +89,14 @@ public class NetRetriever implements HardwareRetriever { return parseResult; } + protected ParseResult parsePingResponse(ArrayList commandOutput) { + ArrayList searchWords = new ArrayList<>(Arrays.asList(PING_SEARCH_WORD)); + ParseInstructions parseInstructions = new ParseInstructions(PING_SEARCH_ELEMENT_INDEX, PING_RETURN_ELEMENT_INDEX, PING_SPLIT_REGEX_STRING, searchWords); + ParseResult parseResult = OutputParser.parseSingleOutput(parseInstructions, commandOutput); + parseResult = parseResult.getSearchWord().equals(PING_SEARCH_WORD) ? parseResult : new ParseResult(PING_SEARCH_WORD, "invalid"); + return parseResult; + } + protected String findInterfaceName(ArrayList parseResults) { for (ParseResult parseResult : parseResults) { if (!parseResult.getSearchWord().matches(SEARCH_WORD_INTERFACE_NAME)) continue; @@ -86,27 +105,43 @@ public class NetRetriever implements HardwareRetriever { return ""; } + protected double convertInterfaceSpeed(String speed) { + return Double.parseDouble(speed.replaceAll("[^\\d.]", "")); + } + protected void updateHardwareInfoWithNet(ArrayList parseResults) { - hardwareInfo.setIpv6Connectivity(false); - hardwareInfo.setIpv4Connectivity(false); + hardwareInfo.setIpv6Interface(false); + hardwareInfo.setIpv4Interface(false); for (ParseResult parseResult : parseResults) { switch (parseResult.getSearchWord()) { case SEARCH_WORD_INTERFACE_IP4: - hardwareInfo.setIpv4Connectivity(true); + hardwareInfo.setIpv4Interface(true); break; case SEARCH_WORD_INTERFACE_IPV6: - hardwareInfo.setIpv6Connectivity(true); + hardwareInfo.setIpv6Interface(true); break; case SEARCH_WORD_INTERFACE_SPEED: - String speedValue = parseResult.getValue().replaceAll("[^\\d.]", ""); - double speed = Double.parseDouble(speedValue); + double speed = convertInterfaceSpeed(parseResult.getValue()); hardwareInfo.setInterfaceSpeedMbs(speed); break; + case PING_SEARCH_WORD: + setIpv6Connectivity(parseResult); + break; default: if (parseResult.getSearchWord().matches(SEARCH_WORD_INTERFACE_NAME)) break; - throw new RuntimeException("Invalid ParseResult search word"); + throw new RuntimeException("Invalid ParseResult search word " + parseResult.getSearchWord()); } } } + protected void setIpv6Connectivity(ParseResult parseResult) { + if (parseResult.getSearchWord().equals(PING_SEARCH_WORD)) { + String pingResponse = parseResult.getValue(); + String packetLoss = pingResponse.replaceAll("[^\\d.]", ""); + if (packetLoss.equals("")) return; + if (Double.parseDouble(packetLoss) > 99) return; + hardwareInfo.setIpv6Connection(true); + } + } + } diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/yamasreport/SpecReportDimensions.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/yamasreport/SpecReportDimensions.java index abf2f93023f..2a1ee4e9a53 100644 --- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/yamasreport/SpecReportDimensions.java +++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/yamasreport/SpecReportDimensions.java @@ -13,7 +13,7 @@ public class SpecReportDimensions { @JsonProperty private boolean cpuCoresMatch; @JsonProperty - private boolean fastDiskMatch; + private boolean diskTypeMatch; @JsonProperty private boolean netInterfaceSpeedMatch; @JsonProperty @@ -47,12 +47,12 @@ public class SpecReportDimensions { this.cpuCoresMatch = cpuCoresMatch; } - public boolean isFastDiskMatch() { - return fastDiskMatch; + public boolean isDiskTypeMatch() { + return diskTypeMatch; } - public void setFastDiskMatch(boolean fastDiskMatch) { - this.fastDiskMatch = fastDiskMatch; + public void setDiskTypeMatch(boolean diskTypeMatch) { + this.diskTypeMatch = diskTypeMatch; } public boolean isDiskAvailableMatch() { diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/yamasreport/SpecReportMetrics.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/yamasreport/SpecReportMetrics.java index 44332dd6aad..d835a1b29d3 100644 --- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/yamasreport/SpecReportMetrics.java +++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/yamasreport/SpecReportMetrics.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.hosted.node.verification.spec.yamasreport; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; +import com.yahoo.vespa.hosted.node.verification.spec.retrievers.HardwareInfo.DiskType; /** * Created by olaa on 12/07/2017. @@ -16,9 +17,9 @@ public class SpecReportMetrics { @JsonProperty private Double actualMemoryAvailable; @JsonProperty - private Boolean expectedFastDisk; + private DiskType expectedDiskType; @JsonProperty - private Boolean actualFastDisk; + private DiskType actualDiskType; @JsonProperty private Double expectedDiskSpaceAvailable; @JsonProperty @@ -33,6 +34,13 @@ public class SpecReportMetrics { private Integer actualcpuCores; @JsonProperty private String[] faultyIpAddresses; + @JsonProperty + private Boolean actualIpv6Connection; + + public void setActualIpv6Connection(boolean actualIpv6Connection) { + this.actualIpv6Connection = actualIpv6Connection; + } + public void setMatch(boolean match) { this.match = match; @@ -50,12 +58,12 @@ public class SpecReportMetrics { this.actualMemoryAvailable = actualMemoryAvailable; } - public void setExpectedDiskType(Boolean expectedFastDisk) { - this.expectedFastDisk = expectedFastDisk; + public void setExpectedDiskType(DiskType expectedFastDisk) { + this.expectedDiskType = expectedFastDisk; } - public void setActualDiskType(Boolean actualFastDisk) { - this.actualFastDisk = actualFastDisk; + public void setActualDiskType(DiskType actualFastDisk) { + this.actualDiskType = actualFastDisk; } public void setExpectedDiskSpaceAvailable(Double expectedDiskSpaceAvailable) { diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/HardwareBenchmarkerTest.java b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/HardwareBenchmarkerTest.java new file mode 100644 index 00000000000..c976aff54c5 --- /dev/null +++ b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/HardwareBenchmarkerTest.java @@ -0,0 +1,38 @@ +package com.yahoo.vespa.hosted.node.verification.hardware; + +import com.yahoo.vespa.hosted.node.verification.mock.MockCommandExecutor; +import org.junit.Before; +import org.junit.Test; + +import static org.junit.Assert.assertTrue; + +public class HardwareBenchmarkerTest { + + private MockCommandExecutor mockCommandExecutor; + private static final String RESOURCE_PATH = "src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/resources/"; + private static final String VALID_DISK_BENCHMARK_PATH = RESOURCE_PATH + "diskBenchmarkValidOutput"; + private static final String VALID_CPU_BENCHMARK_PATH = RESOURCE_PATH + "cpuCyclesWithCommasTimeWithDotTest.txt"; + private static final String VALID_MEMORY_WRITE_BENCHMARK_PATH = RESOURCE_PATH + "validMemoryWriteSpeed"; + private static final String VALID_MEMORY_READ_BENCHMARK_PATH = RESOURCE_PATH + "validMemoryReadSpeed"; + + @Before + public void setup() { + mockCommandExecutor = new MockCommandExecutor(); + + } + + @Test + public void hardwareBenchmarks_should_return_true() { + mockCommandExecutor.addCommand("cat " + VALID_DISK_BENCHMARK_PATH); + mockCommandExecutor.addCommand("cat " + VALID_CPU_BENCHMARK_PATH); + mockCommandExecutor.addDummyCommand(); + mockCommandExecutor.addDummyCommand(); + mockCommandExecutor.addCommand("cat " + VALID_MEMORY_WRITE_BENCHMARK_PATH); + mockCommandExecutor.addCommand("cat " + VALID_MEMORY_READ_BENCHMARK_PATH); + mockCommandExecutor.addDummyCommand(); + mockCommandExecutor.addDummyCommand(); + assertTrue(HardwareBenchmarker.hardwareBenchmarks(mockCommandExecutor)); + } + + +} \ No newline at end of file diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/TerminationControllerTest.java b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/TerminationControllerTest.java new file mode 100644 index 00000000000..82bffd96b1e --- /dev/null +++ b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/TerminationControllerTest.java @@ -0,0 +1,61 @@ +package com.yahoo.vespa.hosted.node.verification.hardware; + +import com.yahoo.vespa.hosted.node.verification.hardware.benchmarks.BenchmarkResults; +import org.junit.Before; +import org.junit.Test; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + + +public class TerminationControllerTest { + + private BenchmarkResults benchmarkResults; + private static final double VALID_CPU_FREQUENCY = 2.031; + private static final double INVALID_CPU_FREQUENCY = 0.1; + private static final double VALID_DISK_SPEED = 1100.0; + private static final double INVALID_DISK_SPEED = 0.1; + private static final double VALID_MEMORY_WRITE_SPEED = 1.7; + private static final double INVALID_MEMORY_WRITE_SPEED = 0.1; + private static final double VALID_MEMORY_READ_SPEED = 4.3; + private static final double INVALID_MEMORY_READ_SPEED = 0.1; + + @Before + public void setup() { + benchmarkResults = new BenchmarkResults(); + benchmarkResults.setCpuCyclesPerSec(VALID_CPU_FREQUENCY); + benchmarkResults.setDiskSpeedMbs(VALID_DISK_SPEED); + benchmarkResults.setMemoryWriteSpeedGBs(VALID_MEMORY_WRITE_SPEED); + benchmarkResults.setMemoryReadSpeedGBs(VALID_MEMORY_READ_SPEED); + } + + @Test + public void isBenchmarkResultsValid_should_return_true() { + assertTrue(TerminationController.isBenchmarkResultsValid(benchmarkResults)); + } + + @Test + public void isBenchmarkResultsValid_should_be_false_because_of_cpu_frequency() { + benchmarkResults.setCpuCyclesPerSec(INVALID_CPU_FREQUENCY); + assertFalse(TerminationController.isBenchmarkResultsValid(benchmarkResults)); + } + + @Test + public void isBenchmarkResultsValid_should_be_false_because_of_disk_speed() { + benchmarkResults.setDiskSpeedMbs(INVALID_DISK_SPEED); + assertFalse(TerminationController.isBenchmarkResultsValid(benchmarkResults)); + } + + @Test + public void isBenchmarkResultsValid_should_be_false_because_of_memory_write_speed() { + benchmarkResults.setMemoryWriteSpeedGBs(INVALID_MEMORY_WRITE_SPEED); + assertFalse(TerminationController.isBenchmarkResultsValid(benchmarkResults)); + } + + @Test + public void isBenchmarkResultsValid_should_be_false_because_of_memory_read_speed() { + benchmarkResults.setMemoryReadSpeedGBs(INVALID_MEMORY_READ_SPEED); + assertFalse(TerminationController.isBenchmarkResultsValid(benchmarkResults)); + } + +} \ No newline at end of file diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/CPUBenchmarkTest.java b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/CPUBenchmarkTest.java index f1d1703f1d9..524dbeab646 100644 --- a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/CPUBenchmarkTest.java +++ b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/CPUBenchmarkTest.java @@ -9,9 +9,9 @@ import org.junit.Test; import java.io.IOException; import java.util.ArrayList; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; -import static org.junit.Assert.assertEquals; /** * Created by sgrostad on 11/07/2017. @@ -21,7 +21,7 @@ public class CPUBenchmarkTest { private static final String cpuEuropeanDelimiters = "src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/resources/cpuCyclesWithDotsTimeWithCommaTest.txt"; private static final String cpuAlternativeDelimiters = "src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/resources/cpuCyclesWithCommasTimeWithDotTest.txt"; private static final String cpuWrongOutput = "src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/resources/cpuWrongOutputTest.txt"; - private HardwareResults hardwareResults; + private BenchmarkResults benchmarkResults; private MockCommandExecutor commandExecutor; private CPUBenchmark cpu; private static final double DELTA = 0.1; @@ -29,8 +29,8 @@ public class CPUBenchmarkTest { @Before public void setup() { commandExecutor = new MockCommandExecutor(); - hardwareResults = new HardwareResults(); - cpu = new CPUBenchmark(hardwareResults, commandExecutor); + benchmarkResults = new BenchmarkResults(); + cpu = new CPUBenchmark(benchmarkResults, commandExecutor); } @Test @@ -38,7 +38,7 @@ public class CPUBenchmarkTest { String mockCommand = "cat " + cpuAlternativeDelimiters; commandExecutor.addCommand(mockCommand); cpu.doBenchmark(); - double result = hardwareResults.getCpuCyclesPerSec(); + double result = benchmarkResults.getCpuCyclesPerSec(); double expected = 2.1576482291815062; assertEquals(expected, result, DELTA); } @@ -48,7 +48,7 @@ public class CPUBenchmarkTest { String mockCommand = "cat " + cpuWrongOutput; commandExecutor.addCommand(mockCommand); cpu.doBenchmark(); - double result = hardwareResults.getCpuCyclesPerSec(); + double result = benchmarkResults.getCpuCyclesPerSec(); double expected = 0; assertEquals(expected, result, DELTA); } @@ -70,7 +70,7 @@ public class CPUBenchmarkTest { parseResults.add(new ParseResult("seconds", "0,957617512")); cpu.setCpuCyclesPerSec(parseResults); double expectedCpuCyclesPerSec = 2.1576482291815062; - assertEquals(expectedCpuCyclesPerSec, hardwareResults.getCpuCyclesPerSec(), DELTA); + assertEquals(expectedCpuCyclesPerSec, benchmarkResults.getCpuCyclesPerSec(), DELTA); } @Test diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/DiskBenchmarkTest.java b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/DiskBenchmarkTest.java index 2a8b0994feb..3fb4015a525 100644 --- a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/DiskBenchmarkTest.java +++ b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/DiskBenchmarkTest.java @@ -7,9 +7,9 @@ import org.junit.Test; import java.util.ArrayList; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; -import static org.junit.Assert.assertEquals; /** * Created by sgrostad on 12/07/2017. @@ -17,7 +17,7 @@ import static org.junit.Assert.assertEquals; public class DiskBenchmarkTest { private DiskBenchmark diskBenchmark; - private HardwareResults hardwareResults; + private BenchmarkResults benchmarkResults; private MockCommandExecutor commandExecutor; private static final String VALID_OUTPUT_FILE = "src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/resources/diskBenchmarkValidOutput"; private static final String INVALID_OUTPUT_FILE = "src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/resources/diskBenchmarkInvalidOutput"; @@ -26,8 +26,8 @@ public class DiskBenchmarkTest { @Before public void setup() { commandExecutor = new MockCommandExecutor(); - hardwareResults = new HardwareResults(); - diskBenchmark = new DiskBenchmark(hardwareResults, commandExecutor); + benchmarkResults = new BenchmarkResults(); + diskBenchmark = new DiskBenchmark(benchmarkResults, commandExecutor); } @Test @@ -36,7 +36,7 @@ public class DiskBenchmarkTest { commandExecutor.addCommand(mockCommand); diskBenchmark.doBenchmark(); double expectedSpeed = 243; - double actualSpeed = hardwareResults.getDiskSpeedMbs(); + double actualSpeed = benchmarkResults.getDiskSpeedMbs(); assertEquals(expectedSpeed, actualSpeed, DELTA); } @@ -46,7 +46,7 @@ public class DiskBenchmarkTest { commandExecutor.addCommand(mockCommand); diskBenchmark.doBenchmark(); double expectedSpeed = 0; - double actualSpeed = hardwareResults.getDiskSpeedMbs(); + double actualSpeed = benchmarkResults.getDiskSpeedMbs(); assertEquals(expectedSpeed, actualSpeed, DELTA); } @@ -72,7 +72,7 @@ public class DiskBenchmarkTest { ParseResult parseResult = new ParseResult("MB/s", "243"); diskBenchmark.setDiskSpeed(parseResult); double expectedDiskSpeed = 243; - assertEquals(expectedDiskSpeed, hardwareResults.getDiskSpeedMbs(), DELTA); + assertEquals(expectedDiskSpeed, benchmarkResults.getDiskSpeedMbs(), DELTA); } @Test @@ -80,7 +80,7 @@ public class DiskBenchmarkTest { ParseResult parseResult = new ParseResult("invalid", "invalid"); diskBenchmark.setDiskSpeed(parseResult); double expectedDiskSpeed = 0; - assertEquals(expectedDiskSpeed, hardwareResults.getDiskSpeedMbs(), DELTA); + assertEquals(expectedDiskSpeed, benchmarkResults.getDiskSpeedMbs(), DELTA); } @Test diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/MemoryBenchmarkTest.java b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/MemoryBenchmarkTest.java index 0f6d515a645..056d26ab2af 100644 --- a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/MemoryBenchmarkTest.java +++ b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/MemoryBenchmarkTest.java @@ -7,10 +7,9 @@ import org.junit.Test; import java.util.ArrayList; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNull; /** * Created by olaa on 14/07/2017. @@ -18,15 +17,15 @@ import static org.junit.Assert.assertNull; public class MemoryBenchmarkTest { private MemoryBenchmark memoryBenchmark; - private HardwareResults hardwareResults; + private BenchmarkResults benchmarkResults; private MockCommandExecutor commandExecutor; private static final double DELTA = 0.1; @Before public void setup() { - hardwareResults = new HardwareResults(); + benchmarkResults = new BenchmarkResults(); commandExecutor = new MockCommandExecutor(); - memoryBenchmark = new MemoryBenchmark(hardwareResults, commandExecutor); + memoryBenchmark = new MemoryBenchmark(benchmarkResults, commandExecutor); } @Test @@ -40,8 +39,8 @@ public class MemoryBenchmarkTest { memoryBenchmark.doBenchmark(); double expectedReadSpeed = 5.9; double expectedWriteSpeed = 3.4; - assertEquals(expectedReadSpeed, hardwareResults.getMemoryReadSpeedGBs(), DELTA); - assertEquals(expectedWriteSpeed, hardwareResults.getMemoryWriteSpeedGBs(), DELTA); + assertEquals(expectedReadSpeed, benchmarkResults.getMemoryReadSpeedGBs(), DELTA); + assertEquals(expectedWriteSpeed, benchmarkResults.getMemoryWriteSpeedGBs(), DELTA); } @Test @@ -69,26 +68,26 @@ public class MemoryBenchmarkTest { public void memoryReadSpeed_valid_input_should_update_hardwareResults() { Double expectedMemoryReadSpeed = 12.1; memoryBenchmark.updateMemoryReadSpeed(expectedMemoryReadSpeed.toString()); - assertEquals(expectedMemoryReadSpeed, hardwareResults.getMemoryReadSpeedGBs(), DELTA); + assertEquals(expectedMemoryReadSpeed, benchmarkResults.getMemoryReadSpeedGBs(), DELTA); } @Test public void memoryReadSpeed_invalid_input_should_not_update_hardwareResults() { memoryBenchmark.updateMemoryReadSpeed("Invalid speed"); - assertNull(hardwareResults.getMemoryReadSpeedGBs()); + assertEquals(0D, benchmarkResults.getMemoryReadSpeedGBs(), DELTA); } @Test public void memoryWriteSpeed_valid_input_should_update_hardwareResults() { Double expectedMemoryWriteSpeed = 3.8; memoryBenchmark.updateMemoryWriteSpeed(expectedMemoryWriteSpeed.toString()); - assertEquals(expectedMemoryWriteSpeed, hardwareResults.getMemoryWriteSpeedGBs(), DELTA); + assertEquals(expectedMemoryWriteSpeed, benchmarkResults.getMemoryWriteSpeedGBs(), DELTA); } @Test public void memoryWriteSpeed_invalid_input_should_not_update_hardwareResults() { memoryBenchmark.updateMemoryWriteSpeed("Invalid speed"); - assertNull(hardwareResults.getMemoryWriteSpeedGBs()); + assertEquals(0D, benchmarkResults.getMemoryWriteSpeedGBs(), DELTA); } @Test diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/NetBenchmarkTest.java b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/NetBenchmarkTest.java deleted file mode 100644 index 5ed20d370b3..00000000000 --- a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/NetBenchmarkTest.java +++ /dev/null @@ -1,97 +0,0 @@ -package com.yahoo.vespa.hosted.node.verification.hardware.benchmarks; - -import com.yahoo.vespa.hosted.node.verification.commons.ParseResult; -import com.yahoo.vespa.hosted.node.verification.mock.MockCommandExecutor; -import org.junit.Before; -import org.junit.Test; - -import java.io.IOException; -import java.util.ArrayList; - -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.assertEquals; - -/** - * Created by olaa on 11/07/2017. - */ -public class NetBenchmarkTest { - - private HardwareResults hardwareResults; - private NetBenchmark netBenchmark; - private MockCommandExecutor commandExecutor; - private static String VALID_PING_RESPONSE = "src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/resources/validpingresponse"; - private static String INVALID_PING_RESPONSE = "src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/resources/invalidpingresponse"; - private static String CRAZY_PING_RESPONSE = "src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/resources/crazypingresponse"; - - @Before - public void setup() { - hardwareResults = new HardwareResults(); - commandExecutor = new MockCommandExecutor(); - netBenchmark = new NetBenchmark(hardwareResults, commandExecutor); - } - - @Test - public void doBenchmark_should_update_hardwareResults_to_true() { - String mockCommand = "cat " + VALID_PING_RESPONSE; - commandExecutor.addCommand(mockCommand); - netBenchmark.doBenchmark(); - assertTrue(hardwareResults.isIpv6Connectivity()); - } - - @Test - public void doBenchmark_should_update_hardwareResults_to_false_1() { - String mockCommand = "cat " + INVALID_PING_RESPONSE; - commandExecutor.addCommand(mockCommand); - netBenchmark.doBenchmark(); - assertFalse(hardwareResults.isIpv6Connectivity()); - } - - @Test - public void doBenchmark_should_update_hardwareResults_to_false_2() { - String mockCommand = "cat " + CRAZY_PING_RESPONSE; - commandExecutor.addCommand(mockCommand); - netBenchmark.doBenchmark(); - assertFalse(hardwareResults.isIpv6Connectivity()); - } - - @Test - public void parsePingResponse_valid_ping_response_should_return_ipv6_connectivity() throws IOException { - String command = "src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/resources/validpingresponse"; - ArrayList mockCommandOutput = MockCommandExecutor.readFromFile(command); - ParseResult parseResult = netBenchmark.parsePingResponse(mockCommandOutput); - String expectedPing = "0%"; - assertEquals(expectedPing, parseResult.getValue()); - } - - @Test - public void parsePingResponse_invalid_ping_response_should_return_invalid_ParseResult() throws IOException { - String command = "src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/resources/crazypingresponse"; - ArrayList mockCommandOutput = MockCommandExecutor.readFromFile(command); - ParseResult parseResult = netBenchmark.parsePingResponse(mockCommandOutput); - ParseResult expectedParseResult = new ParseResult("invalid", "invalid"); - assertEquals(expectedParseResult, parseResult); - } - - @Test - public void setIpv6Connectivity_valid_ping_response_should_return_ipv6_connectivity() { - ParseResult parseResult = new ParseResult("loss,", "0%"); - netBenchmark.setIpv6Connectivity(parseResult); - assertTrue(hardwareResults.isIpv6Connectivity()); - } - - @Test - public void setIpv6Connectivity_invalid_ping_response_should_return_no_ipv6_connectivity_1() { - ParseResult parseResult = new ParseResult("loss,", "100%"); - netBenchmark.setIpv6Connectivity(parseResult); - assertFalse(hardwareResults.isIpv6Connectivity()); - } - - @Test - public void setIpv6Connectivity_invalid_ping_response_should_return_no_ipv6_connectivity_2() { - ParseResult parseResult = new ParseResult("loss,", "invalid"); - netBenchmark.setIpv6Connectivity(parseResult); - assertFalse(hardwareResults.isIpv6Connectivity()); - } - -} \ No newline at end of file diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/resources/crazypingresponse b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/resources/crazypingresponse deleted file mode 100644 index d111ac34ce1..00000000000 --- a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/resources/crazypingresponse +++ /dev/null @@ -1 +0,0 @@ -this is no pingresponse ?!!? 34234 \ No newline at end of file diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/resources/invalidpingresponse b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/resources/invalidpingresponse deleted file mode 100644 index 0e6a67d3d89..00000000000 --- a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/resources/invalidpingresponse +++ /dev/null @@ -1 +0,0 @@ -6 packets transmitted, 6 received, 100% packet loss, time 5002ms diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/resources/validpingresponse b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/resources/validpingresponse deleted file mode 100644 index f6ef0559571..00000000000 --- a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/resources/validpingresponse +++ /dev/null @@ -1 +0,0 @@ -6 packets transmitted, 6 received, 0% packet loss, time 5002ms diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/yamasreport/YamasHardwareReportTest.java b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/yamasreport/YamasHardwareReportTest.java new file mode 100644 index 00000000000..9f3f88163b2 --- /dev/null +++ b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/yamasreport/YamasHardwareReportTest.java @@ -0,0 +1,31 @@ +package com.yahoo.vespa.hosted.node.verification.hardware.yamasreport; + +import com.yahoo.vespa.hosted.node.verification.hardware.benchmarks.BenchmarkResults; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + +public class YamasHardwareReportTest { + + private YamasHardwareReport yamasHardwareReport = new YamasHardwareReport(); + private static final double DELTA = 0.1; + + @Test + public void createFromHardwareResults_should_create_correct_report() { + double expectedCpuCyclesPerSec = 4; + double expectedDiskSpeedMbps = 120; + double expectedMemoryReadSpeedGBs = 7.1; + double expectedMemoryWriteSpeedGBs = 5.9; + BenchmarkResults benchmarkResults = new BenchmarkResults(); + benchmarkResults.setCpuCyclesPerSec(expectedCpuCyclesPerSec); + benchmarkResults.setDiskSpeedMbs(expectedDiskSpeedMbps); + benchmarkResults.setMemoryReadSpeedGBs(expectedMemoryReadSpeedGBs); + benchmarkResults.setMemoryWriteSpeedGBs(expectedMemoryWriteSpeedGBs); + yamasHardwareReport.createReportFromBenchmarkResults(benchmarkResults); + assertEquals(expectedCpuCyclesPerSec, yamasHardwareReport.getMetrics().getCpuCyclesPerSec(), DELTA); + assertEquals(expectedDiskSpeedMbps, yamasHardwareReport.getMetrics().getDiskSpeedMbs(), DELTA); + assertEquals(expectedMemoryReadSpeedGBs, yamasHardwareReport.getMetrics().getMemoryReadSpeedGBs(), DELTA); + assertEquals(expectedMemoryWriteSpeedGBs, yamasHardwareReport.getMetrics().getMemoryWriteSpeedGBs(), DELTA); + } + +} \ No newline at end of file diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/HardwareNodeComparatorTest.java b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/HardwareNodeComparatorTest.java index 0a06ac05c4b..ec621189269 100644 --- a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/HardwareNodeComparatorTest.java +++ b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/HardwareNodeComparatorTest.java @@ -1,6 +1,7 @@ package com.yahoo.vespa.hosted.node.verification.spec; import com.yahoo.vespa.hosted.node.verification.spec.retrievers.HardwareInfo; +import com.yahoo.vespa.hosted.node.verification.spec.retrievers.HardwareInfo.DiskType; import org.junit.Before; import org.junit.Test; @@ -42,4 +43,12 @@ public class HardwareNodeComparatorTest { assertFalse(HardwareNodeComparator.compare(nodeInfo, actualHardware).getMetrics().isMatch()); } + @Test + public void compare_different_disk_type_should_return_false() { + actualHardware.setDiskType(DiskType.UNKNOWN); + nodeInfo.setDiskType(DiskType.FAST); + assertFalse(HardwareNodeComparator.compare(nodeInfo, actualHardware).getMetrics().isMatch()); + } + + } \ No newline at end of file diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/HostURLGeneratorTest.java b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/HostURLGeneratorTest.java index 78d30dbc836..c992759b694 100644 --- a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/HostURLGeneratorTest.java +++ b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/HostURLGeneratorTest.java @@ -1,29 +1,75 @@ package com.yahoo.vespa.hosted.node.verification.spec; +import com.yahoo.vespa.hosted.node.verification.mock.MockCommandExecutor; +import org.junit.Before; import org.junit.Test; +import java.io.IOException; import java.net.URL; +import java.util.ArrayList; import static org.junit.Assert.assertEquals; -import static org.mockito.Mockito.spy; -import static org.mockito.Mockito.when; +import static org.junit.Assert.fail; /** * Created by olaa on 14/07/2017. */ public class HostURLGeneratorTest { + private MockCommandExecutor mockCommandExecutor; + private static final String CAT_NODE_HOST_NAME_PATH = "cat src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/hostURLGeneratorTest"; + private static final String CAT_CONFIG_SERVER_HOST_NAME_PATH = "cat src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/nodeHostNameOutput"; + private static final String CAT_WRONG_HOSTNAME_PATH = "cat src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/hostURLGeneratorExceptionTest"; + private static final String CONFIG_SERVER_HOSTNAME = "http://cfg1.prod.corp-us-east-1.vespahosted.corp.bf1.yahoo.com:4080"; + private static final String NODE_HOSTNAME_PREFIX = "/nodes/v2/node/"; + private static final String EXPECTED_HOSTNAME = "13305821.ostk.bm2.prod.gq1.yahoo.com"; + + @Before + public void setup() { + mockCommandExecutor = new MockCommandExecutor(); + } + @Test public void generateNodeInfoUrl_test_if_url_is_formatted_correctly() throws Exception { - String zoneHostName = "http://cfg1.prod.us-west-1.vespahosted.gq1.yahoo.com:4080"; - String midUrl = "/nodes/v2/node/"; - String nodeHostName = "13305821.ostk.bm2.prod.gq1.yahoo.com"; - HostURLGenerator hostURLGenerator = spy(new HostURLGenerator()); - when(hostURLGenerator.getEnvironmentVariable("HOSTNAME")).thenReturn(nodeHostName); - URL url = hostURLGenerator.generateNodeInfoUrl(zoneHostName); - String expectedUrl = zoneHostName + midUrl + nodeHostName; - String actualUrl = url.toString(); + mockCommandExecutor.addCommand(CAT_CONFIG_SERVER_HOST_NAME_PATH); + mockCommandExecutor.addCommand(CAT_NODE_HOST_NAME_PATH); + ArrayList url = HostURLGenerator.generateNodeInfoUrl(mockCommandExecutor); + String expectedUrl = CONFIG_SERVER_HOSTNAME + NODE_HOSTNAME_PREFIX + EXPECTED_HOSTNAME; + String actualUrl = url.get(0).toString(); assertEquals(expectedUrl, actualUrl); + + } + + @Test + public void generateNodeInfoURL_expected_IOException() { + try { + mockCommandExecutor.addCommand(CAT_CONFIG_SERVER_HOST_NAME_PATH); + mockCommandExecutor.addCommand(CAT_WRONG_HOSTNAME_PATH); + HostURLGenerator.generateNodeInfoUrl(mockCommandExecutor); + fail("Expected an IOExeption to be thrown"); + } catch (IOException e) { + String expectedExceptionMessage = "Unexpected output from \"hostname\" command."; + assertEquals(expectedExceptionMessage, e.getMessage()); + } + } + + @Test + public void buildNodeInfoURL_should_add_protocol_and_port_in_front_when_protocol_is_absent() throws IOException { + String configServerHostName = "www.yahoo.com"; + String nodeHostName = "index.html"; + String nodeHostnamePrefix = "/nodes/v2/node/"; + String portNumber = ":4080"; + String expectedUrl = "http://" + configServerHostName + portNumber + nodeHostnamePrefix + nodeHostName; + assertEquals(expectedUrl, HostURLGenerator.buildNodeInfoURL(configServerHostName, nodeHostName).toString()); + } + + @Test + public void buildNodeInfoURL_should_not_add_protocol_and_port_in_front_when_protocol_already_exists() throws IOException { + String configServerHostName = "http://www.yahoo.com"; + String nodeHostName = "index.html"; + String nodeHostnamePrefix = "/nodes/v2/node/"; + String expectedUrl = configServerHostName + nodeHostnamePrefix + nodeHostName; + assertEquals(expectedUrl, HostURLGenerator.buildNodeInfoURL(configServerHostName, nodeHostName).toString()); } } \ No newline at end of file diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/SpecVerifierTest.java b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/SpecVerifierTest.java new file mode 100644 index 00000000000..461169071a1 --- /dev/null +++ b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/SpecVerifierTest.java @@ -0,0 +1,109 @@ +package com.yahoo.vespa.hosted.node.verification.spec; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.yahoo.vespa.hosted.node.verification.mock.MockCommandExecutor; +import com.yahoo.vespa.hosted.node.verification.spec.noderepo.NodeRepoInfoRetriever; +import com.yahoo.vespa.hosted.node.verification.spec.noderepo.NodeRepoJsonModel; +import com.yahoo.vespa.hosted.node.verification.spec.retrievers.HardwareInfo; +import com.yahoo.vespa.hosted.node.verification.spec.yamasreport.YamasSpecReport; +import org.junit.Before; +import org.junit.Test; + +import java.io.File; +import java.net.URL; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Arrays; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +public class SpecVerifierTest { + + private MockCommandExecutor mockCommandExecutor; + private static final String ABSOLUTE_PATH = Paths.get(".").toAbsolutePath().normalize().toString(); + private static final String RESOURCE_PATH = "src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources"; + private static final String URL_RESOURCE_PATH = "file://" + ABSOLUTE_PATH + "/" + RESOURCE_PATH; + private static final String NODE_REPO_PATH = "src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/nodeInfoTest.json"; + private static final String CPU_INFO_PATH = RESOURCE_PATH + "/cpuinfoTest"; + private static final String MEMORY_INFO_PATH = RESOURCE_PATH + "/meminfoTest"; + private static final String DISK_TYPE_INFO_PATH = RESOURCE_PATH + "/DiskTypeFastDisk"; + private static final String DISK_SIZE_INFO_PATH = RESOURCE_PATH + "/filesize"; + private static final String NET_INTERFACE_INFO_PATH = RESOURCE_PATH + "/ifconfig"; + private static final String NET_INTERFACE_SPEED_INFO_PATH = RESOURCE_PATH + "/eth0"; + private static final String PING_RESPONSE = RESOURCE_PATH + "/validpingresponse"; + private static final double DELTA = 0.1; + + @Before + public void setup() { + mockCommandExecutor = new MockCommandExecutor(); + } + + + @Test + public void verifySpec_equal_nodeRepoInfo_and_hardware_should_return_true() throws Exception { + mockCommandExecutor.addCommand("echo notUsed " + URL_RESOURCE_PATH); + mockCommandExecutor.addCommand("echo nodeRepo.json"); + mockCommandExecutor.addCommand("cat " + CPU_INFO_PATH); + mockCommandExecutor.addCommand("cat " + MEMORY_INFO_PATH); + mockCommandExecutor.addCommand("cat " + DISK_TYPE_INFO_PATH); + mockCommandExecutor.addCommand("cat " + DISK_SIZE_INFO_PATH); + mockCommandExecutor.addCommand("cat " + NET_INTERFACE_INFO_PATH); + mockCommandExecutor.addCommand("cat " + NET_INTERFACE_SPEED_INFO_PATH); + mockCommandExecutor.addCommand("cat " + PING_RESPONSE); + assertTrue(SpecVerifier.verifySpec(mockCommandExecutor)); + } + + @Test + public void verifySpec_inequal_nodeRepoInfo_and_hardware_should_return_false() throws Exception { + mockCommandExecutor.addCommand("echo notUsed " + URL_RESOURCE_PATH); + mockCommandExecutor.addCommand("echo nodeRepo.json"); + mockCommandExecutor.addCommand("cat " + CPU_INFO_PATH); + mockCommandExecutor.addCommand("cat " + MEMORY_INFO_PATH); + mockCommandExecutor.addCommand("cat " + DISK_TYPE_INFO_PATH); + mockCommandExecutor.addCommand("cat " + DISK_SIZE_INFO_PATH); + mockCommandExecutor.addCommand("cat " + NET_INTERFACE_INFO_PATH + "NoIpv6"); + mockCommandExecutor.addCommand("cat " + NET_INTERFACE_SPEED_INFO_PATH); + mockCommandExecutor.addCommand("cat " + PING_RESPONSE); + assertFalse(SpecVerifier.verifySpec(mockCommandExecutor)); + } + + @Test + public void makeYamasSpecReport_should_return_false_interface_speed() throws Exception { + HardwareInfo actualHardware = new HardwareInfo(); + actualHardware.setMinCpuCores(24); + actualHardware.setMinMainMemoryAvailableGb(24); + actualHardware.setInterfaceSpeedMbs(10009); //this is wrong + actualHardware.setMinDiskAvailableGb(500); + actualHardware.setIpv4Interface(true); + actualHardware.setIpv6Interface(false); + actualHardware.setDiskType(HardwareInfo.DiskType.SLOW); + ArrayList url = new ArrayList<>(Arrays.asList(new File(NODE_REPO_PATH).toURI().toURL())); + NodeRepoJsonModel nodeRepoJsonModel = NodeRepoInfoRetriever.retrieve(url); + YamasSpecReport yamasSpecReport = SpecVerifier.makeYamasSpecReport(actualHardware, nodeRepoJsonModel); + long timeStamp = yamasSpecReport.getTimeStamp(); + String expectedJson = "{\"timeStamp\":" + timeStamp + ",\"dimensions\":{\"memoryMatch\":true,\"cpuCoresMatch\":true,\"diskTypeMatch\":true,\"netInterfaceSpeedMatch\":false,\"diskAvailableMatch\":true,\"ipv4Match\":true,\"ipv6Match\":true},\"metrics\":{\"match\":false,\"expectedInterfaceSpeed\":1000.0,\"actualInterfaceSpeed\":10009.0,\"actualIpv6Connection\":false},\"routing\":{\"yamas\":{\"namespace\":[\"Vespa\"]}}}"; + ObjectMapper om = new ObjectMapper(); + String actualJson = om.writeValueAsString(yamasSpecReport); + assertEquals(expectedJson, actualJson); + } + + @Test + public void getNodeRepositoryJSON_should_return_valid_nodeRepoJSONModel() throws Exception { + mockCommandExecutor.addCommand("echo notUsed " + URL_RESOURCE_PATH); + mockCommandExecutor.addCommand("echo nodeRepo.json"); + NodeRepoJsonModel actualNodeRepoJsonModel = SpecVerifier.getNodeRepositoryJSON(mockCommandExecutor); + double expectedMinCpuCores = 4D; + double expectedMinMainMemoryAvailableGb = 4.04D; + double expectedMinDiskAvailableGb = 63D; + boolean expectedFastDisk = true; + String expectedIpv6Address = "2001:4998:c:2940::111c"; + assertEquals(expectedIpv6Address, actualNodeRepoJsonModel.getIpv6Address()); + assertEquals(expectedMinCpuCores, actualNodeRepoJsonModel.getMinCpuCores(), DELTA); + assertEquals(expectedMinMainMemoryAvailableGb, actualNodeRepoJsonModel.getMinMainMemoryAvailableGb(), DELTA); + assertEquals(expectedMinDiskAvailableGb, actualNodeRepoJsonModel.getMinDiskAvailableGb(), DELTA); + assertEquals(expectedFastDisk, actualNodeRepoJsonModel.isFastDisk()); + } + +} \ No newline at end of file diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/noderepo/IPAddressVerifierTest.java b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/noderepo/IPAddressVerifierTest.java index 36e4c7af03d..49a0b15e8c3 100644 --- a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/noderepo/IPAddressVerifierTest.java +++ b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/noderepo/IPAddressVerifierTest.java @@ -47,4 +47,11 @@ public class IPAddressVerifierTest { assertEquals(expectedConvertedAddress, actualConvertedAddress); } + @Test + public void getFaultyIpAddresses_should_return_empty_array_when_parameters_are_invalid () { + assertEquals(0, ipAddressVerifier.getFaultyIpAddresses(null, null).length); + String invalidIpAddress = "This is an invalid IP address"; + assertEquals(0, ipAddressVerifier.getFaultyIpAddresses(invalidIpAddress, additionalIpAddresses).length); + } + } \ No newline at end of file diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/noderepo/NodeGeneratorTest.java b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/noderepo/NodeGeneratorTest.java deleted file mode 100644 index 9ccf48f8494..00000000000 --- a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/noderepo/NodeGeneratorTest.java +++ /dev/null @@ -1,38 +0,0 @@ -package com.yahoo.vespa.hosted.node.verification.spec.noderepo; - -import com.yahoo.vespa.hosted.node.verification.spec.retrievers.HardwareInfo; -import org.junit.Test; - -import java.io.File; -import java.net.URL; - -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.assertEquals; - -/** - * Created by olaa on 07/07/2017. - */ -public class NodeGeneratorTest { - - private static final double DELTA = 0.1; - - @Test - public void convertJsonModel_should_return_correct_HardwareInfo() throws Exception { - URL url = new File("src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/nodeInfoTest.json").toURI().toURL(); - NodeJsonModel nodeJsonModel = NodeInfoRetriever.retrieve(url); - HardwareInfo hardwareInfo = NodeGenerator.convertJsonModel(nodeJsonModel); - double expectedMinDiskAvailable = 500.0; - double expectedMinMainMemoryAvailable = 24.0; - double expectedMinCpuCores = 24.0; - double expectedInterfaceSpeedMbs = 1000; - assertEquals(expectedMinDiskAvailable, hardwareInfo.getMinDiskAvailableGb(), DELTA); - assertEquals(expectedMinMainMemoryAvailable, hardwareInfo.getMinMainMemoryAvailableGb(), DELTA); - assertEquals(expectedMinCpuCores, hardwareInfo.getMinCpuCores(), DELTA); - assertTrue(hardwareInfo.getIpv4Connectivity()); - assertTrue(hardwareInfo.getIpv6Connectivity()); - assertEquals(expectedInterfaceSpeedMbs, hardwareInfo.getInterfaceSpeedMbs(), DELTA); - assertFalse(hardwareInfo.getFastDisk()); - } - -} \ No newline at end of file diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/noderepo/NodeJsonConverterTest.java b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/noderepo/NodeJsonConverterTest.java new file mode 100644 index 00000000000..5d76f7fb012 --- /dev/null +++ b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/noderepo/NodeJsonConverterTest.java @@ -0,0 +1,40 @@ +package com.yahoo.vespa.hosted.node.verification.spec.noderepo; + +import com.yahoo.vespa.hosted.node.verification.spec.retrievers.HardwareInfo; +import org.junit.Test; + +import java.io.File; +import java.net.URL; +import java.util.ArrayList; +import java.util.Arrays; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +/** + * Created by olaa on 07/07/2017. + */ +public class NodeJsonConverterTest { + + private static final double DELTA = 0.1; + + @Test + public void convertJsonModel_should_return_correct_HardwareInfo() throws Exception { + ArrayList urls = new ArrayList<>(Arrays.asList(new File("src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/nodeInfoTest.json").toURI().toURL())); + NodeRepoJsonModel nodeRepoJsonModel = NodeRepoInfoRetriever.retrieve(urls); + HardwareInfo hardwareInfo = NodeJsonConverter.convertJsonModelToHardwareInfo(nodeRepoJsonModel); + double expectedMinDiskAvailable = 500.0; + double expectedMinMainMemoryAvailable = 24.0; + double expectedMinCpuCores = 24.0; + double expectedInterfaceSpeedMbs = 1000; + assertEquals(expectedMinDiskAvailable, hardwareInfo.getMinDiskAvailableGb(), DELTA); + assertEquals(expectedMinMainMemoryAvailable, hardwareInfo.getMinMainMemoryAvailableGb(), DELTA); + assertEquals(expectedMinCpuCores, hardwareInfo.getMinCpuCores(), DELTA); + assertTrue(hardwareInfo.getIpv4Interface()); + assertFalse(hardwareInfo.getIpv6Interface()); + assertEquals(expectedInterfaceSpeedMbs, hardwareInfo.getInterfaceSpeedMbs(), DELTA); + assertEquals(hardwareInfo.getDiskType(), HardwareInfo.DiskType.SLOW); + } + +} \ No newline at end of file diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/noderepo/NodeRepoInfoRetrieverTest.java b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/noderepo/NodeRepoInfoRetrieverTest.java new file mode 100644 index 00000000000..1eeb6423645 --- /dev/null +++ b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/noderepo/NodeRepoInfoRetrieverTest.java @@ -0,0 +1,60 @@ +package com.yahoo.vespa.hosted.node.verification.spec.noderepo; + +import org.junit.Before; +import org.junit.Test; + +import java.io.IOException; +import java.net.MalformedURLException; +import java.net.URL; +import java.nio.file.Paths; +import java.util.ArrayList; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +public class NodeRepoInfoRetrieverTest { + + private NodeRepoInfoRetriever nodeRepoInfoRetriever; + private ArrayList urls; + private static final double DELTA = 0.1; + private static final String ABSOLUTE_PATH = Paths.get(".").toAbsolutePath().normalize().toString(); + private static final String RESOURCE_PATH = "src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources"; + private static final String URL_RESOURCE_PATH = "file://" + ABSOLUTE_PATH + "/" + RESOURCE_PATH; + + @Before + public void setup() { + nodeRepoInfoRetriever = new NodeRepoInfoRetriever(); + urls = new ArrayList<>(); + } + + @Test + public void retrieve_should_return_nodeJSONModel_when_parameter_contains_valid_url () throws IOException{ + urls.add(new URL(URL_RESOURCE_PATH + "/nodeInfoTest.json")); + NodeRepoJsonModel nodeRepoJsonModel = NodeRepoInfoRetriever.retrieve(urls); + double expectedMinDiskAvailable = 500.0; + double expectedMinMainMemoryAvailable = 24.0; + double expectedMinCpuCores = 24.0; + assertEquals(expectedMinDiskAvailable, nodeRepoJsonModel.getMinDiskAvailableGb(), DELTA); + assertEquals(expectedMinMainMemoryAvailable, nodeRepoJsonModel.getMinMainMemoryAvailableGb(), DELTA); + assertEquals(expectedMinCpuCores, nodeRepoJsonModel.getMinCpuCores(), DELTA); + } + @Test + public void retrieve_should_throw_IOEqxception_when_no_valid_URLs() throws MalformedURLException { + urls = new ArrayList<>(); + String exceptionMessage = "Failed to parse JSON from all possible config servers."; + try { + NodeRepoInfoRetriever.retrieve(urls); + fail("Retrieve should have thrown IOException"); + } catch (IOException e) { + assertEquals(exceptionMessage, e.getMessage()); + } + urls.add(new URL("file:///dev/null")); + try { + NodeRepoInfoRetriever.retrieve(urls); + fail("Retrieve should have thrown IOException"); + } catch (IOException e) { + assertEquals(exceptionMessage, e.getMessage()); + } + } + +} \ No newline at end of file diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/DiskTypeFastDisk b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/DiskTypeFastDisk new file mode 100644 index 00000000000..3b0eed77569 --- /dev/null +++ b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/DiskTypeFastDisk @@ -0,0 +1,2 @@ +Name Rota +sda 0 \ No newline at end of file diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/DisktypeFastDisk b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/DisktypeFastDisk deleted file mode 100644 index 3b0eed77569..00000000000 --- a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/DisktypeFastDisk +++ /dev/null @@ -1,2 +0,0 @@ -Name Rota -sda 0 \ No newline at end of file diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/hostURLGeneratorExceptionTest b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/hostURLGeneratorExceptionTest new file mode 100644 index 00000000000..e48aae651f9 --- /dev/null +++ b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/hostURLGeneratorExceptionTest @@ -0,0 +1,3 @@ +Not +valid +hostname \ No newline at end of file diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/hostURLGeneratorTest b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/hostURLGeneratorTest new file mode 100644 index 00000000000..32258f52c05 --- /dev/null +++ b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/hostURLGeneratorTest @@ -0,0 +1 @@ +13305821.ostk.bm2.prod.gq1.yahoo.com \ No newline at end of file diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/invalidpingresponse b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/invalidpingresponse new file mode 100644 index 00000000000..d111ac34ce1 --- /dev/null +++ b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/invalidpingresponse @@ -0,0 +1 @@ +this is no pingresponse ?!!? 34234 \ No newline at end of file diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/nodeHostNameOutput b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/nodeHostNameOutput new file mode 100644 index 00000000000..a4edd13a851 --- /dev/null +++ b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/nodeHostNameOutput @@ -0,0 +1 @@ +services.addr_configserver: cfg1.prod.corp-us-east-1.vespahosted.corp.bf1.yahoo.com,cfg2.prod.corp-us-east-1.vespahosted.corp.bf1.yahoo.com,cfg3.prod.corp-us-east-1.vespahosted.corp.bf1.yahoo.com \ No newline at end of file diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/nodes/v2/node/nodeRepo.json b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/nodes/v2/node/nodeRepo.json new file mode 100644 index 00000000000..8631c2a51aa --- /dev/null +++ b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/nodes/v2/node/nodeRepo.json @@ -0,0 +1,81 @@ +{ + "url": "https://api.vespa.corp.yahoo.com:4443/zone/v2/prod/us-west-1/nodes/v2/node/zt74722.ostk.bm2.prod.gq1.yahoo.com", + "id": "zt74722.ostk.bm2.prod.gq1.yahoo.com", + "state": "active", + "type": "host", + "hostname": "zt74722.ostk.bm2.prod.gq1.yahoo.com", + "openStackId": "653c39c1-bda4-47ee-a277-d5319eb57af7", + "flavor": "C-77E/256/960", + "canonicalFlavor": "C-77E/256/960", + "minDiskAvailableGb": 63.0, + "minMainMemoryAvailableGb": 4.042128, + "description": "BARE_METAL with 48.0 CPUs, 256.0 Gb memory and 1920.0 Gb ssd", + "minCpuCores": 4.0, + "cost": 150, + "fastDisk": true, + "environment": "BARE_METAL", + "owner": { + "tenant": "hosted-vespa", + "application": "routing", + "instance": "default" + }, + "membership": { + "clustertype": "container", + "clusterid": "node-admin", + "group": "0", + "index": 77, + "retired": false + }, + "restartGeneration": 0, + "currentRestartGeneration": 0, + "wantedDockerImage": "docker-registry.ops.yahoo.com:4443/vespa/ci:6.128.19", + "wantedVespaVersion": "6.128.19", + "rebootGeneration": 3, + "currentRebootGeneration": 3, + "vespaVersion": "6.130.27", + "currentDockerImage": "docker-registry.ops.yahoo.com:4443/vespa/ci:6.130.27", + "hostedVersion": "6.130.27", + "convergedStateVersion": "6.130.27", + "failCount": 1, + "hardwareFailure": false, + "wantToRetire": false, + "wantToDeprovision": false, + "history": [ + { + "event": "failed", + "at": 1499293603487, + "agent": "system" + }, + { + "event": "rebooted", + "at": 1499351267086, + "agent": "system" + }, + { + "event": "readied", + "at": 1499351270674, + "agent": "system" + }, + { + "event": "requested", + "at": 1499350936347, + "agent": "system" + }, + { + "event": "reserved", + "at": 1499353058227, + "agent": "application" + }, + { + "event": "activated", + "at": 1499353080459, + "agent": "application" + } + ], + "ipAddresses": [ + "10.213.181.113", + "2001:4998:c:2940::111c" + ], + "additionalIpAddresses": [ + ] +} \ No newline at end of file diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/pingresponse-all-packets-lost b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/pingresponse-all-packets-lost new file mode 100644 index 00000000000..0e6a67d3d89 --- /dev/null +++ b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/pingresponse-all-packets-lost @@ -0,0 +1 @@ +6 packets transmitted, 6 received, 100% packet loss, time 5002ms diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/validpingresponse b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/validpingresponse new file mode 100644 index 00000000000..f6ef0559571 --- /dev/null +++ b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/validpingresponse @@ -0,0 +1 @@ +6 packets transmitted, 6 received, 0% packet loss, time 5002ms diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/DiskRetrieverTest.java b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/DiskRetrieverTest.java index 0cb383a2459..cf035206040 100644 --- a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/DiskRetrieverTest.java +++ b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/DiskRetrieverTest.java @@ -2,15 +2,14 @@ package com.yahoo.vespa.hosted.node.verification.spec.retrievers; import com.yahoo.vespa.hosted.node.verification.commons.ParseResult; import com.yahoo.vespa.hosted.node.verification.mock.MockCommandExecutor; +import com.yahoo.vespa.hosted.node.verification.spec.retrievers.HardwareInfo.DiskType; import org.junit.Before; -import org.junit.Ignore; import org.junit.Test; import java.io.IOException; import java.util.ArrayList; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; /** * Created by olaa on 06/07/2017. @@ -31,22 +30,20 @@ public class DiskRetrieverTest { } @Test - @Ignore // Temporary - Just to get factory going again public void updateInfo_should_store_diskType_and_diskSize_in_hardware_info() { commandExecutor.addCommand(CAT_RESOURCE_PATH + "DiskTypeFastDisk"); commandExecutor.addCommand(CAT_RESOURCE_PATH + "filesize"); diskRetriever.updateInfo(); - assertTrue(hardwareInfo.getFastDisk()); + assertEquals(DiskType.FAST, hardwareInfo.getDiskType()); double expectedSize = 63D; assertEquals(expectedSize, hardwareInfo.getMinDiskAvailableGb(), DELTA); } @Test - @Ignore // Temporary - Just to get factory going again public void updateDiskType__should_store_diskType_in_hardwareInfo() throws IOException { commandExecutor.addCommand(CAT_RESOURCE_PATH + "DiskTypeFastDisk"); diskRetriever.updateDiskType(); - assertTrue(hardwareInfo.getFastDisk()); + assertEquals(DiskType.FAST, hardwareInfo.getDiskType()); } @Test @@ -95,4 +92,12 @@ public class DiskRetrieverTest { assertEquals(expectedParseResult, parseResult); } + @Test + public void setDiskType_invalid_ParseResult_should_set_fastDisk_to_invalid() { + ParseResult parseResult = new ParseResult("Invalid", "Invalid"); + diskRetriever.setDiskType(parseResult); + HardwareInfo.DiskType expectedDiskType = HardwareInfo.DiskType.UNKNOWN; + assertEquals(expectedDiskType, hardwareInfo.getDiskType()); + } + } \ No newline at end of file diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/HardwareInfoRetrieverTest.java b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/HardwareInfoRetrieverTest.java new file mode 100644 index 00000000000..f549753918a --- /dev/null +++ b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/HardwareInfoRetrieverTest.java @@ -0,0 +1,58 @@ +package com.yahoo.vespa.hosted.node.verification.spec.retrievers; + + +import com.yahoo.vespa.hosted.node.verification.mock.MockCommandExecutor; +import org.junit.Before; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + +public class HardwareInfoRetrieverTest { + + private static final String RESOURCE_PATH = "src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/"; + private static final String CPU_INFO_PATH = RESOURCE_PATH + "cpuinfoTest"; + private static final String MEMORY_INFO_PATH = RESOURCE_PATH + "meminfoTest"; + private static final String DISK_TYPE_INFO_PATH = RESOURCE_PATH + "DiskTypeFastDisk"; + private static final String DISK_SIZE_INFO_PATH = RESOURCE_PATH + "filesize"; + private static final String NET_INTERFACE_INFO_PATH = RESOURCE_PATH + "ifconfigNoIpv6"; + private static final String NET_INTERFACE_SPEED_INFO_PATH = RESOURCE_PATH + "eth0"; + private static String PING_RESPONSE = RESOURCE_PATH + "invalidpingresponse"; + private MockCommandExecutor mockCommandExecutor; + private HardwareInfo expectedHardwareInfo; + private static final double DELTA = 0.1; + + @Before + public void setup() { + mockCommandExecutor = new MockCommandExecutor(); + mockCommandExecutor.addCommand("cat " + CPU_INFO_PATH); + mockCommandExecutor.addCommand("cat " + MEMORY_INFO_PATH); + mockCommandExecutor.addCommand("cat " + DISK_TYPE_INFO_PATH); + mockCommandExecutor.addCommand("cat " + DISK_SIZE_INFO_PATH); + mockCommandExecutor.addCommand("cat " + NET_INTERFACE_INFO_PATH); + mockCommandExecutor.addCommand("cat " + NET_INTERFACE_SPEED_INFO_PATH); + mockCommandExecutor.addCommand("cat " + PING_RESPONSE); + + expectedHardwareInfo = new HardwareInfo(); + expectedHardwareInfo.setMinCpuCores(4); + expectedHardwareInfo.setMinMainMemoryAvailableGb(4.042128); + expectedHardwareInfo.setInterfaceSpeedMbs(1000); + expectedHardwareInfo.setMinDiskAvailableGb(63); + expectedHardwareInfo.setIpv4Interface(true); + expectedHardwareInfo.setIpv6Interface(false); + expectedHardwareInfo.setIpv6Connection(false); + expectedHardwareInfo.setDiskType(HardwareInfo.DiskType.FAST); + } + + @Test + public void retriever_should_return_valid_HardwareInfo() { + HardwareInfo actualHardwareInfo = HardwareInfoRetriever.retrieve(mockCommandExecutor); + assertEquals(expectedHardwareInfo.getMinDiskAvailableGb(), actualHardwareInfo.getMinDiskAvailableGb(), DELTA); + assertEquals(expectedHardwareInfo.getMinMainMemoryAvailableGb(), actualHardwareInfo.getMinMainMemoryAvailableGb(), DELTA); + assertEquals(expectedHardwareInfo.getMinCpuCores(), actualHardwareInfo.getMinCpuCores()); + assertEquals(expectedHardwareInfo.getIpv4Interface(), actualHardwareInfo.getIpv4Interface()); + assertEquals(expectedHardwareInfo.getIpv6Interface(), actualHardwareInfo.getIpv6Interface()); + assertEquals(expectedHardwareInfo.getInterfaceSpeedMbs(), actualHardwareInfo.getInterfaceSpeedMbs(), DELTA); + assertEquals(expectedHardwareInfo.getDiskType(), actualHardwareInfo.getDiskType()); + assertEquals(expectedHardwareInfo.isIpv6Connection(), actualHardwareInfo.isIpv6Connection()); + } +} \ No newline at end of file diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/NetRetrieverTest.java b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/NetRetrieverTest.java index 8f02917237e..b3a273ddd39 100644 --- a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/NetRetrieverTest.java +++ b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/NetRetrieverTest.java @@ -10,6 +10,7 @@ import java.util.ArrayList; import java.util.Arrays; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; /** @@ -17,8 +18,12 @@ import static org.junit.Assert.assertTrue; */ public class NetRetrieverTest { - private static final String NET_FIND_INTERFACE = "src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/ifconfig"; - private static final String NET_CHECK_INTERFACE_SPEED = "src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/"; + private static final String RESOURCE_PATH = "src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/"; + private static final String NET_FIND_INTERFACE = RESOURCE_PATH + "ifconfig"; + private static final String NET_CHECK_INTERFACE_SPEED = RESOURCE_PATH + "eth0"; + private static String VALID_PING_RESPONSE = RESOURCE_PATH + "validpingresponse"; + private static String INVALID_PING_RESPONSE = RESOURCE_PATH + "invalidpingresponse"; + private static String PING_SEARCH_WORD = "loss,"; private HardwareInfo hardwareInfo; private MockCommandExecutor commandExecutor; private NetRetriever net; @@ -34,12 +39,14 @@ public class NetRetrieverTest { } @Test - public void updateInfo_should_store_ipv4_ipv6_connectivity_and_interface_speed() { + public void updateInfo_should_store_ipv4_ipv6_interface_and_interface_speed() { commandExecutor.addCommand("cat " + NET_FIND_INTERFACE); - commandExecutor.addCommand("cat " + NET_CHECK_INTERFACE_SPEED + "eth0"); + commandExecutor.addCommand("cat " + NET_CHECK_INTERFACE_SPEED); + commandExecutor.addCommand("cat " + VALID_PING_RESPONSE); net.updateInfo(); - assertTrue(hardwareInfo.getIpv4Connectivity()); - assertTrue(hardwareInfo.getIpv6Connectivity()); + assertTrue(hardwareInfo.getIpv4Interface()); + assertTrue(hardwareInfo.getIpv6Interface()); + assertTrue(hardwareInfo.isIpv6Connection()); double expectedInterfaceSpeed = 1000; assertEquals(expectedInterfaceSpeed, hardwareInfo.getInterfaceSpeedMbs(), DELTA); } @@ -55,7 +62,7 @@ public class NetRetrieverTest { @Test public void findInterfaceSpeed_valid_input() throws IOException { commandExecutor.addCommand("cat " + NET_FIND_INTERFACE); - commandExecutor.addCommand("cat " + NET_CHECK_INTERFACE_SPEED + "eth0"); + commandExecutor.addCommand("cat " + NET_CHECK_INTERFACE_SPEED); parseResults = net.findInterface(); net.findInterfaceSpeed(parseResults); ParseResult expectedParseResults = new ParseResult("Speed", "1000Mb/s"); @@ -67,8 +74,8 @@ public class NetRetrieverTest { ArrayList mockOutput = MockCommandExecutor.readFromFile(NET_FIND_INTERFACE); parseResults = net.parseNetInterface(mockOutput); net.updateHardwareInfoWithNet(parseResults); - assertTrue(hardwareInfo.getIpv4Connectivity()); - assertTrue(hardwareInfo.getIpv6Connectivity()); + assertTrue(hardwareInfo.getIpv4Interface()); + assertTrue(hardwareInfo.getIpv6Interface()); } @Test @@ -121,8 +128,53 @@ public class NetRetrieverTest { net.updateHardwareInfoWithNet(parseResults); double expectedInterfaceSpeed = 1000; assertEquals(expectedInterfaceSpeed, hardwareInfo.getInterfaceSpeedMbs(), DELTA); - assertTrue(hardwareInfo.getIpv4Connectivity()); - assertTrue(hardwareInfo.getIpv6Connectivity()); + assertTrue(hardwareInfo.getIpv4Interface()); + assertTrue(hardwareInfo.getIpv6Interface()); + } + + @Test + public void stripInterfaceSpeed_should_return_correct_double() { + String interfaceSpeedToConvert = "1000Mb/s"; + double expectedInterfaceSpeed = 1000; + double actualInterfaceSpeed = net.convertInterfaceSpeed(interfaceSpeedToConvert); + assertEquals(expectedInterfaceSpeed, actualInterfaceSpeed, DELTA); + } + + @Test + public void parsePingResponse_valid_ping_response_should_return_ipv6_connectivity() throws IOException { + ArrayList mockCommandOutput = MockCommandExecutor.readFromFile(VALID_PING_RESPONSE); + ParseResult parseResult = net.parsePingResponse(mockCommandOutput); + String expectedPing = "0%"; + assertEquals(expectedPing, parseResult.getValue()); + } + + @Test + public void parsePingResponse_invalid_ping_response_should_return_invalid_ParseResult() throws IOException { + ArrayList mockCommandOutput = MockCommandExecutor.readFromFile(INVALID_PING_RESPONSE); + ParseResult parseResult = net.parsePingResponse(mockCommandOutput); + ParseResult expectedParseResult = new ParseResult(PING_SEARCH_WORD, "invalid"); + assertEquals(expectedParseResult, parseResult); + } + + @Test + public void setIpv6Connectivity_valid_ping_response_should_return_ipv6_connectivity() { + ParseResult parseResult = new ParseResult(PING_SEARCH_WORD, "0%"); + net.setIpv6Connectivity(parseResult); + assertTrue(hardwareInfo.isIpv6Connection()); + } + + @Test + public void setIpv6Connectivity_invalid_ping_response_should_return_no_ipv6_connectivity_1() { + ParseResult parseResult = new ParseResult(PING_SEARCH_WORD, "100%"); + net.setIpv6Connectivity(parseResult); + assertFalse(hardwareInfo.isIpv6Connection()); + } + + @Test + public void setIpv6Connectivity_invalid_ping_response_should_return_no_ipv6_connectivity_2() { + ParseResult parseResult = new ParseResult(PING_SEARCH_WORD, "invalid"); + net.setIpv6Connectivity(parseResult); + assertFalse(hardwareInfo.isIpv6Connection()); } } \ No newline at end of file diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/yamasreport/YamasSpecReportTest.java b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/yamasreport/YamasSpecReportTest.java index 2131d09c0dd..db41f78d42d 100644 --- a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/yamasreport/YamasSpecReportTest.java +++ b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/yamasreport/YamasSpecReportTest.java @@ -1,6 +1,7 @@ package com.yahoo.vespa.hosted.node.verification.spec.yamasreport; import com.fasterxml.jackson.databind.ObjectMapper; +import com.yahoo.vespa.hosted.node.verification.spec.retrievers.HardwareInfo; import org.junit.Before; import org.junit.Test; @@ -24,13 +25,13 @@ public class YamasSpecReportTest { specReportDimensions.setIpv6Match(true); specReportDimensions.setMemoryMatch(true); specReportDimensions.setNetInterfaceSpeedMatch(true); - specReportDimensions.setFastDiskMatch(true); + specReportDimensions.setDiskTypeMatch(true); specReportMetrics.setActualInterfaceSpeed(100D); specReportMetrics.setExpectedInterfaceSpeed(100D); specReportMetrics.setActualDiskSpaceAvailable(500D); specReportMetrics.setExpectedDiskSpaceAvailable(500D); - specReportMetrics.setActualDiskType(true); - specReportMetrics.setExpectedDiskType(true); + specReportMetrics.setActualDiskType(HardwareInfo.DiskType.FAST); + specReportMetrics.setExpectedDiskType(HardwareInfo.DiskType.FAST); specReportMetrics.setActualMemoryAvailable(123D); specReportMetrics.setExpectedMemoryAvailable(123D); specReportMetrics.setActualcpuCores(4); @@ -45,7 +46,7 @@ public class YamasSpecReportTest { yamasSpecReport.setDimensions(specReportDimensions); yamasSpecReport.setMetrics(specReportMetrics); long time = yamasSpecReport.getTimeStamp(); - String expectedJson = "{\"timeStamp\":" + time + ",\"dimensions\":{\"memoryMatch\":true,\"cpuCoresMatch\":true,\"fastDiskMatch\":true,\"netInterfaceSpeedMatch\":true,\"diskAvailableMatch\":true,\"ipv4Match\":true,\"ipv6Match\":true},\"metrics\":{\"match\":true,\"expectedMemoryAvailable\":123.0,\"actualMemoryAvailable\":123.0,\"expectedFastDisk\":true,\"actualFastDisk\":true,\"expectedDiskSpaceAvailable\":500.0,\"actualDiskSpaceAvailable\":500.0,\"expectedInterfaceSpeed\":100.0,\"actualInterfaceSpeed\":100.0,\"expectedcpuCores\":4,\"actualcpuCores\":4},\"routing\":{\"yamas\":{\"namespace\":[\"Vespa\"]}}}"; + String expectedJson = "{\"timeStamp\":" + time + ",\"dimensions\":{\"memoryMatch\":true,\"cpuCoresMatch\":true,\"diskTypeMatch\":true,\"netInterfaceSpeedMatch\":true,\"diskAvailableMatch\":true,\"ipv4Match\":true,\"ipv6Match\":true},\"metrics\":{\"match\":true,\"expectedMemoryAvailable\":123.0,\"actualMemoryAvailable\":123.0,\"expectedDiskType\":\"FAST\",\"actualDiskType\":\"FAST\",\"expectedDiskSpaceAvailable\":500.0,\"actualDiskSpaceAvailable\":500.0,\"expectedInterfaceSpeed\":100.0,\"actualInterfaceSpeed\":100.0,\"expectedcpuCores\":4,\"actualcpuCores\":4},\"routing\":{\"yamas\":{\"namespace\":[\"Vespa\"]}}}"; ObjectMapper om = new ObjectMapper(); String json = om.writeValueAsString(yamasSpecReport); assertEquals(expectedJson, json); -- cgit v1.2.3