diff options
author | Sindre Grøstad <30319420+sgrostad@users.noreply.github.com> | 2017-07-31 15:21:29 +0200 |
---|---|---|
committer | Andreas Eriksen <andreer@pvv.ntnu.no> | 2017-07-31 15:21:29 +0200 |
commit | b4fc7b0ea0f710b04ebe64c6c4a9f981b6b0c25d (patch) | |
tree | 74d0a1a9b15169c5fd89679fd3698b02dda4d582 | |
parent | 517363ffc47c516da1eed186f3dd5d414f462508 (diff) |
interns/bug-fixes-net-disksize (#3031)
* Now stores netInfo also if ping6 did not work
* Added information to README
* Changed disk benchmark command to use conv = fdatasync
* Diskretriever now sums up all sizes from df to get disk size
* Extracted test JSON strings to separate files
* Disk retriever now ignores RAM and first row
11 files changed, 39 insertions, 21 deletions
diff --git a/node-maintainer/README.md b/node-maintainer/README.md index eafffd0b857..a76fccc8de9 100644 --- a/node-maintainer/README.md +++ b/node-maintainer/README.md @@ -2,3 +2,7 @@ Executes maintenance jobs, such as deleting old logs, processing and reporting coredumps, on behalf of node-admin. Node admin maintenance runs as a separate JVM from node-admin to make it possible to run it as root if needed. + +## Node Verification +Node verification for both hardware and spec. Hardware is verified by performing different benchmarking tasks, +while spec is verified by comparing information reported by the OS with the spec from node repository. diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/DiskBenchmark.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/DiskBenchmark.java index d2a83bf94c5..191b26ee584 100644 --- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/DiskBenchmark.java +++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/DiskBenchmark.java @@ -17,7 +17,7 @@ import java.util.logging.Logger; */ public class DiskBenchmark implements Benchmark { - private static final String DISK_BENCHMARK_COMMAND = "time (dd if=/dev/zero of=/tmp/tempfile bs=16G count=1 > /dev/null; sync; rm /tmp/tempfile) 2>&1 | grep bytes | awk '{ print $8 \" \" $9 }'"; + private static final String DISK_BENCHMARK_COMMAND = "(dd if=/dev/zero of=/tmp/output conv=fdatasync bs=4G count=4; rm -f /tmp/output;) 2>&1 | grep bytes | awk '{ print $8 \" \" $9 }'"; private static final String KILO_BYTE_SEARCH_WORD = "kB/s"; private static final String MEGA_BYTE_SEARCH_WORD = "MB/s"; private static final String GIGA_BYTE_SEARCH_WORD = "GB/s"; diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/DiskRetriever.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/DiskRetriever.java index 80c076efd5f..8fe9e04db30 100644 --- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/DiskRetriever.java +++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/DiskRetriever.java @@ -17,15 +17,15 @@ import java.util.logging.Logger; */ public class DiskRetriever implements HardwareRetriever { private static final String DISK_CHECK_TYPE = "lsblk -d -o name,rota"; - private static final String DISK_CHECK_SIZE = "df -BG /"; + private static final String DISK_CHECK_SIZE = "df -BG | grep -v tmpfs | awk '{s+=$2} END {print s-1}'"; private static final String DISK_NAME = "sda"; private static final String DISK_TYPE_REGEX_SPLIT = "\\s+"; private static final int DISK_TYPE_SEARCH_ELEMENT_INDEX = 0; private static final int DISK_TYPE_RETURN_ELEMENT_INDEX = 1; private static final String DISK_SIZE_SEARCH_WORD = ".*\\d+.*"; private static final String DISK_SIZE_REGEX_SPLIT = "\\s+"; - private static final int DISK_SIZE_SEARCH_ELEMENT_INDEX = 3; - private static final int DISK_SIZE_RETURN_ELEMENT_INDEX = 1; + private static final int DISK_SIZE_SEARCH_ELEMENT_INDEX = 0; + private static final int DISK_SIZE_RETURN_ELEMENT_INDEX = 0; private static final Logger logger = Logger.getLogger(DiskRetriever.class.getName()); private final HardwareInfo hardwareInfo; private final CommandExecutor commandExecutor; @@ -77,6 +77,12 @@ public class DiskRetriever implements HardwareRetriever { } } + protected ParseResult parseDiskSize(ArrayList<String> commandOutput) { + ArrayList<String> searchWords = new ArrayList<>(Arrays.asList(DISK_SIZE_SEARCH_WORD)); + ParseInstructions parseInstructions = new ParseInstructions(DISK_SIZE_SEARCH_ELEMENT_INDEX, DISK_SIZE_RETURN_ELEMENT_INDEX, DISK_SIZE_REGEX_SPLIT, searchWords); + return OutputParser.parseSingleOutput(parseInstructions, commandOutput); + } + protected void setDiskSize(ParseResult parseResult) { try { String sizeValue = parseResult.getValue().replaceAll("[^\\d.]", ""); @@ -87,10 +93,4 @@ public class DiskRetriever implements HardwareRetriever { } } - protected ParseResult parseDiskSize(ArrayList<String> commandOutput) { - ArrayList<String> searchWords = new ArrayList<>(Arrays.asList(DISK_SIZE_SEARCH_WORD)); - ParseInstructions parseInstructions = new ParseInstructions(DISK_SIZE_SEARCH_ELEMENT_INDEX, DISK_SIZE_RETURN_ELEMENT_INDEX, DISK_SIZE_REGEX_SPLIT, searchWords); - return OutputParser.parseSingleOutput(parseInstructions, commandOutput); - } - }
\ No newline at end of file diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/NetRetriever.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/NetRetriever.java index 21c269119cb..0170d79b22d 100644 --- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/NetRetriever.java +++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/NetRetriever.java @@ -45,14 +45,17 @@ public class NetRetriever implements HardwareRetriever { } public void updateInfo() { + ArrayList<ParseResult> parseResults = new ArrayList<>(); try { - ArrayList<ParseResult> parseResults = findInterface(); + parseResults = findInterface(); findInterfaceSpeed(parseResults); testPingResponse(parseResults); - updateHardwareInfoWithNet(parseResults); } catch (IOException e) { logger.log(Level.WARNING, "Failed to retrieve net info", e); } + finally { + updateHardwareInfoWithNet(parseResults); + } } protected ArrayList<ParseResult> findInterface() throws IOException { diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/yamasreport/YamasSpecReport.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/yamasreport/YamasSpecReport.java index 8ab3c479cb2..20cca45a7ad 100644 --- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/yamasreport/YamasSpecReport.java +++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/yamasreport/YamasSpecReport.java @@ -50,6 +50,10 @@ public class YamasSpecReport { return this.timeStamp; } + public void setTimeStamp(long timeStamp) { + this.timeStamp = timeStamp; + } + private void setRouting() { JsonObjectWrapper<String[]> wrap = new JsonObjectWrapper<>("namespace", new String[]{"Vespa"}); routing = new JsonObjectWrapper<>("yamas", wrap); diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/SpecVerifierTest.java b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/SpecVerifierTest.java index 461169071a1..a995c295ee2 100644 --- a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/SpecVerifierTest.java +++ b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/SpecVerifierTest.java @@ -26,6 +26,7 @@ public class SpecVerifierTest { private static final String RESOURCE_PATH = "src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources"; private static final String URL_RESOURCE_PATH = "file://" + ABSOLUTE_PATH + "/" + RESOURCE_PATH; private static final String NODE_REPO_PATH = "src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/nodeInfoTest.json"; + private static final String YAMAS_REPORT_PATH = "src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/SpecVerifierReport"; private static final String CPU_INFO_PATH = RESOURCE_PATH + "/cpuinfoTest"; private static final String MEMORY_INFO_PATH = RESOURCE_PATH + "/meminfoTest"; private static final String DISK_TYPE_INFO_PATH = RESOURCE_PATH + "/DiskTypeFastDisk"; @@ -82,8 +83,9 @@ public class SpecVerifierTest { ArrayList<URL> url = new ArrayList<>(Arrays.asList(new File(NODE_REPO_PATH).toURI().toURL())); NodeRepoJsonModel nodeRepoJsonModel = NodeRepoInfoRetriever.retrieve(url); YamasSpecReport yamasSpecReport = SpecVerifier.makeYamasSpecReport(actualHardware, nodeRepoJsonModel); - long timeStamp = yamasSpecReport.getTimeStamp(); - String expectedJson = "{\"timeStamp\":" + timeStamp + ",\"dimensions\":{\"memoryMatch\":true,\"cpuCoresMatch\":true,\"diskTypeMatch\":true,\"netInterfaceSpeedMatch\":false,\"diskAvailableMatch\":true,\"ipv4Match\":true,\"ipv6Match\":true},\"metrics\":{\"match\":false,\"expectedInterfaceSpeed\":1000.0,\"actualInterfaceSpeed\":10009.0,\"actualIpv6Connection\":false},\"routing\":{\"yamas\":{\"namespace\":[\"Vespa\"]}}}"; + long timeStamp = 1501504035; + yamasSpecReport.setTimeStamp(timeStamp); + String expectedJson = MockCommandExecutor.readFromFile(YAMAS_REPORT_PATH).get(0); ObjectMapper om = new ObjectMapper(); String actualJson = om.writeValueAsString(yamasSpecReport); assertEquals(expectedJson, actualJson); diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/SpecVerifierReport b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/SpecVerifierReport new file mode 100644 index 00000000000..8bfad7a32d1 --- /dev/null +++ b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/SpecVerifierReport @@ -0,0 +1 @@ +{"timeStamp":1501504035,"dimensions":{"memoryMatch":true,"cpuCoresMatch":true,"diskTypeMatch":true,"netInterfaceSpeedMatch":false,"diskAvailableMatch":true,"ipv4Match":true,"ipv6Match":true},"metrics":{"match":false,"expectedInterfaceSpeed":1000.0,"actualInterfaceSpeed":10009.0,"actualIpv6Connection":false},"routing":{"yamas":{"namespace":["Vespa"]}}} diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/filesize b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/filesize index 24aeccfdb0f..4e9e2884874 100644 --- a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/filesize +++ b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/filesize @@ -1,2 +1 @@ -Filesystem 1G-blocks Used Available Use% Mounted on -overlay 63G 17G 44G 28% /
\ No newline at end of file +63
\ No newline at end of file diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/yamasJSON b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/yamasJSON new file mode 100644 index 00000000000..1a9d7b610f0 --- /dev/null +++ b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/yamasJSON @@ -0,0 +1 @@ +{"timeStamp":1501504035,"dimensions":{"memoryMatch":true,"cpuCoresMatch":true,"diskTypeMatch":true,"netInterfaceSpeedMatch":true,"diskAvailableMatch":true,"ipv4Match":true,"ipv6Match":true},"metrics":{"match":true,"expectedMemoryAvailable":123.0,"actualMemoryAvailable":123.0,"expectedDiskType":"FAST","actualDiskType":"FAST","expectedDiskSpaceAvailable":500.0,"actualDiskSpaceAvailable":500.0,"expectedInterfaceSpeed":100.0,"actualInterfaceSpeed":100.0,"expectedcpuCores":4,"actualcpuCores":4},"routing":{"yamas":{"namespace":["Vespa"]}}} diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/DiskRetrieverTest.java b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/DiskRetrieverTest.java index cf035206040..f93791bb78d 100644 --- a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/DiskRetrieverTest.java +++ b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/DiskRetrieverTest.java @@ -88,7 +88,7 @@ public class DiskRetrieverTest { String filepath = "src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/filesize"; ArrayList<String> mockOutput = MockCommandExecutor.readFromFile(filepath); ParseResult parseResult = diskRetriever.parseDiskSize(mockOutput); - ParseResult expectedParseResult = new ParseResult("44G", "63G"); + ParseResult expectedParseResult = new ParseResult("63", "63"); assertEquals(expectedParseResult, parseResult); } diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/yamasreport/YamasSpecReportTest.java b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/yamasreport/YamasSpecReportTest.java index db41f78d42d..5fe7b015c44 100644 --- a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/yamasreport/YamasSpecReportTest.java +++ b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/yamasreport/YamasSpecReportTest.java @@ -1,6 +1,7 @@ package com.yahoo.vespa.hosted.node.verification.spec.yamasreport; import com.fasterxml.jackson.databind.ObjectMapper; +import com.yahoo.vespa.hosted.node.verification.mock.MockCommandExecutor; import com.yahoo.vespa.hosted.node.verification.spec.retrievers.HardwareInfo; import org.junit.Before; import org.junit.Test; @@ -12,8 +13,10 @@ import static org.junit.Assert.assertEquals; */ public class YamasSpecReportTest { - SpecReportDimensions specReportDimensions; - SpecReportMetrics specReportMetrics; + private SpecReportDimensions specReportDimensions; + private SpecReportMetrics specReportMetrics; + private static final String YAMAS_REPORT_PATH = "src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/yamasJSON"; + @Before public void setup() { @@ -45,8 +48,9 @@ public class YamasSpecReportTest { yamasSpecReport.setMetrics(specReportMetrics); yamasSpecReport.setDimensions(specReportDimensions); yamasSpecReport.setMetrics(specReportMetrics); - long time = yamasSpecReport.getTimeStamp(); - String expectedJson = "{\"timeStamp\":" + time + ",\"dimensions\":{\"memoryMatch\":true,\"cpuCoresMatch\":true,\"diskTypeMatch\":true,\"netInterfaceSpeedMatch\":true,\"diskAvailableMatch\":true,\"ipv4Match\":true,\"ipv6Match\":true},\"metrics\":{\"match\":true,\"expectedMemoryAvailable\":123.0,\"actualMemoryAvailable\":123.0,\"expectedDiskType\":\"FAST\",\"actualDiskType\":\"FAST\",\"expectedDiskSpaceAvailable\":500.0,\"actualDiskSpaceAvailable\":500.0,\"expectedInterfaceSpeed\":100.0,\"actualInterfaceSpeed\":100.0,\"expectedcpuCores\":4,\"actualcpuCores\":4},\"routing\":{\"yamas\":{\"namespace\":[\"Vespa\"]}}}"; + long timeStamp = 1501504035; + yamasSpecReport.setTimeStamp(timeStamp); + String expectedJson = MockCommandExecutor.readFromFile(YAMAS_REPORT_PATH).get(0); ObjectMapper om = new ObjectMapper(); String json = om.writeValueAsString(yamasSpecReport); assertEquals(expectedJson, json); |