interns/bug-fixes-net-disksize (#3031)

* Now stores netInfo also if ping6 did not work

* Added information to README

* Changed disk benchmark command to use conv = fdatasync

* Diskretriever now sums up all sizes from df to get disk size

* Extracted test JSON strings to separate files

* Disk retriever now ignores RAM and first row

10 files changed, 35 insertions, 21 deletions

diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/DiskBenchmark.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/DiskBenchmark.java
index d2a83bf94c5..191b26ee584 100644
--- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/DiskBenchmark.java
+++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/DiskBenchmark.java
@@ -17,7 +17,7 @@ import java.util.logging.Logger;
  */
 public class DiskBenchmark implements Benchmark {
 
-    private static final String DISK_BENCHMARK_COMMAND = "time (dd if=/dev/zero of=/tmp/tempfile bs=16G count=1 > /dev/null; sync; rm /tmp/tempfile) 2>&1 | grep bytes | awk  '{ print $8 \" \" $9 }'";
+    private static final String DISK_BENCHMARK_COMMAND = "(dd if=/dev/zero of=/tmp/output conv=fdatasync bs=4G count=4; rm -f /tmp/output;) 2>&1 | grep bytes | awk  '{ print $8 \" \" $9 }'";
     private static final String KILO_BYTE_SEARCH_WORD = "kB/s";
     private static final String MEGA_BYTE_SEARCH_WORD = "MB/s";
     private static final String GIGA_BYTE_SEARCH_WORD = "GB/s";
diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/DiskRetriever.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/DiskRetriever.java
index 80c076efd5f..8fe9e04db30 100644
--- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/DiskRetriever.java
+++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/DiskRetriever.java
@@ -17,15 +17,15 @@ import java.util.logging.Logger;
  */
 public class DiskRetriever implements HardwareRetriever {
     private static final String DISK_CHECK_TYPE = "lsblk -d -o name,rota";
-    private static final String DISK_CHECK_SIZE = "df -BG /";
+    private static final String DISK_CHECK_SIZE = "df -BG | grep -v tmpfs | awk '{s+=$2} END {print s-1}'";
     private static final String DISK_NAME = "sda";
     private static final String DISK_TYPE_REGEX_SPLIT = "\\s+";
     private static final int DISK_TYPE_SEARCH_ELEMENT_INDEX = 0;
     private static final int DISK_TYPE_RETURN_ELEMENT_INDEX = 1;
     private static final String DISK_SIZE_SEARCH_WORD = ".*\\d+.*";
     private static final String DISK_SIZE_REGEX_SPLIT = "\\s+";
-    private static final int DISK_SIZE_SEARCH_ELEMENT_INDEX = 3;
-    private static final int DISK_SIZE_RETURN_ELEMENT_INDEX = 1;
+    private static final int DISK_SIZE_SEARCH_ELEMENT_INDEX = 0;
+    private static final int DISK_SIZE_RETURN_ELEMENT_INDEX = 0;
     private static final Logger logger = Logger.getLogger(DiskRetriever.class.getName());
     private final HardwareInfo hardwareInfo;
     private final CommandExecutor commandExecutor;
@@ -77,6 +77,12 @@ public class DiskRetriever implements HardwareRetriever {
         }
     }
 
+    protected ParseResult parseDiskSize(ArrayList<String> commandOutput) {
+        ArrayList<String> searchWords = new ArrayList<>(Arrays.asList(DISK_SIZE_SEARCH_WORD));
+        ParseInstructions parseInstructions = new ParseInstructions(DISK_SIZE_SEARCH_ELEMENT_INDEX, DISK_SIZE_RETURN_ELEMENT_INDEX, DISK_SIZE_REGEX_SPLIT, searchWords);
+        return OutputParser.parseSingleOutput(parseInstructions, commandOutput);
+    }
+
     protected void setDiskSize(ParseResult parseResult) {
         try {
             String sizeValue = parseResult.getValue().replaceAll("[^\\d.]", "");
@@ -87,10 +93,4 @@ public class DiskRetriever implements HardwareRetriever {
         }
     }
 
-    protected ParseResult parseDiskSize(ArrayList<String> commandOutput) {
-        ArrayList<String> searchWords = new ArrayList<>(Arrays.asList(DISK_SIZE_SEARCH_WORD));
-        ParseInstructions parseInstructions = new ParseInstructions(DISK_SIZE_SEARCH_ELEMENT_INDEX, DISK_SIZE_RETURN_ELEMENT_INDEX, DISK_SIZE_REGEX_SPLIT, searchWords);
-        return OutputParser.parseSingleOutput(parseInstructions, commandOutput);
-    }
-
 }
\ No newline at end of file
diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/NetRetriever.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/NetRetriever.java
index 21c269119cb..0170d79b22d 100644
--- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/NetRetriever.java
+++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/NetRetriever.java
@@ -45,14 +45,17 @@ public class NetRetriever implements HardwareRetriever {
     }
 
     public void updateInfo() {
+        ArrayList<ParseResult> parseResults = new ArrayList<>();
         try {
-            ArrayList<ParseResult> parseResults = findInterface();
+            parseResults = findInterface();
             findInterfaceSpeed(parseResults);
             testPingResponse(parseResults);
-            updateHardwareInfoWithNet(parseResults);
         } catch (IOException e) {
             logger.log(Level.WARNING, "Failed to retrieve net info", e);
         }
+        finally {
+            updateHardwareInfoWithNet(parseResults);
+        }
     }
 
     protected ArrayList<ParseResult> findInterface() throws IOException {
diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/yamasreport/YamasSpecReport.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/yamasreport/YamasSpecReport.java
index 8ab3c479cb2..20cca45a7ad 100644
--- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/yamasreport/YamasSpecReport.java
+++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/yamasreport/YamasSpecReport.java
@@ -50,6 +50,10 @@ public class YamasSpecReport {
         return this.timeStamp;
     }
 
+    public void setTimeStamp(long timeStamp) {
+        this.timeStamp = timeStamp;
+    }
+
     private void setRouting() {
         JsonObjectWrapper<String[]> wrap = new JsonObjectWrapper<>("namespace", new String[]{"Vespa"});
         routing = new JsonObjectWrapper<>("yamas", wrap);
diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/SpecVerifierTest.java b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/SpecVerifierTest.java
index 461169071a1..a995c295ee2 100644
--- a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/SpecVerifierTest.java
+++ b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/SpecVerifierTest.java
@@ -26,6 +26,7 @@ public class SpecVerifierTest {
     private static final String RESOURCE_PATH = "src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources";
     private static final String URL_RESOURCE_PATH = "file://" + ABSOLUTE_PATH + "/" + RESOURCE_PATH;
     private static final String NODE_REPO_PATH = "src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/nodeInfoTest.json";
+    private static final String YAMAS_REPORT_PATH = "src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/SpecVerifierReport";
     private static final String CPU_INFO_PATH = RESOURCE_PATH + "/cpuinfoTest";
     private static final String MEMORY_INFO_PATH = RESOURCE_PATH + "/meminfoTest";
     private static final String DISK_TYPE_INFO_PATH = RESOURCE_PATH + "/DiskTypeFastDisk";
@@ -82,8 +83,9 @@ public class SpecVerifierTest {
         ArrayList<URL> url = new ArrayList<>(Arrays.asList(new File(NODE_REPO_PATH).toURI().toURL()));
         NodeRepoJsonModel nodeRepoJsonModel = NodeRepoInfoRetriever.retrieve(url);
         YamasSpecReport yamasSpecReport = SpecVerifier.makeYamasSpecReport(actualHardware, nodeRepoJsonModel);
-        long timeStamp = yamasSpecReport.getTimeStamp();
-        String expectedJson = "{\"timeStamp\":" + timeStamp + ",\"dimensions\":{\"memoryMatch\":true,\"cpuCoresMatch\":true,\"diskTypeMatch\":true,\"netInterfaceSpeedMatch\":false,\"diskAvailableMatch\":true,\"ipv4Match\":true,\"ipv6Match\":true},\"metrics\":{\"match\":false,\"expectedInterfaceSpeed\":1000.0,\"actualInterfaceSpeed\":10009.0,\"actualIpv6Connection\":false},\"routing\":{\"yamas\":{\"namespace\":[\"Vespa\"]}}}";
+        long timeStamp = 1501504035;
+        yamasSpecReport.setTimeStamp(timeStamp);
+        String expectedJson = MockCommandExecutor.readFromFile(YAMAS_REPORT_PATH).get(0);
         ObjectMapper om = new ObjectMapper();
         String actualJson = om.writeValueAsString(yamasSpecReport);
         assertEquals(expectedJson, actualJson);
diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/SpecVerifierReport b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/SpecVerifierReport
new file mode 100644
index 00000000000..8bfad7a32d1
--- /dev/null
+++ b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/SpecVerifierReport
@@ -0,0 +1 @@
+{"timeStamp":1501504035,"dimensions":{"memoryMatch":true,"cpuCoresMatch":true,"diskTypeMatch":true,"netInterfaceSpeedMatch":false,"diskAvailableMatch":true,"ipv4Match":true,"ipv6Match":true},"metrics":{"match":false,"expectedInterfaceSpeed":1000.0,"actualInterfaceSpeed":10009.0,"actualIpv6Connection":false},"routing":{"yamas":{"namespace":["Vespa"]}}}
diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/filesize b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/filesize
index 24aeccfdb0f..4e9e2884874 100644
--- a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/filesize
+++ b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/filesize
@@ -1,2 +1 @@
-Filesystem     1G-blocks  Used Available Use% Mounted on
-overlay              63G   17G       44G  28% /
\ No newline at end of file
+63
\ No newline at end of file
diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/yamasJSON b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/yamasJSON
new file mode 100644
index 00000000000..1a9d7b610f0
--- /dev/null
+++ b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/yamasJSON
@@ -0,0 +1 @@
+{"timeStamp":1501504035,"dimensions":{"memoryMatch":true,"cpuCoresMatch":true,"diskTypeMatch":true,"netInterfaceSpeedMatch":true,"diskAvailableMatch":true,"ipv4Match":true,"ipv6Match":true},"metrics":{"match":true,"expectedMemoryAvailable":123.0,"actualMemoryAvailable":123.0,"expectedDiskType":"FAST","actualDiskType":"FAST","expectedDiskSpaceAvailable":500.0,"actualDiskSpaceAvailable":500.0,"expectedInterfaceSpeed":100.0,"actualInterfaceSpeed":100.0,"expectedcpuCores":4,"actualcpuCores":4},"routing":{"yamas":{"namespace":["Vespa"]}}}
diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/DiskRetrieverTest.java b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/DiskRetrieverTest.java
index cf035206040..f93791bb78d 100644
--- a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/DiskRetrieverTest.java
+++ b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/DiskRetrieverTest.java
@@ -88,7 +88,7 @@ public class DiskRetrieverTest {
         String filepath = "src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/filesize";
         ArrayList<String> mockOutput = MockCommandExecutor.readFromFile(filepath);
         ParseResult parseResult = diskRetriever.parseDiskSize(mockOutput);
-        ParseResult expectedParseResult = new ParseResult("44G", "63G");
+        ParseResult expectedParseResult = new ParseResult("63", "63");
         assertEquals(expectedParseResult, parseResult);
     }
 
diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/yamasreport/YamasSpecReportTest.java b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/yamasreport/YamasSpecReportTest.java
index db41f78d42d..5fe7b015c44 100644
--- a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/yamasreport/YamasSpecReportTest.java
+++ b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/yamasreport/YamasSpecReportTest.java
@@ -1,6 +1,7 @@
 package com.yahoo.vespa.hosted.node.verification.spec.yamasreport;
 
 import com.fasterxml.jackson.databind.ObjectMapper;
+import com.yahoo.vespa.hosted.node.verification.mock.MockCommandExecutor;
 import com.yahoo.vespa.hosted.node.verification.spec.retrievers.HardwareInfo;
 import org.junit.Before;
 import org.junit.Test;
@@ -12,8 +13,10 @@ import static org.junit.Assert.assertEquals;
  */
 public class YamasSpecReportTest {
 
-    SpecReportDimensions specReportDimensions;
-    SpecReportMetrics specReportMetrics;
+    private SpecReportDimensions specReportDimensions;
+    private SpecReportMetrics specReportMetrics;
+    private static final String YAMAS_REPORT_PATH = "src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/yamasJSON";
+
 
     @Before
     public void setup() {
@@ -45,8 +48,9 @@ public class YamasSpecReportTest {
         yamasSpecReport.setMetrics(specReportMetrics);
         yamasSpecReport.setDimensions(specReportDimensions);
         yamasSpecReport.setMetrics(specReportMetrics);
-        long time = yamasSpecReport.getTimeStamp();
-        String expectedJson = "{\"timeStamp\":" + time + ",\"dimensions\":{\"memoryMatch\":true,\"cpuCoresMatch\":true,\"diskTypeMatch\":true,\"netInterfaceSpeedMatch\":true,\"diskAvailableMatch\":true,\"ipv4Match\":true,\"ipv6Match\":true},\"metrics\":{\"match\":true,\"expectedMemoryAvailable\":123.0,\"actualMemoryAvailable\":123.0,\"expectedDiskType\":\"FAST\",\"actualDiskType\":\"FAST\",\"expectedDiskSpaceAvailable\":500.0,\"actualDiskSpaceAvailable\":500.0,\"expectedInterfaceSpeed\":100.0,\"actualInterfaceSpeed\":100.0,\"expectedcpuCores\":4,\"actualcpuCores\":4},\"routing\":{\"yamas\":{\"namespace\":[\"Vespa\"]}}}";
+        long timeStamp = 1501504035;
+        yamasSpecReport.setTimeStamp(timeStamp);
+        String expectedJson = MockCommandExecutor.readFromFile(YAMAS_REPORT_PATH).get(0);
         ObjectMapper om = new ObjectMapper();
         String json = om.writeValueAsString(yamasSpecReport);
         assertEquals(expectedJson, json);