From bf9c5c49df085578d04e5c64a0f1692bf39ab6a6 Mon Sep 17 00:00:00 2001 From: Ola Aunrønning Date: Tue, 8 Aug 2017 14:19:36 +0200 Subject: Begun work on result reporting. Moved noderepo to commons --- node-maintainer/pom.xml | 5 + .../node/verification/commons/ReportSender.java | 62 ++++++++++ .../commons/noderepo/IPAddressVerifier.java | 125 +++++++++++++++++++++ .../commons/noderepo/NodeJsonConverter.java | 35 ++++++ .../commons/noderepo/NodeRepoInfoRetriever.java | 35 ++++++ .../commons/noderepo/NodeRepoJsonModel.java | 90 +++++++++++++++ .../commons/report/BenchmarkReport.java | 55 +++++++++ .../commons/report/HardwareDivergenceReport.java | 23 ++++ .../commons/report/SpecVerificationReport.java | 58 ++++++++++ .../hardware/BenchmarkResultInspector.java | 2 +- .../verification/hardware/HardwareBenchmarker.java | 33 +++--- .../hardware/report/BenchmarkReport.java | 55 --------- .../verification/spec/HardwareNodeComparator.java | 48 ++++---- .../node/verification/spec/SpecVerifier.java | 41 +++---- .../node/verification/spec/VerifierSettings.java | 2 +- .../spec/noderepo/IPAddressVerifier.java | 125 --------------------- .../spec/noderepo/NodeJsonConverter.java | 35 ------ .../spec/noderepo/NodeRepoInfoRetriever.java | 35 ------ .../spec/noderepo/NodeRepoJsonModel.java | 84 -------------- .../spec/report/VerificationReport.java | 58 ---------- .../commons/noderepo/IPAddressVerifierTest.java | 76 +++++++++++++ .../commons/noderepo/NodeJsonConverterTest.java | 40 +++++++ .../noderepo/NodeRepoInfoRetrieverTest.java | 61 ++++++++++ .../hardware/BenchmarkResultInspectorTest.java | 2 +- .../hardware/HardwareBenchmarkerTest.java | 6 +- .../hardware/report/BenchmarkReportTest.java | 2 +- .../node/verification/spec/SpecVerifierTest.java | 11 +- .../spec/noderepo/IPAddressVerifierTest.java | 76 ------------- .../spec/noderepo/NodeJsonConverterTest.java | 40 ------- .../spec/noderepo/NodeRepoInfoRetrieverTest.java | 61 ---------- .../spec/report/SpecVerificationReportTest.java | 44 ++++++++ .../spec/report/VerificationReportTest.java | 43 ------- 32 files changed, 782 insertions(+), 686 deletions(-) create mode 100644 node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/ReportSender.java create mode 100644 node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/noderepo/IPAddressVerifier.java create mode 100644 node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/noderepo/NodeJsonConverter.java create mode 100644 node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/noderepo/NodeRepoInfoRetriever.java create mode 100644 node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/noderepo/NodeRepoJsonModel.java create mode 100644 node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/report/BenchmarkReport.java create mode 100644 node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/report/HardwareDivergenceReport.java create mode 100644 node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/report/SpecVerificationReport.java delete mode 100644 node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/report/BenchmarkReport.java delete mode 100644 node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/noderepo/IPAddressVerifier.java delete mode 100644 node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/noderepo/NodeJsonConverter.java delete mode 100644 node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/noderepo/NodeRepoInfoRetriever.java delete mode 100644 node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/noderepo/NodeRepoJsonModel.java delete mode 100644 node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/report/VerificationReport.java create mode 100644 node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/commons/noderepo/IPAddressVerifierTest.java create mode 100644 node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/commons/noderepo/NodeJsonConverterTest.java create mode 100644 node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/commons/noderepo/NodeRepoInfoRetrieverTest.java delete mode 100644 node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/noderepo/IPAddressVerifierTest.java delete mode 100644 node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/noderepo/NodeJsonConverterTest.java delete mode 100644 node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/noderepo/NodeRepoInfoRetrieverTest.java create mode 100644 node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/report/SpecVerificationReportTest.java delete mode 100644 node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/report/VerificationReportTest.java (limited to 'node-maintainer') diff --git a/node-maintainer/pom.xml b/node-maintainer/pom.xml index d56e12d92c6..03ed8a77f7d 100644 --- a/node-maintainer/pom.xml +++ b/node-maintainer/pom.xml @@ -32,6 +32,11 @@ config ${project.version} + + com.yahoo.vespa + application + ${project.version} + org.apache.httpcomponents httpclient diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/ReportSender.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/ReportSender.java new file mode 100644 index 00000000000..1f57392d2e9 --- /dev/null +++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/ReportSender.java @@ -0,0 +1,62 @@ +package com.yahoo.vespa.hosted.node.verification.commons; + +import com.fasterxml.jackson.databind.ObjectMapper; + +import com.yahoo.application.Networking; +import com.yahoo.application.container.handler.Request; +import com.yahoo.application.container.JDisc; +import com.yahoo.application.container.handler.Response; +import com.yahoo.text.Utf8; +import com.yahoo.vespa.hosted.node.verification.commons.noderepo.NodeRepoInfoRetriever; +import com.yahoo.vespa.hosted.node.verification.commons.noderepo.NodeRepoJsonModel; +import com.yahoo.vespa.hosted.node.verification.commons.report.BenchmarkReport; +import com.yahoo.vespa.hosted.node.verification.commons.report.HardwareDivergenceReport; +import com.yahoo.vespa.hosted.node.verification.commons.report.SpecVerificationReport; + +import java.io.IOException; +import java.net.URL; +import java.util.ArrayList; + +public class ReportSender { + + private static void updateNodeRepository(ArrayList nodeInfoUrls, HardwareDivergenceReport hardwareDivergenceReport) throws IOException { + ObjectMapper om = new ObjectMapper(); + String report = "{\"hardwareDivergence\": " + om.writeValueAsString(hardwareDivergenceReport) + "}"; + System.out.println(report); + /* + //TODO: Update node repo + String url = nodeInfoUrls.get(0).toString(); + Request request = new Request(url, Utf8.toBytes(report), Request.Method.PATCH); + JDisc container = JDisc.fromServicesXml("", Networking.disable); + Response response = container.handleRequest(request); + container.close(); + + for (URL nodeInfoUrl : nodeInfoUrls) { + new Request(nodeInfoUrl.toString(), + Utf8.toBytes(report), + Request.Method.PATCH); + + }*/ + } + + public static void reportBenchmarkResults(BenchmarkReport benchmarkReport, ArrayList nodeInfoUrls) throws IOException { + HardwareDivergenceReport hardwareDivergenceReport = generateHardwareDivergenceReport(nodeInfoUrls); + hardwareDivergenceReport.setBenchmarkReport(benchmarkReport); + updateNodeRepository(nodeInfoUrls, hardwareDivergenceReport); + } + + public static void reportSpecVerificationResults(SpecVerificationReport specVerificationReport, ArrayList nodeInfoUrls) throws IOException { + HardwareDivergenceReport hardwareDivergenceReport = generateHardwareDivergenceReport(nodeInfoUrls); + hardwareDivergenceReport.setSpecVerificationReport(specVerificationReport); + updateNodeRepository(nodeInfoUrls, hardwareDivergenceReport); + } + + private static HardwareDivergenceReport generateHardwareDivergenceReport(ArrayList nodeInfoUrls) throws IOException { + NodeRepoJsonModel nodeRepoJsonModel = NodeRepoInfoRetriever.retrieve(nodeInfoUrls); + ObjectMapper om = new ObjectMapper(); + if (nodeRepoJsonModel.getHardwareDivergence() == null) { + return new HardwareDivergenceReport(); + } + return om.readValue(nodeRepoJsonModel.getHardwareDivergence(), HardwareDivergenceReport.class); + } +} diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/noderepo/IPAddressVerifier.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/noderepo/IPAddressVerifier.java new file mode 100644 index 00000000000..b9704f464c7 --- /dev/null +++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/noderepo/IPAddressVerifier.java @@ -0,0 +1,125 @@ +package com.yahoo.vespa.hosted.node.verification.commons.noderepo; + +import com.yahoo.vespa.hosted.node.verification.commons.report.SpecVerificationReport; + +import javax.naming.NamingEnumeration; +import javax.naming.NamingException; +import javax.naming.directory.Attribute; +import javax.naming.directory.Attributes; +import javax.naming.directory.DirContext; +import javax.naming.directory.InitialDirContext; +import java.util.ArrayList; +import java.util.Enumeration; +import java.util.Hashtable; +import java.util.logging.Level; +import java.util.logging.Logger; + +/** + * Created by olaa on 14/07/2017. + * Verifies that the IP addresses of a node points to the correct hostname + */ + +public class IPAddressVerifier { + + private static final Logger logger = Logger.getLogger(IPAddressVerifier.class.getName()); + + public void reportFaultyIpAddresses(NodeRepoJsonModel nodeRepoJsonModel, SpecVerificationReport specVerificationReport) { + String[] faultyIpAddresses = getFaultyIpAddresses(nodeRepoJsonModel); + if (faultyIpAddresses.length > 0) { + specVerificationReport.setFaultyIpAddresses(faultyIpAddresses); + } + } + + public String[] getFaultyIpAddresses(NodeRepoJsonModel jsonModel) { + String expectedHostname = jsonModel.getHostname(); + ArrayList faultyIpAddresses = new ArrayList<>(); + if (expectedHostname == null || expectedHostname.equals("")) + return new String[0]; + if (!isValidIpv4(jsonModel.getIpv4Address(), expectedHostname)) { + faultyIpAddresses.add(jsonModel.getIpv4Address()); + } + if (!isValidIpv6(jsonModel.getIpv6Address(), expectedHostname)) { + faultyIpAddresses.add(jsonModel.getIpv6Address()); + } + return faultyIpAddresses.stream().toArray(String[]::new); + } + + private boolean isValidIpv4(String ipv4Address, String expectedHostname) { + if (ipv4Address == null) { + return true; + } + String ipv4LookupFormat = convertIpv4ToLookupFormat(ipv4Address); + try { + String ipv4Hostname = reverseLookUp(ipv4LookupFormat); + return ipv4Hostname.equals(expectedHostname); + } catch (NamingException e) { + logger.log(Level.WARNING, "Could not get IPv4 hostname", e); + } + return false; + } + + private boolean isValidIpv6(String ipv6Address, String expectedHostname) { + if (ipv6Address == null) { + return true; + } + String ipv6LookupFormat = convertIpv6ToLookupFormat(ipv6Address); + try { + String ipv6Hostname = reverseLookUp(ipv6LookupFormat); + return ipv6Hostname.equals(expectedHostname); + } catch (NamingException e) { + logger.log(Level.WARNING, "Could not get IPv6 hostname", e); + } + return false; + } + + protected String reverseLookUp(String ipAddress) throws NamingException { + Hashtable env = new Hashtable<>(); + env.put("java.naming.factory.initial", "com.sun.jndi.dns.DnsContextFactory"); + String attributeName = ipAddress; + DirContext ctx = new InitialDirContext(env); + Attributes attrs = ctx.getAttributes(attributeName, new String[]{"PTR"}); + for (NamingEnumeration ae = attrs.getAll(); ae.hasMoreElements(); ) { + Attribute attr = ae.next(); + Enumeration vals = attr.getAll(); + if (vals.hasMoreElements()) { + String hostname = vals.nextElement().toString(); + ctx.close(); + return hostname.substring(0, hostname.length() - 1); + } + } + ctx.close(); + return ""; + } + + protected String convertIpv6ToLookupFormat(String ipAddress) { + StringBuilder newIpAddress = new StringBuilder(); + String doubleColonReplacement = "0.0.0.0.0.0.0.0.0.0.0.0."; + String domain = "ip6.arpa"; + String[] hextets = ipAddress.split(":"); + for (int i = hextets.length - 1; i >= 0; i--) { + String reversedHextet = new StringBuilder(hextets[i]).reverse().toString(); + if (reversedHextet.equals("")) { + newIpAddress.append(doubleColonReplacement); + continue; + } + String trailingZeroes = "0000"; + String paddedHextet = (reversedHextet + trailingZeroes).substring(0, trailingZeroes.length()); + String punctuatedHextet = paddedHextet.replaceAll(".(?=)", "$0."); + newIpAddress.append(punctuatedHextet); + } + newIpAddress.append(domain); + return newIpAddress.toString(); + } + + protected String convertIpv4ToLookupFormat(String ipAddress) { + String domain = "in-addr.arpa"; + String[] octets = ipAddress.split("\\."); + StringBuilder convertedIpAddress = new StringBuilder(); + for (int i = octets.length - 1; i >= 0; i--) { + convertedIpAddress.append(octets[i]).append("."); + } + convertedIpAddress.append(domain); + return convertedIpAddress.toString(); + } + +} diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/noderepo/NodeJsonConverter.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/noderepo/NodeJsonConverter.java new file mode 100644 index 00000000000..b1e20fbbb3e --- /dev/null +++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/noderepo/NodeJsonConverter.java @@ -0,0 +1,35 @@ +package com.yahoo.vespa.hosted.node.verification.commons.noderepo; + +import com.yahoo.vespa.hosted.node.verification.spec.retrievers.HardwareInfo; + +/** + * Created by olaa on 07/07/2017. + * Converts a NodeRepoJsonModel object to a HardwareInfo object. + */ +public class NodeJsonConverter { + + private static void addStandardSpecifications(HardwareInfo nodeRepoHardwareInfo) { + nodeRepoHardwareInfo.setInterfaceSpeedMbs(1000); + } + + protected static void setIpv6Interface(NodeRepoJsonModel nodeRepoJsonModel, HardwareInfo nodeRepoHardwareInfo) { + if (nodeRepoJsonModel.getIpv6Address() != null) { + nodeRepoHardwareInfo.setIpv6Interface(true); + } + } + + protected static void setIpv4Interface(NodeRepoJsonModel nodeRepoJsonModel, HardwareInfo nodeRepoHardwareInfo) { + if (nodeRepoJsonModel.getIpv4Address() != null) { + nodeRepoHardwareInfo.setIpv4Interface(true); + } + } + + public static HardwareInfo convertJsonModelToHardwareInfo(NodeRepoJsonModel nodeRepoJsonModel) { + HardwareInfo nodeRepoHardwareInfo = nodeRepoJsonModel.copyToHardwareInfo(); + addStandardSpecifications(nodeRepoHardwareInfo); + setIpv4Interface(nodeRepoJsonModel, nodeRepoHardwareInfo); + setIpv6Interface(nodeRepoJsonModel, nodeRepoHardwareInfo); + return nodeRepoHardwareInfo; + } + +} diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/noderepo/NodeRepoInfoRetriever.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/noderepo/NodeRepoInfoRetriever.java new file mode 100644 index 00000000000..59cf87bd44c --- /dev/null +++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/noderepo/NodeRepoInfoRetriever.java @@ -0,0 +1,35 @@ +package com.yahoo.vespa.hosted.node.verification.commons.noderepo; + +import com.fasterxml.jackson.databind.ObjectMapper; + +import java.io.IOException; +import java.net.URL; +import java.util.ArrayList; +import java.util.logging.Level; +import java.util.logging.Logger; + +/** + * Created by olaa on 04/07/2017. + * Parse JSON from node repository and stores information as a NodeRepoJsonModel object. + */ +public class NodeRepoInfoRetriever { + + private static final Logger logger = Logger.getLogger(NodeRepoInfoRetriever.class.getName()); + + public static NodeRepoJsonModel retrieve(ArrayList nodeInfoUrls) throws IOException { + NodeRepoJsonModel nodeRepoJsonModel; + ObjectMapper objectMapper = new ObjectMapper(); + for (URL nodeInfoURL : nodeInfoUrls) { + try { + nodeRepoJsonModel = objectMapper.readValue(nodeInfoURL, NodeRepoJsonModel.class); + return nodeRepoJsonModel; + + } catch (IOException e) { + logger.log(Level.WARNING, "Failed to parse JSON from config server: " + nodeInfoURL.toString(), e); + } + } + throw new IOException("Failed to parse JSON from all possible config servers."); + } + +} + diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/noderepo/NodeRepoJsonModel.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/noderepo/NodeRepoJsonModel.java new file mode 100644 index 00000000000..9a314b061c4 --- /dev/null +++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/noderepo/NodeRepoJsonModel.java @@ -0,0 +1,90 @@ +package com.yahoo.vespa.hosted.node.verification.commons.noderepo; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.net.InetAddresses; +import com.yahoo.vespa.hosted.node.verification.spec.retrievers.HardwareInfo; +import com.yahoo.vespa.hosted.node.verification.spec.retrievers.HardwareInfo.DiskType; + +import java.net.Inet4Address; +import java.net.Inet6Address; +import java.net.InetAddress; +import java.util.stream.Stream; + +/** + * Created by olaa on 05/07/2017. + * Object with the information node repositories has about the node. + */ + +@JsonIgnoreProperties(ignoreUnknown = true) +public class NodeRepoJsonModel { + @JsonProperty("minDiskAvailableGb") + private double minDiskAvailableGb; + @JsonProperty("minMainMemoryAvailableGb") + private double minMainMemoryAvailableGb; + @JsonProperty("minCpuCores") + private double minCpuCores; + @JsonProperty("fastDisk") + private boolean fastDisk; + @JsonProperty("ipAddresses") + private String[] ipAddresses; + @JsonProperty + private String hostname; + @JsonProperty + private String environment; + @JsonProperty + private String hardwareDivergence; + private String nodeRepoUrl; + + public HardwareInfo copyToHardwareInfo() { + HardwareInfo hardwareInfo = new HardwareInfo(); + hardwareInfo.setMinMainMemoryAvailableGb(this.minMainMemoryAvailableGb); + hardwareInfo.setMinDiskAvailableGb(this.minDiskAvailableGb); + hardwareInfo.setMinCpuCores((int) Math.round(this.minCpuCores)); + hardwareInfo.setDiskType(this.fastDisk ? DiskType.FAST : DiskType.SLOW); + hardwareInfo.setIpv6Connection(getIpv6Address() != null); + return hardwareInfo; + } + + public String getIpv6Address() { + return Stream.of(ipAddresses) + .map(InetAddresses::forString) + .filter(ip -> ip instanceof Inet6Address) + .findFirst().map(InetAddress::getHostAddress).orElse(null); + } + + public String getIpv4Address() { + return Stream.of(ipAddresses) + .map(InetAddresses::forString) + .filter(ip -> ip instanceof Inet4Address) + .findFirst().map(InetAddress::getHostAddress).orElse(null); + } + + public double getMinDiskAvailableGb() { + return minDiskAvailableGb; + } + + public double getMinMainMemoryAvailableGb() { + return minMainMemoryAvailableGb; + } + + public double getMinCpuCores() { + return minCpuCores; + } + + public boolean isFastDisk() { + return fastDisk; + } + + public String getHostname() { + return hostname; + } + + public String getEnvironment() { + return environment; + } + + public String getHardwareDivergence() { + return hardwareDivergence; + } +} diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/report/BenchmarkReport.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/report/BenchmarkReport.java new file mode 100644 index 00000000000..50794af278d --- /dev/null +++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/report/BenchmarkReport.java @@ -0,0 +1,55 @@ +package com.yahoo.vespa.hosted.node.verification.commons.report; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; + +/** + * Created by sgrostad on 12/07/2017. + * JSON-mapped class for reporting benchmark results to node repo + */ +@JsonInclude(JsonInclude.Include.NON_NULL) +public class BenchmarkReport { + + @JsonProperty + private Double cpuCyclesPerSec; + @JsonProperty + private Double diskSpeedMbs; + @JsonProperty + private Double memoryWriteSpeedGBs; + @JsonProperty + private Double memoryReadSpeedGBs; + + public void setCpuCyclesPerSec(double cpuCyclesPerSec) { + this.cpuCyclesPerSec = cpuCyclesPerSec; + } + + public void setDiskSpeedMbs(Double diskSpeedMbs) { + this.diskSpeedMbs = diskSpeedMbs != null ? diskSpeedMbs : -1; + } + + + public void setMemoryWriteSpeedGBs(Double memoryWriteSpeedGBs) { + this.memoryWriteSpeedGBs = memoryWriteSpeedGBs != null ? memoryWriteSpeedGBs : -1; + } + + public void setMemoryReadSpeedGBs(Double memoryReadSpeedGBs) { + this.memoryReadSpeedGBs = memoryReadSpeedGBs != null ? memoryReadSpeedGBs : -1; + } + + public Double getCpuCyclesPerSec() { + return cpuCyclesPerSec; + } + + public Double getDiskSpeedMbs() { + return diskSpeedMbs; + } + + public Double getMemoryWriteSpeedGBs() { + return memoryWriteSpeedGBs; + } + + public Double getMemoryReadSpeedGBs() { + return memoryReadSpeedGBs; + } + +} diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/report/HardwareDivergenceReport.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/report/HardwareDivergenceReport.java new file mode 100644 index 00000000000..af85b142141 --- /dev/null +++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/report/HardwareDivergenceReport.java @@ -0,0 +1,23 @@ +package com.yahoo.vespa.hosted.node.verification.commons.report; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; + +@JsonInclude(JsonInclude.Include.NON_NULL) +public class HardwareDivergenceReport { + + @JsonProperty + SpecVerificationReport specVerificationReport; + + @JsonProperty + BenchmarkReport benchmarkReport; + + public void setSpecVerificationReport(SpecVerificationReport specVerificationReport) { + this.specVerificationReport = specVerificationReport; + } + + public void setBenchmarkReport(BenchmarkReport benchmarkReport) { + this.benchmarkReport = benchmarkReport; + } + +} diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/report/SpecVerificationReport.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/report/SpecVerificationReport.java new file mode 100644 index 00000000000..b6af61e4d20 --- /dev/null +++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/report/SpecVerificationReport.java @@ -0,0 +1,58 @@ +package com.yahoo.vespa.hosted.node.verification.commons.report; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.yahoo.vespa.hosted.node.verification.spec.retrievers.HardwareInfo; + + +/** + * Stores results of caomparing node repo spec and actual hardware info. + * In case of divergent values, set the corresponding attribute to the actual hardware info value. + * Attributes of equal value remain null. + */ +@JsonInclude(JsonInclude.Include.NON_NULL) +public class SpecVerificationReport { + + @JsonProperty + private Double actualMemoryAvailable; + @JsonProperty + private HardwareInfo.DiskType actualDiskType; + @JsonProperty + private Double actualDiskSpaceAvailable; + @JsonProperty + private Double actualInterfaceSpeed; + @JsonProperty + private Integer actualcpuCores; + @JsonProperty + private String[] faultyIpAddresses; + @JsonProperty + private Boolean actualIpv6Connection; + + public void setActualIpv6Connection(boolean actualIpv6Connection) { + this.actualIpv6Connection = actualIpv6Connection; + } + + public void setActualMemoryAvailable(Double actualMemoryAvailable) { + this.actualMemoryAvailable = actualMemoryAvailable; + } + + public void setActualDiskType(HardwareInfo.DiskType actualFastDisk) { + this.actualDiskType = actualFastDisk; + } + + public void setActualDiskSpaceAvailable(Double actualDiskSpaceAvailable) { + this.actualDiskSpaceAvailable = actualDiskSpaceAvailable; + } + + public void setActualcpuCores(int actualcpuCores) { + this.actualcpuCores = actualcpuCores; + } + + public void setActualInterfaceSpeed(Double actualInterfaceSpeed) { + this.actualInterfaceSpeed = actualInterfaceSpeed; + } + + public void setFaultyIpAddresses(String[] faultyIpAddresses) { + this.faultyIpAddresses = faultyIpAddresses; + } +} diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/BenchmarkResultInspector.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/BenchmarkResultInspector.java index d5174549b9d..23f6e245e86 100644 --- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/BenchmarkResultInspector.java +++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/BenchmarkResultInspector.java @@ -1,7 +1,7 @@ package com.yahoo.vespa.hosted.node.verification.hardware; import com.yahoo.vespa.hosted.node.verification.hardware.benchmarks.BenchmarkResults; -import com.yahoo.vespa.hosted.node.verification.hardware.report.BenchmarkReport; +import com.yahoo.vespa.hosted.node.verification.commons.report.BenchmarkReport; public class BenchmarkResultInspector { diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/HardwareBenchmarker.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/HardwareBenchmarker.java index 5bd407ae0fd..7e7d3c65dc0 100644 --- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/HardwareBenchmarker.java +++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/HardwareBenchmarker.java @@ -3,13 +3,17 @@ package com.yahoo.vespa.hosted.node.verification.hardware; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.yahoo.vespa.hosted.node.verification.commons.CommandExecutor; +import com.yahoo.vespa.hosted.node.verification.commons.HostURLGenerator; +import com.yahoo.vespa.hosted.node.verification.commons.ReportSender; import com.yahoo.vespa.hosted.node.verification.hardware.benchmarks.Benchmark; import com.yahoo.vespa.hosted.node.verification.hardware.benchmarks.BenchmarkResults; import com.yahoo.vespa.hosted.node.verification.hardware.benchmarks.CPUBenchmark; import com.yahoo.vespa.hosted.node.verification.hardware.benchmarks.DiskBenchmark; import com.yahoo.vespa.hosted.node.verification.hardware.benchmarks.MemoryBenchmark; -import com.yahoo.vespa.hosted.node.verification.hardware.report.BenchmarkReport; +import com.yahoo.vespa.hosted.node.verification.commons.report.BenchmarkReport; +import java.io.IOException; +import java.net.URL; import java.util.ArrayList; import java.util.Arrays; @@ -18,7 +22,7 @@ import java.util.Arrays; */ public class HardwareBenchmarker { - public static boolean hardwareBenchmarks(CommandExecutor commandExecutor) { + public static boolean hardwareBenchmarks(CommandExecutor commandExecutor, ArrayList nodeInfoUrls) throws IOException { BenchmarkResults benchmarkResults = new BenchmarkResults(); ArrayList benchmarks = new ArrayList<>(Arrays.asList( new DiskBenchmark(benchmarkResults, commandExecutor), @@ -28,10 +32,14 @@ public class HardwareBenchmarker { benchmark.doBenchmark(); } BenchmarkReport benchmarkReport = BenchmarkResultInspector.makeBenchmarkReport(benchmarkResults); - printBenchmarkResults(benchmarkReport); + if (!isAllBenchmarksOK(benchmarkReport)) { + ReportSender.reportBenchmarkResults(benchmarkReport, nodeInfoUrls); + } return isAllBenchmarksOK(benchmarkReport); } + + private static boolean isAllBenchmarksOK(BenchmarkReport benchmarkReport) { ObjectMapper om = new ObjectMapper(); try { @@ -43,18 +51,15 @@ public class HardwareBenchmarker { } } - private static void printBenchmarkResults(BenchmarkReport benchmarkReport) { - ObjectMapper om = new ObjectMapper(); - try { - System.out.println(om.writeValueAsString(benchmarkReport)); - } catch (Exception e) { - e.printStackTrace(); - } - } - - public static void main(String[] args) { + public static void main(String[] args) throws IOException { CommandExecutor commandExecutor = new CommandExecutor(); - if (!HardwareBenchmarker.hardwareBenchmarks(commandExecutor)){ + ArrayList nodeInfoUrls; + if (args.length == 0) { + nodeInfoUrls = HostURLGenerator.generateNodeInfoUrl(commandExecutor); + } else { + nodeInfoUrls = HostURLGenerator.generateNodeInfoUrl(commandExecutor, args[0]); + } + if (!HardwareBenchmarker.hardwareBenchmarks(commandExecutor, nodeInfoUrls)){ System.exit(2); } } diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/report/BenchmarkReport.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/report/BenchmarkReport.java deleted file mode 100644 index e1b8cd9bb8c..00000000000 --- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/report/BenchmarkReport.java +++ /dev/null @@ -1,55 +0,0 @@ -package com.yahoo.vespa.hosted.node.verification.hardware.report; - -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonProperty; - -/** - * Created by sgrostad on 12/07/2017. - * JSON-mapped class for reporting benchmark results to node repo - */ -@JsonInclude(JsonInclude.Include.NON_NULL) -public class BenchmarkReport { - - @JsonProperty - private Double cpuCyclesPerSec; - @JsonProperty - private Double diskSpeedMbs; - @JsonProperty - private Double memoryWriteSpeedGBs; - @JsonProperty - private Double memoryReadSpeedGBs; - - public void setCpuCyclesPerSec(double cpuCyclesPerSec) { - this.cpuCyclesPerSec = cpuCyclesPerSec; - } - - public void setDiskSpeedMbs(Double diskSpeedMbs) { - this.diskSpeedMbs = diskSpeedMbs != null ? diskSpeedMbs : -1; - } - - - public void setMemoryWriteSpeedGBs(Double memoryWriteSpeedGBs) { - this.memoryWriteSpeedGBs = memoryWriteSpeedGBs != null ? memoryWriteSpeedGBs : -1; - } - - public void setMemoryReadSpeedGBs(Double memoryReadSpeedGBs) { - this.memoryReadSpeedGBs = memoryReadSpeedGBs != null ? memoryReadSpeedGBs : -1; - } - - public Double getCpuCyclesPerSec() { - return cpuCyclesPerSec; - } - - public Double getDiskSpeedMbs() { - return diskSpeedMbs; - } - - public Double getMemoryWriteSpeedGBs() { - return memoryWriteSpeedGBs; - } - - public Double getMemoryReadSpeedGBs() { - return memoryReadSpeedGBs; - } - -} diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/HardwareNodeComparator.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/HardwareNodeComparator.java index 82664424c0a..6d4bcf575ab 100644 --- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/HardwareNodeComparator.java +++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/HardwareNodeComparator.java @@ -1,75 +1,75 @@ package com.yahoo.vespa.hosted.node.verification.spec; +import com.yahoo.vespa.hosted.node.verification.commons.report.SpecVerificationReport; import com.yahoo.vespa.hosted.node.verification.spec.retrievers.HardwareInfo; import com.yahoo.vespa.hosted.node.verification.spec.retrievers.HardwareInfo.DiskType; -import com.yahoo.vespa.hosted.node.verification.spec.report.VerificationReport; /** * Created by olaa on 04/07/2017. - * Compares two HardwareInfo objects and stores divergent values in a VerificationReport + * Compares two HardwareInfo objects and stores divergent values in a SpecVerificationReport */ public class HardwareNodeComparator { private static final double PERCENTAGE_THRESHOLD = 0.05; - public static VerificationReport compare(HardwareInfo nodeRepoHardwareInfo, HardwareInfo actualHardware) { - VerificationReport verificationReport = new VerificationReport(); + public static SpecVerificationReport compare(HardwareInfo nodeRepoHardwareInfo, HardwareInfo actualHardware) { + SpecVerificationReport specVerificationReport = new SpecVerificationReport(); if (nodeRepoHardwareInfo == null || actualHardware == null) { - return verificationReport; + return specVerificationReport; } - setReportMetrics(nodeRepoHardwareInfo, actualHardware, verificationReport); - return verificationReport; + setReportMetrics(nodeRepoHardwareInfo, actualHardware, specVerificationReport); + return specVerificationReport; } - private static void setReportMetrics(HardwareInfo nodeRepoHardwareInfo, HardwareInfo actualHardware, VerificationReport verificationReport) { - setMemoryMetrics(nodeRepoHardwareInfo, actualHardware, verificationReport); - setCpuMetrics(nodeRepoHardwareInfo, actualHardware, verificationReport); - setDiskTypeMetrics(nodeRepoHardwareInfo, actualHardware, verificationReport); - setDiskSpaceMetrics(nodeRepoHardwareInfo, actualHardware, verificationReport); - setNetMetrics(nodeRepoHardwareInfo, actualHardware, verificationReport); + private static void setReportMetrics(HardwareInfo nodeRepoHardwareInfo, HardwareInfo actualHardware, SpecVerificationReport specVerificationReport) { + setMemoryMetrics(nodeRepoHardwareInfo, actualHardware, specVerificationReport); + setCpuMetrics(nodeRepoHardwareInfo, actualHardware, specVerificationReport); + setDiskTypeMetrics(nodeRepoHardwareInfo, actualHardware, specVerificationReport); + setDiskSpaceMetrics(nodeRepoHardwareInfo, actualHardware, specVerificationReport); + setNetMetrics(nodeRepoHardwareInfo, actualHardware, specVerificationReport); } - private static void setMemoryMetrics(HardwareInfo nodeRepoHardwareInfo, HardwareInfo actualHardware, VerificationReport verificationReport) { + private static void setMemoryMetrics(HardwareInfo nodeRepoHardwareInfo, HardwareInfo actualHardware, SpecVerificationReport specVerificationReport) { double expectedMemory = nodeRepoHardwareInfo.getMinMainMemoryAvailableGb(); double actualMemory = actualHardware.getMinMainMemoryAvailableGb(); if (outsideThreshold(expectedMemory, actualMemory, PERCENTAGE_THRESHOLD)) { - verificationReport.setActualMemoryAvailable(actualMemory); + specVerificationReport.setActualMemoryAvailable(actualMemory); } } - private static void setCpuMetrics(HardwareInfo nodeRepoHardwareInfo, HardwareInfo actualHardware, VerificationReport verificationReport) { + private static void setCpuMetrics(HardwareInfo nodeRepoHardwareInfo, HardwareInfo actualHardware, SpecVerificationReport specVerificationReport) { int expectedCpuCores = nodeRepoHardwareInfo.getMinCpuCores(); int actualCpuCores = actualHardware.getMinCpuCores(); if (expectedCpuCores != actualCpuCores) { - verificationReport.setActualcpuCores(actualCpuCores); + specVerificationReport.setActualcpuCores(actualCpuCores); } } - private static void setDiskTypeMetrics(HardwareInfo nodeRepoHardwareInfo, HardwareInfo actualHardware, VerificationReport verificationReport) { + private static void setDiskTypeMetrics(HardwareInfo nodeRepoHardwareInfo, HardwareInfo actualHardware, SpecVerificationReport specVerificationReport) { DiskType expectedFastDisk = nodeRepoHardwareInfo.getDiskType(); DiskType actualFastDisk = actualHardware.getDiskType(); if (expectedFastDisk != null && actualFastDisk != null && expectedFastDisk != actualFastDisk) { - verificationReport.setActualDiskType(actualFastDisk); + specVerificationReport.setActualDiskType(actualFastDisk); } } - private static void setDiskSpaceMetrics(HardwareInfo nodeRepoHardwareInfo, HardwareInfo actualHardware, VerificationReport verificationReport) { + private static void setDiskSpaceMetrics(HardwareInfo nodeRepoHardwareInfo, HardwareInfo actualHardware, SpecVerificationReport specVerificationReport) { double expectedDiskSpace = nodeRepoHardwareInfo.getMinDiskAvailableGb(); double actualDiskSpace = actualHardware.getMinDiskAvailableGb(); if (outsideThreshold(expectedDiskSpace, actualDiskSpace, PERCENTAGE_THRESHOLD)) { - verificationReport.setActualDiskSpaceAvailable(actualDiskSpace); + specVerificationReport.setActualDiskSpaceAvailable(actualDiskSpace); } } - private static void setNetMetrics(HardwareInfo nodeRepoHardwareInfo, HardwareInfo actualHardware, VerificationReport verificationReport) { + private static void setNetMetrics(HardwareInfo nodeRepoHardwareInfo, HardwareInfo actualHardware, SpecVerificationReport specVerificationReport) { double expectedInterfaceSpeed = nodeRepoHardwareInfo.getInterfaceSpeedMbs(); double actualInterfaceSpeed = actualHardware.getInterfaceSpeedMbs(); if (expectedInterfaceSpeed > actualInterfaceSpeed) { - verificationReport.setActualInterfaceSpeed(actualInterfaceSpeed); + specVerificationReport.setActualInterfaceSpeed(actualInterfaceSpeed); } if (nodeRepoHardwareInfo.isIpv6Connection() && !actualHardware.isIpv6Connection()) { - verificationReport.setActualIpv6Connection(actualHardware.isIpv6Connection()); + specVerificationReport.setActualIpv6Connection(actualHardware.isIpv6Connection()); } private static boolean outsideThreshold(double value1, double value2 , double thresholdPercentage) { diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/SpecVerifier.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/SpecVerifier.java index 0fcb6206237..2ef7668cc2d 100644 --- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/SpecVerifier.java +++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/SpecVerifier.java @@ -4,13 +4,14 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.yahoo.vespa.hosted.node.verification.commons.CommandExecutor; import com.yahoo.vespa.hosted.node.verification.commons.HostURLGenerator; -import com.yahoo.vespa.hosted.node.verification.spec.noderepo.IPAddressVerifier; -import com.yahoo.vespa.hosted.node.verification.spec.noderepo.NodeJsonConverter; -import com.yahoo.vespa.hosted.node.verification.spec.noderepo.NodeRepoInfoRetriever; -import com.yahoo.vespa.hosted.node.verification.spec.noderepo.NodeRepoJsonModel; +import com.yahoo.vespa.hosted.node.verification.commons.ReportSender; +import com.yahoo.vespa.hosted.node.verification.commons.noderepo.IPAddressVerifier; +import com.yahoo.vespa.hosted.node.verification.commons.noderepo.NodeJsonConverter; +import com.yahoo.vespa.hosted.node.verification.commons.noderepo.NodeRepoInfoRetriever; +import com.yahoo.vespa.hosted.node.verification.commons.noderepo.NodeRepoJsonModel; +import com.yahoo.vespa.hosted.node.verification.commons.report.SpecVerificationReport; import com.yahoo.vespa.hosted.node.verification.spec.retrievers.HardwareInfo; import com.yahoo.vespa.hosted.node.verification.spec.retrievers.HardwareInfoRetriever; -import com.yahoo.vespa.hosted.node.verification.spec.report.VerificationReport; import java.io.IOException; import java.net.URL; @@ -36,21 +37,23 @@ public class SpecVerifier { } VerifierSettings verifierSettings = new VerifierSettings(nodeRepoJsonModel); HardwareInfo actualHardware = HardwareInfoRetriever.retrieve(commandExecutor, verifierSettings); - VerificationReport verificationReport = makeVerificationReport(actualHardware, nodeRepoJsonModel); - printResults(verificationReport); - return isValidSpec(verificationReport); + SpecVerificationReport specVerificationReport = makeVerificationReport(actualHardware, nodeRepoJsonModel); + if (!isValidSpec(specVerificationReport)) { + ReportSender.reportSpecVerificationResults(specVerificationReport, nodeInfoUrls); + } + return isValidSpec(specVerificationReport); } - private static boolean isValidSpec(VerificationReport verificationReport) throws JsonProcessingException { + private static boolean isValidSpec(SpecVerificationReport specVerificationReport) throws JsonProcessingException { ObjectMapper om = new ObjectMapper(); - return om.writeValueAsString(verificationReport).length() == 2; + return om.writeValueAsString(specVerificationReport).length() == 2; } - protected static VerificationReport makeVerificationReport(HardwareInfo actualHardware, NodeRepoJsonModel nodeRepoJsonModel) { - VerificationReport verificationReport = HardwareNodeComparator.compare(NodeJsonConverter.convertJsonModelToHardwareInfo(nodeRepoJsonModel), actualHardware); + protected static SpecVerificationReport makeVerificationReport(HardwareInfo actualHardware, NodeRepoJsonModel nodeRepoJsonModel) { + SpecVerificationReport specVerificationReport = HardwareNodeComparator.compare(NodeJsonConverter.convertJsonModelToHardwareInfo(nodeRepoJsonModel), actualHardware); IPAddressVerifier ipAddressVerifier = new IPAddressVerifier(); - ipAddressVerifier.reportFaultyIpAddresses(nodeRepoJsonModel, verificationReport); - return verificationReport; + ipAddressVerifier.reportFaultyIpAddresses(nodeRepoJsonModel, specVerificationReport); + return specVerificationReport; } protected static NodeRepoJsonModel getNodeRepositoryJSON(ArrayList nodeInfoUrls) throws IOException { @@ -58,16 +61,6 @@ public class SpecVerifier { return nodeRepoJsonModel; } - private static void printResults(VerificationReport verificationReport) { - //TODO: Instead of println, report JSON to node repo - ObjectMapper om = new ObjectMapper(); - try { - System.out.println(om.writeValueAsString(verificationReport)); - } catch (JsonProcessingException e) { - e.printStackTrace(); - } - } - public static void main(String[] args) throws IOException { CommandExecutor commandExecutor = new CommandExecutor(); ArrayList nodeInfoUrls; diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/VerifierSettings.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/VerifierSettings.java index 936c19f6c7f..c170ac0bedf 100644 --- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/VerifierSettings.java +++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/VerifierSettings.java @@ -1,6 +1,6 @@ package com.yahoo.vespa.hosted.node.verification.spec; -import com.yahoo.vespa.hosted.node.verification.spec.noderepo.NodeRepoJsonModel; +import com.yahoo.vespa.hosted.node.verification.commons.noderepo.NodeRepoJsonModel; /** * Created by sgrostad on 07/08/2017. diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/noderepo/IPAddressVerifier.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/noderepo/IPAddressVerifier.java deleted file mode 100644 index 72149d561c5..00000000000 --- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/noderepo/IPAddressVerifier.java +++ /dev/null @@ -1,125 +0,0 @@ -package com.yahoo.vespa.hosted.node.verification.spec.noderepo; - -import com.yahoo.vespa.hosted.node.verification.spec.report.VerificationReport; - -import javax.naming.NamingEnumeration; -import javax.naming.NamingException; -import javax.naming.directory.Attribute; -import javax.naming.directory.Attributes; -import javax.naming.directory.DirContext; -import javax.naming.directory.InitialDirContext; -import java.util.ArrayList; -import java.util.Enumeration; -import java.util.Hashtable; -import java.util.logging.Level; -import java.util.logging.Logger; - -/** - * Created by olaa on 14/07/2017. - * Verifies that the IP addresses of a node points to the correct hostname - */ - -public class IPAddressVerifier { - - private static final Logger logger = Logger.getLogger(IPAddressVerifier.class.getName()); - - public void reportFaultyIpAddresses(NodeRepoJsonModel nodeRepoJsonModel, VerificationReport verificationReport) { - String[] faultyIpAddresses = getFaultyIpAddresses(nodeRepoJsonModel); - if (faultyIpAddresses.length > 0) { - verificationReport.setFaultyIpAddresses(faultyIpAddresses); - } - } - - public String[] getFaultyIpAddresses(NodeRepoJsonModel jsonModel) { - String expectedHostname = jsonModel.getHostname(); - ArrayList faultyIpAddresses = new ArrayList<>(); - if (expectedHostname == null || expectedHostname.equals("")) - return new String[0]; - if (!isValidIpv4(jsonModel.getIpv4Address(), expectedHostname)) { - faultyIpAddresses.add(jsonModel.getIpv4Address()); - } - if (!isValidIpv6(jsonModel.getIpv6Address(), expectedHostname)) { - faultyIpAddresses.add(jsonModel.getIpv6Address()); - } - return faultyIpAddresses.stream().toArray(String[]::new); - } - - private boolean isValidIpv4(String ipv4Address, String expectedHostname) { - if (ipv4Address == null) { - return true; - } - String ipv4LookupFormat = convertIpv4ToLookupFormat(ipv4Address); - try { - String ipv4Hostname = reverseLookUp(ipv4LookupFormat); - return ipv4Hostname.equals(expectedHostname); - } catch (NamingException e) { - logger.log(Level.WARNING, "Could not get IPv4 hostname", e); - } - return false; - } - - private boolean isValidIpv6(String ipv6Address, String expectedHostname) { - if (ipv6Address == null) { - return true; - } - String ipv6LookupFormat = convertIpv6ToLookupFormat(ipv6Address); - try { - String ipv6Hostname = reverseLookUp(ipv6LookupFormat); - return ipv6Hostname.equals(expectedHostname); - } catch (NamingException e) { - logger.log(Level.WARNING, "Could not get IPv6 hostname", e); - } - return false; - } - - protected String reverseLookUp(String ipAddress) throws NamingException { - Hashtable env = new Hashtable<>(); - env.put("java.naming.factory.initial", "com.sun.jndi.dns.DnsContextFactory"); - String attributeName = ipAddress; - DirContext ctx = new InitialDirContext(env); - Attributes attrs = ctx.getAttributes(attributeName, new String[]{"PTR"}); - for (NamingEnumeration ae = attrs.getAll(); ae.hasMoreElements(); ) { - Attribute attr = ae.next(); - Enumeration vals = attr.getAll(); - if (vals.hasMoreElements()) { - String hostname = vals.nextElement().toString(); - ctx.close(); - return hostname.substring(0, hostname.length() - 1); - } - } - ctx.close(); - return ""; - } - - protected String convertIpv6ToLookupFormat(String ipAddress) { - StringBuilder newIpAddress = new StringBuilder(); - String doubleColonReplacement = "0.0.0.0.0.0.0.0.0.0.0.0."; - String domain = "ip6.arpa"; - String[] hextets = ipAddress.split(":"); - for (int i = hextets.length - 1; i >= 0; i--) { - String reversedHextet = new StringBuilder(hextets[i]).reverse().toString(); - if (reversedHextet.equals("")) { - newIpAddress.append(doubleColonReplacement); - continue; - } - String trailingZeroes = "0000"; - String paddedHextet = (reversedHextet + trailingZeroes).substring(0, trailingZeroes.length()); - String punctuatedHextet = paddedHextet.replaceAll(".(?=)", "$0."); - newIpAddress.append(punctuatedHextet); - } - newIpAddress.append(domain); - return newIpAddress.toString(); - } - - protected String convertIpv4ToLookupFormat(String ipAddress) { - String domain = "in-addr.arpa"; - String[] octets = ipAddress.split("\\."); - StringBuilder convertedIpAddress = new StringBuilder(); - for (int i = octets.length - 1; i >= 0; i--) { - convertedIpAddress.append(octets[i]).append("."); - } - convertedIpAddress.append(domain); - return convertedIpAddress.toString(); - } - -} diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/noderepo/NodeJsonConverter.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/noderepo/NodeJsonConverter.java deleted file mode 100644 index ec078624920..00000000000 --- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/noderepo/NodeJsonConverter.java +++ /dev/null @@ -1,35 +0,0 @@ -package com.yahoo.vespa.hosted.node.verification.spec.noderepo; - -import com.yahoo.vespa.hosted.node.verification.spec.retrievers.HardwareInfo; - -/** - * Created by olaa on 07/07/2017. - * Converts a NodeRepoJsonModel object to a HardwareInfo object. - */ -public class NodeJsonConverter { - - private static void addStandardSpecifications(HardwareInfo nodeRepoHardwareInfo) { - nodeRepoHardwareInfo.setInterfaceSpeedMbs(1000); - } - - protected static void setIpv6Interface(NodeRepoJsonModel nodeRepoJsonModel, HardwareInfo nodeRepoHardwareInfo) { - if (nodeRepoJsonModel.getIpv6Address() != null) { - nodeRepoHardwareInfo.setIpv6Interface(true); - } - } - - protected static void setIpv4Interface(NodeRepoJsonModel nodeRepoJsonModel, HardwareInfo nodeRepoHardwareInfo) { - if (nodeRepoJsonModel.getIpv4Address() != null) { - nodeRepoHardwareInfo.setIpv4Interface(true); - } - } - - public static HardwareInfo convertJsonModelToHardwareInfo(NodeRepoJsonModel nodeRepoJsonModel) { - HardwareInfo nodeRepoHardwareInfo = nodeRepoJsonModel.copyToHardwareInfo(); - addStandardSpecifications(nodeRepoHardwareInfo); - setIpv4Interface(nodeRepoJsonModel, nodeRepoHardwareInfo); - setIpv6Interface(nodeRepoJsonModel, nodeRepoHardwareInfo); - return nodeRepoHardwareInfo; - } - -} diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/noderepo/NodeRepoInfoRetriever.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/noderepo/NodeRepoInfoRetriever.java deleted file mode 100644 index f3bc70310ce..00000000000 --- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/noderepo/NodeRepoInfoRetriever.java +++ /dev/null @@ -1,35 +0,0 @@ -package com.yahoo.vespa.hosted.node.verification.spec.noderepo; - -import com.fasterxml.jackson.databind.ObjectMapper; - -import java.io.IOException; -import java.net.URL; -import java.util.ArrayList; -import java.util.logging.Level; -import java.util.logging.Logger; - -/** - * Created by olaa on 04/07/2017. - * Parse JSON from node repository and stores information as a NodeRepoJsonModel object. - */ -public class NodeRepoInfoRetriever { - - private static final Logger logger = Logger.getLogger(NodeRepoInfoRetriever.class.getName()); - - public static NodeRepoJsonModel retrieve(ArrayList nodeInfoUrls) throws IOException { - NodeRepoJsonModel nodeRepoJsonModel; - ObjectMapper objectMapper = new ObjectMapper(); - for (URL nodeInfoURL : nodeInfoUrls) { - try { - nodeRepoJsonModel = objectMapper.readValue(nodeInfoURL, NodeRepoJsonModel.class); - return nodeRepoJsonModel; - - } catch (IOException e) { - logger.log(Level.WARNING, "Failed to parse JSON from config server: " + nodeInfoURL.toString(), e); - } - } - throw new IOException("Failed to parse JSON from all possible config servers."); - } - -} - diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/noderepo/NodeRepoJsonModel.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/noderepo/NodeRepoJsonModel.java deleted file mode 100644 index 319fa08f28f..00000000000 --- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/noderepo/NodeRepoJsonModel.java +++ /dev/null @@ -1,84 +0,0 @@ -package com.yahoo.vespa.hosted.node.verification.spec.noderepo; - -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.google.common.net.InetAddresses; -import com.yahoo.vespa.hosted.node.verification.spec.retrievers.HardwareInfo; -import com.yahoo.vespa.hosted.node.verification.spec.retrievers.HardwareInfo.DiskType; - -import java.net.Inet4Address; -import java.net.Inet6Address; -import java.net.InetAddress; -import java.util.stream.Stream; - -/** - * Created by olaa on 05/07/2017. - * Object with the information node repositories has about the node. - */ - -@JsonIgnoreProperties(ignoreUnknown = true) -public class NodeRepoJsonModel { - @JsonProperty("minDiskAvailableGb") - private double minDiskAvailableGb; - @JsonProperty("minMainMemoryAvailableGb") - private double minMainMemoryAvailableGb; - @JsonProperty("minCpuCores") - private double minCpuCores; - @JsonProperty("fastDisk") - private boolean fastDisk; - @JsonProperty("ipAddresses") - private String[] ipAddresses; - @JsonProperty - private String hostname; - @JsonProperty - private String environment; - - public HardwareInfo copyToHardwareInfo() { - HardwareInfo hardwareInfo = new HardwareInfo(); - hardwareInfo.setMinMainMemoryAvailableGb(this.minMainMemoryAvailableGb); - hardwareInfo.setMinDiskAvailableGb(this.minDiskAvailableGb); - hardwareInfo.setMinCpuCores((int) Math.round(this.minCpuCores)); - hardwareInfo.setDiskType(this.fastDisk ? DiskType.FAST : DiskType.SLOW); - hardwareInfo.setIpv6Connection(getIpv6Address() != null); - return hardwareInfo; - } - - public String getIpv6Address() { - return Stream.of(ipAddresses) - .map(InetAddresses::forString) - .filter(ip -> ip instanceof Inet6Address) - .findFirst().map(InetAddress::getHostAddress).orElse(null); - } - - public String getIpv4Address() { - return Stream.of(ipAddresses) - .map(InetAddresses::forString) - .filter(ip -> ip instanceof Inet4Address) - .findFirst().map(InetAddress::getHostAddress).orElse(null); - } - - public double getMinDiskAvailableGb() { - return minDiskAvailableGb; - } - - public double getMinMainMemoryAvailableGb() { - return minMainMemoryAvailableGb; - } - - public double getMinCpuCores() { - return minCpuCores; - } - - public boolean isFastDisk() { - return fastDisk; - } - - public String getHostname() { - return hostname; - } - - public String getEnvironment() { - return environment; - } - -} diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/report/VerificationReport.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/report/VerificationReport.java deleted file mode 100644 index ede4b0fc73a..00000000000 --- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/report/VerificationReport.java +++ /dev/null @@ -1,58 +0,0 @@ -package com.yahoo.vespa.hosted.node.verification.spec.report; - -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.yahoo.vespa.hosted.node.verification.spec.retrievers.HardwareInfo; - - -/** - * Stores results of caomparing node repo spec and actual hardware info. - * In case of divergent values, set the corresponding attribute to the actual hardware info value. - * Attributes of equal value remain null. - */ -@JsonInclude(JsonInclude.Include.NON_NULL) -public class VerificationReport { - - @JsonProperty - private Double actualMemoryAvailable; - @JsonProperty - private HardwareInfo.DiskType actualDiskType; - @JsonProperty - private Double actualDiskSpaceAvailable; - @JsonProperty - private Double actualInterfaceSpeed; - @JsonProperty - private Integer actualcpuCores; - @JsonProperty - private String[] faultyIpAddresses; - @JsonProperty - private Boolean actualIpv6Connection; - - public void setActualIpv6Connection(boolean actualIpv6Connection) { - this.actualIpv6Connection = actualIpv6Connection; - } - - public void setActualMemoryAvailable(Double actualMemoryAvailable) { - this.actualMemoryAvailable = actualMemoryAvailable; - } - - public void setActualDiskType(HardwareInfo.DiskType actualFastDisk) { - this.actualDiskType = actualFastDisk; - } - - public void setActualDiskSpaceAvailable(Double actualDiskSpaceAvailable) { - this.actualDiskSpaceAvailable = actualDiskSpaceAvailable; - } - - public void setActualcpuCores(int actualcpuCores) { - this.actualcpuCores = actualcpuCores; - } - - public void setActualInterfaceSpeed(Double actualInterfaceSpeed) { - this.actualInterfaceSpeed = actualInterfaceSpeed; - } - - public void setFaultyIpAddresses(String[] faultyIpAddresses) { - this.faultyIpAddresses = faultyIpAddresses; - } -} diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/commons/noderepo/IPAddressVerifierTest.java b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/commons/noderepo/IPAddressVerifierTest.java new file mode 100644 index 00000000000..50f350aea63 --- /dev/null +++ b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/commons/noderepo/IPAddressVerifierTest.java @@ -0,0 +1,76 @@ +package com.yahoo.vespa.hosted.node.verification.commons.noderepo; + +import org.junit.Before; +import org.junit.Test; + +import java.net.URL; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Arrays; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.spy; + +public class IPAddressVerifierTest { + + private IPAddressVerifier ipAddressVerifier = spy(new IPAddressVerifier()); + private String ipv4Address; + private String ipv6Address; + private NodeRepoJsonModel nodeRepoJsonModel; + private static final String ABSOLUTE_PATH = Paths.get(".").toAbsolutePath().normalize().toString(); + private static final String RESOURCE_PATH = "src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/IPAddressVerifierTest.json"; + private static final String URL_RESOURCE_PATH = "file://" + ABSOLUTE_PATH + "/" + RESOURCE_PATH; + private String ipv4LookupFormat; + private String ipv6LookupFormat; + + @Before + public void setup() throws Exception { + ipv4Address = "10.213.181.113"; + ipv6Address = "2001:4998:c:2940:0:0:0:111c"; + ArrayList nodeRepoUrl = new ArrayList<>(Arrays.asList(new URL(URL_RESOURCE_PATH))); + nodeRepoJsonModel = NodeRepoInfoRetriever.retrieve(nodeRepoUrl); + ipv4LookupFormat = "113.181.213.10.in-addr.arpa"; + ipv6LookupFormat = "c.1.1.1.0.0.0.0.0.0.0.0.0.0.0.0.0.4.9.2.c.0.0.0.8.9.9.4.1.0.0.2.ip6.arpa"; + } + + @Test + public void getFaultyIpAddresses_should_return_IP_address_when_different_hostname() throws Exception { + String realHostName = "host.name"; + String wrongHostName = "www.yahoo.com"; + doReturn(realHostName).when(ipAddressVerifier).reverseLookUp(ipv4LookupFormat); + doReturn(wrongHostName).when(ipAddressVerifier).reverseLookUp(ipv6LookupFormat); + String[] faultyIpAddresses = ipAddressVerifier.getFaultyIpAddresses(nodeRepoJsonModel); + String[] expectedFaultyIpAddresses = new String[]{ipv6Address}; + assertArrayEquals(expectedFaultyIpAddresses, faultyIpAddresses); + } + + @Test + public void getFaultyIpAddresses_should_return_empty_array_when_all_addresses_point_to_correct_hostname() throws Exception { + String realHostName = "host.name"; + doReturn(realHostName).when(ipAddressVerifier).reverseLookUp(ipv4LookupFormat); + doReturn(realHostName).when(ipAddressVerifier).reverseLookUp(ipv6LookupFormat); + String[] faultyIpAddresses = ipAddressVerifier.getFaultyIpAddresses(nodeRepoJsonModel); + String[] expectedFaultyIpAddresses = new String[]{}; + assertArrayEquals(expectedFaultyIpAddresses, faultyIpAddresses); + } + + @Test + public void convertIpv6ToLookupFormat_should_return_properly_converted_ipv6_address() { + String actualConvertedAddress = ipAddressVerifier.convertIpv6ToLookupFormat(ipv6Address); + assertEquals(ipv6LookupFormat, actualConvertedAddress); + } + + @Test + public void convertIpv4ToLookupFormat_should_return_properly_converted_ipv6_address() { + String actualConvertedAddress = ipAddressVerifier.convertIpv4ToLookupFormat(ipv4Address); + assertEquals(ipv4LookupFormat, actualConvertedAddress); + } + + @Test + public void getFaultyIpAddresses_should_return_empty_array_when_parameters_are_invalid() { + assertEquals(0, ipAddressVerifier.getFaultyIpAddresses(new NodeRepoJsonModel()).length); + } + +} \ No newline at end of file diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/commons/noderepo/NodeJsonConverterTest.java b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/commons/noderepo/NodeJsonConverterTest.java new file mode 100644 index 00000000000..24280e99cfa --- /dev/null +++ b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/commons/noderepo/NodeJsonConverterTest.java @@ -0,0 +1,40 @@ +package com.yahoo.vespa.hosted.node.verification.commons.noderepo; + +import com.yahoo.vespa.hosted.node.verification.spec.retrievers.HardwareInfo; +import org.junit.Test; + +import java.io.File; +import java.net.URL; +import java.util.ArrayList; +import java.util.Arrays; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +/** + * Created by olaa on 07/07/2017. + */ +public class NodeJsonConverterTest { + + private static final double DELTA = 0.1; + + @Test + public void convertJsonModel_should_return_correct_HardwareInfo() throws Exception { + ArrayList urls = new ArrayList<>(Arrays.asList(new File("src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/nodeInfoTest.json").toURI().toURL())); + NodeRepoJsonModel nodeRepoJsonModel = NodeRepoInfoRetriever.retrieve(urls); + HardwareInfo hardwareInfo = NodeJsonConverter.convertJsonModelToHardwareInfo(nodeRepoJsonModel); + double expectedMinDiskAvailable = 500.0; + double expectedMinMainMemoryAvailable = 24.0; + double expectedMinCpuCores = 24.0; + double expectedInterfaceSpeedMbs = 1000; + assertEquals(expectedMinDiskAvailable, hardwareInfo.getMinDiskAvailableGb(), DELTA); + assertEquals(expectedMinMainMemoryAvailable, hardwareInfo.getMinMainMemoryAvailableGb(), DELTA); + assertEquals(expectedMinCpuCores, hardwareInfo.getMinCpuCores(), DELTA); + assertTrue(hardwareInfo.getIpv4Interface()); + assertFalse(hardwareInfo.getIpv6Interface()); + assertEquals(expectedInterfaceSpeedMbs, hardwareInfo.getInterfaceSpeedMbs(), DELTA); + assertEquals(hardwareInfo.getDiskType(), HardwareInfo.DiskType.SLOW); + } + +} \ No newline at end of file diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/commons/noderepo/NodeRepoInfoRetrieverTest.java b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/commons/noderepo/NodeRepoInfoRetrieverTest.java new file mode 100644 index 00000000000..8d7cf4d06f0 --- /dev/null +++ b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/commons/noderepo/NodeRepoInfoRetrieverTest.java @@ -0,0 +1,61 @@ +package com.yahoo.vespa.hosted.node.verification.commons.noderepo; + +import org.junit.Before; +import org.junit.Test; + +import java.io.IOException; +import java.net.MalformedURLException; +import java.net.URL; +import java.nio.file.Paths; +import java.util.ArrayList; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +public class NodeRepoInfoRetrieverTest { + + private NodeRepoInfoRetriever nodeRepoInfoRetriever; + private ArrayList urls; + private static final double DELTA = 0.1; + private static final String ABSOLUTE_PATH = Paths.get(".").toAbsolutePath().normalize().toString(); + private static final String RESOURCE_PATH = "src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources"; + private static final String URL_RESOURCE_PATH = "file://" + ABSOLUTE_PATH + "/" + RESOURCE_PATH; + + @Before + public void setup() { + nodeRepoInfoRetriever = new NodeRepoInfoRetriever(); + urls = new ArrayList<>(); + } + + @Test + public void retrieve_should_return_nodeJSONModel_when_parameter_contains_valid_url() throws IOException { + urls.add(new URL(URL_RESOURCE_PATH + "/nodeInfoTest.json")); + NodeRepoJsonModel nodeRepoJsonModel = NodeRepoInfoRetriever.retrieve(urls); + double expectedMinDiskAvailable = 500.0; + double expectedMinMainMemoryAvailable = 24.0; + double expectedMinCpuCores = 24.0; + assertEquals(expectedMinDiskAvailable, nodeRepoJsonModel.getMinDiskAvailableGb(), DELTA); + assertEquals(expectedMinMainMemoryAvailable, nodeRepoJsonModel.getMinMainMemoryAvailableGb(), DELTA); + assertEquals(expectedMinCpuCores, nodeRepoJsonModel.getMinCpuCores(), DELTA); + } + + @Test + public void retrieve_should_throw_IOException_when_no_valid_URLs() throws MalformedURLException { + urls = new ArrayList<>(); + String exceptionMessage = "Failed to parse JSON from all possible config servers."; + try { + NodeRepoInfoRetriever.retrieve(urls); + fail("Retrieve should have thrown IOException"); + } catch (IOException e) { + assertEquals(exceptionMessage, e.getMessage()); + } + urls.add(new URL("file:///dev/null")); + try { + NodeRepoInfoRetriever.retrieve(urls); + fail("Retrieve should have thrown IOException"); + } catch (IOException e) { + assertEquals(exceptionMessage, e.getMessage()); + } + } + +} \ No newline at end of file diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/BenchmarkResultInspectorTest.java b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/BenchmarkResultInspectorTest.java index 55f5af05771..7314839c438 100644 --- a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/BenchmarkResultInspectorTest.java +++ b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/BenchmarkResultInspectorTest.java @@ -1,7 +1,7 @@ package com.yahoo.vespa.hosted.node.verification.hardware; import com.yahoo.vespa.hosted.node.verification.hardware.benchmarks.BenchmarkResults; -import com.yahoo.vespa.hosted.node.verification.hardware.report.BenchmarkReport; +import com.yahoo.vespa.hosted.node.verification.commons.report.BenchmarkReport; import org.junit.Before; import org.junit.Test; diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/HardwareBenchmarkerTest.java b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/HardwareBenchmarkerTest.java index c976aff54c5..1b1eafd8cd9 100644 --- a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/HardwareBenchmarkerTest.java +++ b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/HardwareBenchmarkerTest.java @@ -4,6 +4,8 @@ import com.yahoo.vespa.hosted.node.verification.mock.MockCommandExecutor; import org.junit.Before; import org.junit.Test; +import java.util.ArrayList; + import static org.junit.Assert.assertTrue; public class HardwareBenchmarkerTest { @@ -22,7 +24,7 @@ public class HardwareBenchmarkerTest { } @Test - public void hardwareBenchmarks_should_return_true() { + public void hardwareBenchmarks_should_return_true() throws Exception { mockCommandExecutor.addCommand("cat " + VALID_DISK_BENCHMARK_PATH); mockCommandExecutor.addCommand("cat " + VALID_CPU_BENCHMARK_PATH); mockCommandExecutor.addDummyCommand(); @@ -31,7 +33,7 @@ public class HardwareBenchmarkerTest { mockCommandExecutor.addCommand("cat " + VALID_MEMORY_READ_BENCHMARK_PATH); mockCommandExecutor.addDummyCommand(); mockCommandExecutor.addDummyCommand(); - assertTrue(HardwareBenchmarker.hardwareBenchmarks(mockCommandExecutor)); + assertTrue(HardwareBenchmarker.hardwareBenchmarks(mockCommandExecutor, new ArrayList<>())); } diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/report/BenchmarkReportTest.java b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/report/BenchmarkReportTest.java index 8b501d4bcff..a9568ed4a3e 100644 --- a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/report/BenchmarkReportTest.java +++ b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/report/BenchmarkReportTest.java @@ -1,7 +1,7 @@ package com.yahoo.vespa.hosted.node.verification.hardware.report; import com.fasterxml.jackson.databind.ObjectMapper; -import com.yahoo.vespa.hosted.node.verification.hardware.benchmarks.BenchmarkResults; +import com.yahoo.vespa.hosted.node.verification.commons.report.BenchmarkReport; import org.junit.Test; import static org.junit.Assert.assertEquals; diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/SpecVerifierTest.java b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/SpecVerifierTest.java index 6d6708f938f..c0e5cc62b6e 100644 --- a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/SpecVerifierTest.java +++ b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/SpecVerifierTest.java @@ -2,10 +2,10 @@ package com.yahoo.vespa.hosted.node.verification.spec; import com.fasterxml.jackson.databind.ObjectMapper; import com.yahoo.vespa.hosted.node.verification.mock.MockCommandExecutor; -import com.yahoo.vespa.hosted.node.verification.spec.noderepo.NodeRepoInfoRetriever; -import com.yahoo.vespa.hosted.node.verification.spec.noderepo.NodeRepoJsonModel; +import com.yahoo.vespa.hosted.node.verification.commons.noderepo.NodeRepoInfoRetriever; +import com.yahoo.vespa.hosted.node.verification.commons.noderepo.NodeRepoJsonModel; import com.yahoo.vespa.hosted.node.verification.spec.retrievers.HardwareInfo; -import com.yahoo.vespa.hosted.node.verification.spec.report.VerificationReport; +import com.yahoo.vespa.hosted.node.verification.commons.report.SpecVerificationReport; import org.junit.Before; import org.junit.Ignore; import org.junit.Test; @@ -14,7 +14,6 @@ import java.io.File; import java.net.URL; import java.nio.file.Paths; import java.util.ArrayList; -import java.util.Arrays; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -92,10 +91,10 @@ public class SpecVerifierTest { actualHardware.setDiskType(HardwareInfo.DiskType.SLOW); nodeInfoUrls.add(new File(NODE_REPO_PATH).toURI().toURL()); NodeRepoJsonModel nodeRepoJsonModel = NodeRepoInfoRetriever.retrieve(nodeInfoUrls); - VerificationReport verificationSpecReport = SpecVerifier.makeVerificationReport(actualHardware, nodeRepoJsonModel); + SpecVerificationReport verificationSpecVerificationReport = SpecVerifier.makeVerificationReport(actualHardware, nodeRepoJsonModel); String expectedJson = "{\"actualInterfaceSpeed\":100.0}"; ObjectMapper om = new ObjectMapper(); - String actualJson = om.writeValueAsString(verificationSpecReport); + String actualJson = om.writeValueAsString(verificationSpecVerificationReport); assertEquals(expectedJson, actualJson); } diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/noderepo/IPAddressVerifierTest.java b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/noderepo/IPAddressVerifierTest.java deleted file mode 100644 index eee2244b6ff..00000000000 --- a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/noderepo/IPAddressVerifierTest.java +++ /dev/null @@ -1,76 +0,0 @@ -package com.yahoo.vespa.hosted.node.verification.spec.noderepo; - -import org.junit.Before; -import org.junit.Test; - -import java.net.URL; -import java.nio.file.Paths; -import java.util.ArrayList; -import java.util.Arrays; - -import static org.junit.Assert.assertArrayEquals; -import static org.junit.Assert.assertEquals; -import static org.mockito.Mockito.doReturn; -import static org.mockito.Mockito.spy; - -public class IPAddressVerifierTest { - - private IPAddressVerifier ipAddressVerifier = spy(new IPAddressVerifier()); - private String ipv4Address; - private String ipv6Address; - private NodeRepoJsonModel nodeRepoJsonModel; - private static final String ABSOLUTE_PATH = Paths.get(".").toAbsolutePath().normalize().toString(); - private static final String RESOURCE_PATH = "src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/IPAddressVerifierTest.json"; - private static final String URL_RESOURCE_PATH = "file://" + ABSOLUTE_PATH + "/" + RESOURCE_PATH; - private String ipv4LookupFormat; - private String ipv6LookupFormat; - - @Before - public void setup() throws Exception { - ipv4Address = "10.213.181.113"; - ipv6Address = "2001:4998:c:2940:0:0:0:111c"; - ArrayList nodeRepoUrl = new ArrayList<>(Arrays.asList(new URL(URL_RESOURCE_PATH))); - nodeRepoJsonModel = NodeRepoInfoRetriever.retrieve(nodeRepoUrl); - ipv4LookupFormat = "113.181.213.10.in-addr.arpa"; - ipv6LookupFormat = "c.1.1.1.0.0.0.0.0.0.0.0.0.0.0.0.0.4.9.2.c.0.0.0.8.9.9.4.1.0.0.2.ip6.arpa"; - } - - @Test - public void getFaultyIpAddresses_should_return_IP_address_when_different_hostname() throws Exception { - String realHostName = "host.name"; - String wrongHostName = "www.yahoo.com"; - doReturn(realHostName).when(ipAddressVerifier).reverseLookUp(ipv4LookupFormat); - doReturn(wrongHostName).when(ipAddressVerifier).reverseLookUp(ipv6LookupFormat); - String[] faultyIpAddresses = ipAddressVerifier.getFaultyIpAddresses(nodeRepoJsonModel); - String[] expectedFaultyIpAddresses = new String[]{ipv6Address}; - assertArrayEquals(expectedFaultyIpAddresses, faultyIpAddresses); - } - - @Test - public void getFaultyIpAddresses_should_return_empty_array_when_all_addresses_point_to_correct_hostname() throws Exception { - String realHostName = "host.name"; - doReturn(realHostName).when(ipAddressVerifier).reverseLookUp(ipv4LookupFormat); - doReturn(realHostName).when(ipAddressVerifier).reverseLookUp(ipv6LookupFormat); - String[] faultyIpAddresses = ipAddressVerifier.getFaultyIpAddresses(nodeRepoJsonModel); - String[] expectedFaultyIpAddresses = new String[]{}; - assertArrayEquals(expectedFaultyIpAddresses, faultyIpAddresses); - } - - @Test - public void convertIpv6ToLookupFormat_should_return_properly_converted_ipv6_address() { - String actualConvertedAddress = ipAddressVerifier.convertIpv6ToLookupFormat(ipv6Address); - assertEquals(ipv6LookupFormat, actualConvertedAddress); - } - - @Test - public void convertIpv4ToLookupFormat_should_return_properly_converted_ipv6_address() { - String actualConvertedAddress = ipAddressVerifier.convertIpv4ToLookupFormat(ipv4Address); - assertEquals(ipv4LookupFormat, actualConvertedAddress); - } - - @Test - public void getFaultyIpAddresses_should_return_empty_array_when_parameters_are_invalid() { - assertEquals(0, ipAddressVerifier.getFaultyIpAddresses(new NodeRepoJsonModel()).length); - } - -} \ No newline at end of file diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/noderepo/NodeJsonConverterTest.java b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/noderepo/NodeJsonConverterTest.java deleted file mode 100644 index 5d76f7fb012..00000000000 --- a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/noderepo/NodeJsonConverterTest.java +++ /dev/null @@ -1,40 +0,0 @@ -package com.yahoo.vespa.hosted.node.verification.spec.noderepo; - -import com.yahoo.vespa.hosted.node.verification.spec.retrievers.HardwareInfo; -import org.junit.Test; - -import java.io.File; -import java.net.URL; -import java.util.ArrayList; -import java.util.Arrays; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; - -/** - * Created by olaa on 07/07/2017. - */ -public class NodeJsonConverterTest { - - private static final double DELTA = 0.1; - - @Test - public void convertJsonModel_should_return_correct_HardwareInfo() throws Exception { - ArrayList urls = new ArrayList<>(Arrays.asList(new File("src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/nodeInfoTest.json").toURI().toURL())); - NodeRepoJsonModel nodeRepoJsonModel = NodeRepoInfoRetriever.retrieve(urls); - HardwareInfo hardwareInfo = NodeJsonConverter.convertJsonModelToHardwareInfo(nodeRepoJsonModel); - double expectedMinDiskAvailable = 500.0; - double expectedMinMainMemoryAvailable = 24.0; - double expectedMinCpuCores = 24.0; - double expectedInterfaceSpeedMbs = 1000; - assertEquals(expectedMinDiskAvailable, hardwareInfo.getMinDiskAvailableGb(), DELTA); - assertEquals(expectedMinMainMemoryAvailable, hardwareInfo.getMinMainMemoryAvailableGb(), DELTA); - assertEquals(expectedMinCpuCores, hardwareInfo.getMinCpuCores(), DELTA); - assertTrue(hardwareInfo.getIpv4Interface()); - assertFalse(hardwareInfo.getIpv6Interface()); - assertEquals(expectedInterfaceSpeedMbs, hardwareInfo.getInterfaceSpeedMbs(), DELTA); - assertEquals(hardwareInfo.getDiskType(), HardwareInfo.DiskType.SLOW); - } - -} \ No newline at end of file diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/noderepo/NodeRepoInfoRetrieverTest.java b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/noderepo/NodeRepoInfoRetrieverTest.java deleted file mode 100644 index 729506fcab0..00000000000 --- a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/noderepo/NodeRepoInfoRetrieverTest.java +++ /dev/null @@ -1,61 +0,0 @@ -package com.yahoo.vespa.hosted.node.verification.spec.noderepo; - -import org.junit.Before; -import org.junit.Test; - -import java.io.IOException; -import java.net.MalformedURLException; -import java.net.URL; -import java.nio.file.Paths; -import java.util.ArrayList; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.fail; - -public class NodeRepoInfoRetrieverTest { - - private NodeRepoInfoRetriever nodeRepoInfoRetriever; - private ArrayList urls; - private static final double DELTA = 0.1; - private static final String ABSOLUTE_PATH = Paths.get(".").toAbsolutePath().normalize().toString(); - private static final String RESOURCE_PATH = "src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources"; - private static final String URL_RESOURCE_PATH = "file://" + ABSOLUTE_PATH + "/" + RESOURCE_PATH; - - @Before - public void setup() { - nodeRepoInfoRetriever = new NodeRepoInfoRetriever(); - urls = new ArrayList<>(); - } - - @Test - public void retrieve_should_return_nodeJSONModel_when_parameter_contains_valid_url() throws IOException { - urls.add(new URL(URL_RESOURCE_PATH + "/nodeInfoTest.json")); - NodeRepoJsonModel nodeRepoJsonModel = NodeRepoInfoRetriever.retrieve(urls); - double expectedMinDiskAvailable = 500.0; - double expectedMinMainMemoryAvailable = 24.0; - double expectedMinCpuCores = 24.0; - assertEquals(expectedMinDiskAvailable, nodeRepoJsonModel.getMinDiskAvailableGb(), DELTA); - assertEquals(expectedMinMainMemoryAvailable, nodeRepoJsonModel.getMinMainMemoryAvailableGb(), DELTA); - assertEquals(expectedMinCpuCores, nodeRepoJsonModel.getMinCpuCores(), DELTA); - } - - @Test - public void retrieve_should_throw_IOException_when_no_valid_URLs() throws MalformedURLException { - urls = new ArrayList<>(); - String exceptionMessage = "Failed to parse JSON from all possible config servers."; - try { - NodeRepoInfoRetriever.retrieve(urls); - fail("Retrieve should have thrown IOException"); - } catch (IOException e) { - assertEquals(exceptionMessage, e.getMessage()); - } - urls.add(new URL("file:///dev/null")); - try { - NodeRepoInfoRetriever.retrieve(urls); - fail("Retrieve should have thrown IOException"); - } catch (IOException e) { - assertEquals(exceptionMessage, e.getMessage()); - } - } - -} \ No newline at end of file diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/report/SpecVerificationReportTest.java b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/report/SpecVerificationReportTest.java new file mode 100644 index 00000000000..e5e7f2ca49a --- /dev/null +++ b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/report/SpecVerificationReportTest.java @@ -0,0 +1,44 @@ +package com.yahoo.vespa.hosted.node.verification.spec.report; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.yahoo.vespa.hosted.node.verification.commons.report.SpecVerificationReport; +import com.yahoo.vespa.hosted.node.verification.mock.MockCommandExecutor; +import com.yahoo.vespa.hosted.node.verification.spec.retrievers.HardwareInfo; +import org.junit.Before; +import org.junit.Test; + +import static org.junit.Assert.*; + +public class SpecVerificationReportTest { + + private SpecVerificationReport specVerificationReport; + private static final String REPORT_PATH = "src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/reportJSON"; + + @Before + public void setup() { + specVerificationReport = new SpecVerificationReport(); + } + + @Test + public void VerificationReport_returns_empty_string_when_all_specs_are_correct() throws Exception { + String expectedJson = "{}"; + ObjectMapper om = new ObjectMapper(); + String actualJson = om.writeValueAsString(specVerificationReport); + assertEquals(expectedJson, actualJson); + } + + @Test + public void Json_is_in_wanted_format_when_all_specs_are_wrong() throws Exception { + specVerificationReport.setActualInterfaceSpeed(100D); + specVerificationReport.setActualDiskSpaceAvailable(500D); + specVerificationReport.setActualDiskType(HardwareInfo.DiskType.FAST); + specVerificationReport.setActualMemoryAvailable(123D); + specVerificationReport.setActualcpuCores(4); + specVerificationReport.setFaultyIpAddresses(new String[]{"2001:4998:44:505d:0:0:0:2618"}); + String expectedJson = MockCommandExecutor.readFromFile(REPORT_PATH).get(0); + ObjectMapper om = new ObjectMapper(); + String actualJson = om.writeValueAsString(specVerificationReport); + assertEquals(expectedJson, actualJson); + } + +} \ No newline at end of file diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/report/VerificationReportTest.java b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/report/VerificationReportTest.java deleted file mode 100644 index 2f7bc75af75..00000000000 --- a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/report/VerificationReportTest.java +++ /dev/null @@ -1,43 +0,0 @@ -package com.yahoo.vespa.hosted.node.verification.spec.report; - -import com.fasterxml.jackson.databind.ObjectMapper; -import com.yahoo.vespa.hosted.node.verification.mock.MockCommandExecutor; -import com.yahoo.vespa.hosted.node.verification.spec.retrievers.HardwareInfo; -import org.junit.Before; -import org.junit.Test; - -import static org.junit.Assert.*; - -public class VerificationReportTest { - - private VerificationReport verificationReport; - private static final String REPORT_PATH = "src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/reportJSON"; - - @Before - public void setup() { - verificationReport = new VerificationReport(); - } - - @Test - public void VerificationReport_returns_empty_string_when_all_specs_are_correct() throws Exception { - String expectedJson = "{}"; - ObjectMapper om = new ObjectMapper(); - String actualJson = om.writeValueAsString(verificationReport); - assertEquals(expectedJson, actualJson); - } - - @Test - public void Json_is_in_wanted_format_when_all_specs_are_wrong() throws Exception { - verificationReport.setActualInterfaceSpeed(100D); - verificationReport.setActualDiskSpaceAvailable(500D); - verificationReport.setActualDiskType(HardwareInfo.DiskType.FAST); - verificationReport.setActualMemoryAvailable(123D); - verificationReport.setActualcpuCores(4); - verificationReport.setFaultyIpAddresses(new String[]{"2001:4998:44:505d:0:0:0:2618"}); - String expectedJson = MockCommandExecutor.readFromFile(REPORT_PATH).get(0); - ObjectMapper om = new ObjectMapper(); - String actualJson = om.writeValueAsString(verificationReport); - assertEquals(expectedJson, actualJson); - } - -} \ No newline at end of file -- cgit v1.2.3