From f04d13f6ade7b7744ad08a262529b76b936a7d9f Mon Sep 17 00:00:00 2001 From: Valerij Fredriksen Date: Wed, 17 Jan 2018 13:28:18 +0100 Subject: Take in nodeSpec through command line arguments --- node-maintainer/pom.xml | 5 ++ .../yahoo/vespa/hosted/node/verification/Main.java | 84 ++++++++++++++++++++++ .../commons/noderepo/IPAddressVerifier.java | 7 +- .../verification/commons/noderepo/NodeSpec.java | 61 ++++------------ .../verification/hardware/HardwareBenchmarker.java | 43 ++++------- .../node/verification/spec/SpecVerifier.java | 79 ++++++++++---------- 6 files changed, 163 insertions(+), 116 deletions(-) create mode 100644 node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/Main.java (limited to 'node-maintainer') diff --git a/node-maintainer/pom.xml b/node-maintainer/pom.xml index 1fe3eb23857..3c4bc1fe7d4 100644 --- a/node-maintainer/pom.xml +++ b/node-maintainer/pom.xml @@ -41,6 +41,11 @@ com.fasterxml.jackson.core jackson-databind + + io.airlift + airline + 0.6 + org.hamcrest diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/Main.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/Main.java new file mode 100644 index 00000000000..a33dd07dff2 --- /dev/null +++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/Main.java @@ -0,0 +1,84 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.verification; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.yahoo.log.LogLevel; +import com.yahoo.log.LogSetup; +import com.yahoo.vespa.hosted.node.verification.commons.CommandExecutor; +import com.yahoo.vespa.hosted.node.verification.commons.report.HardwareDivergenceReport; +import com.yahoo.vespa.hosted.node.verification.hardware.HardwareBenchmarker; +import com.yahoo.vespa.hosted.node.verification.spec.SpecVerifier; +import io.airlift.command.Cli; +import io.airlift.command.Help; +import io.airlift.command.Option; + +import java.io.IOException; +import java.util.logging.Level; +import java.util.logging.Logger; + +/** + * @author freva + */ +public class Main { + private static final String EXECUTABLE_NAME = "node-verifier"; + private static final Logger logger = Logger.getLogger(Main.class.getName()); + private static final ObjectMapper om = new ObjectMapper(); + + public static void main(String[] args) { + LogSetup.initVespaLogging(EXECUTABLE_NAME); + + try { + System.out.println(execute(args, new CommandExecutor())); + } catch (Exception e) { + logger.log(LogLevel.ERROR, "Something went wrong", e); + System.exit(1); + } + } + + @SuppressWarnings("unchecked") + public static String execute(String[] args, CommandExecutor commandExecutor) throws IOException { + Cli.CliBuilder builder = Cli.builder(EXECUTABLE_NAME) + .withDescription("Verifies that node meets the expected specification and benchmarks") + .withDefaultCommand(Help.class) + .withCommands(Help.class, SpecVerifier.class, HardwareBenchmarker.class); + + Object command = builder.build().parse(args); + if (command instanceof VerifierCommand) { + HardwareDivergenceReport report = ((VerifierCommand) command).getPreviousHardwareDivergence(); + ((VerifierCommand) command).run(report, commandExecutor); + return hardwareDivergenceReportToString(report); + } else if (command instanceof Runnable) { + ((Runnable) command).run(); + return ""; + } + + throw new RuntimeException("Unknown command class " + command.getClass().getName()); + } + + public static abstract class VerifierCommand { + @Option(name = {"-h", "--divergence"}, description = "JSON of the previous hardware divergence report") + private String hardwareDivergence; + + private HardwareDivergenceReport getPreviousHardwareDivergence() { + if (hardwareDivergence == null) { + return new HardwareDivergenceReport(); + } + try { + return om.readValue(hardwareDivergence, HardwareDivergenceReport.class); + } catch (IOException e) { + logger.log(Level.WARNING, "Failed to parse hardware divergence:\n" + hardwareDivergence, e.getMessage()); + return new HardwareDivergenceReport(); + } + } + + protected abstract void run(HardwareDivergenceReport hardwareDivergenceReport, CommandExecutor commandExecutor); + } + + private static String hardwareDivergenceReportToString(HardwareDivergenceReport hardwareDivergenceReport) throws IOException { + if (hardwareDivergenceReport.isHardwareDivergenceReportEmpty()) { + return "null"; + } else { + return om.writeValueAsString(hardwareDivergenceReport); + } + } +} diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/noderepo/IPAddressVerifier.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/noderepo/IPAddressVerifier.java index 4d62fe88c1c..001a07ea00e 100644 --- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/noderepo/IPAddressVerifier.java +++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/noderepo/IPAddressVerifier.java @@ -27,6 +27,12 @@ public class IPAddressVerifier { private static final Logger logger = Logger.getLogger(IPAddressVerifier.class.getName()); + private final String expectedHostname; + + public IPAddressVerifier(String expectedHostname) { + this.expectedHostname = expectedHostname; + } + public void reportFaultyIpAddresses(NodeSpec nodeSpec, SpecVerificationReport specVerificationReport) { String[] faultyIpAddresses = getFaultyIpAddresses(nodeSpec); if (faultyIpAddresses.length > 0) { @@ -35,7 +41,6 @@ public class IPAddressVerifier { } public String[] getFaultyIpAddresses(NodeSpec nodeSpec) { - String expectedHostname = nodeSpec.getHostname(); List faultyIpAddresses = new ArrayList<>(); if (expectedHostname == null || expectedHostname.equals("")) return new String[0]; diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/noderepo/NodeSpec.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/noderepo/NodeSpec.java index a2483435e18..68a8060a6ad 100644 --- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/noderepo/NodeSpec.java +++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/noderepo/NodeSpec.java @@ -1,8 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.node.verification.commons.noderepo; -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.net.InetAddresses; import com.yahoo.vespa.hosted.node.verification.spec.retrievers.HardwareInfo; import com.yahoo.vespa.hosted.node.verification.spec.retrievers.HardwareInfo.DiskType; @@ -13,29 +11,25 @@ import java.net.InetAddress; import java.util.stream.Stream; /** - * Object with the information node repositories has about the node. + * Object with the information about a node * - * @author olaaun - * @author sgrostad + * @author freva */ -@JsonIgnoreProperties(ignoreUnknown = true) public class NodeSpec { + private final double minDiskAvailableGb; + private final double minMainMemoryAvailableGb; + private final double minCpuCores; + private final boolean fastDisk; + private final String[] ipAddresses; - @JsonProperty("minDiskAvailableGb") - private double minDiskAvailableGb; - @JsonProperty("minMainMemoryAvailableGb") - private double minMainMemoryAvailableGb; - @JsonProperty("minCpuCores") - private double minCpuCores; - @JsonProperty("fastDisk") - private boolean fastDisk; - @JsonProperty("ipAddresses") - private String[] ipAddresses; - @JsonProperty - private String hostname; - @JsonProperty - private String hardwareDivergence; - private String nodeRepoUrl; + public NodeSpec(double minDiskAvailableGb, double minMainMemoryAvailableGb, double minCpuCores, boolean fastDisk, + String[] ipAddresses) { + this.minDiskAvailableGb = minDiskAvailableGb; + this.minMainMemoryAvailableGb = minMainMemoryAvailableGb; + this.minCpuCores = minCpuCores; + this.fastDisk = fastDisk; + this.ipAddresses = ipAddresses; + } public HardwareInfo copyToHardwareInfo() { HardwareInfo hardwareInfo = new HardwareInfo(); @@ -60,29 +54,4 @@ public class NodeSpec { .filter(ip -> ip instanceof Inet4Address) .findFirst().map(InetAddress::getHostAddress).orElse(null); } - - public double getMinDiskAvailableGb() { - return minDiskAvailableGb; - } - - public double getMinMainMemoryAvailableGb() { - return minMainMemoryAvailableGb; - } - - public double getMinCpuCores() { - return minCpuCores; - } - - public boolean isFastDisk() { - return fastDisk; - } - - public String getHostname() { - return hostname; - } - - public String getHardwareDivergence() { - return hardwareDivergence; - } - } diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/HardwareBenchmarker.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/HardwareBenchmarker.java index 7e151a6a87e..9b3d8f45e1d 100644 --- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/HardwareBenchmarker.java +++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/HardwareBenchmarker.java @@ -1,33 +1,35 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.node.verification.hardware; -import com.yahoo.log.LogSetup; +import com.yahoo.vespa.hosted.node.verification.Main; import com.yahoo.vespa.hosted.node.verification.commons.CommandExecutor; -import com.yahoo.vespa.hosted.node.verification.commons.HostURLGenerator; import com.yahoo.vespa.hosted.node.verification.commons.report.BenchmarkReport; -import com.yahoo.vespa.hosted.node.verification.commons.report.Reporter; +import com.yahoo.vespa.hosted.node.verification.commons.report.HardwareDivergenceReport; import com.yahoo.vespa.hosted.node.verification.hardware.benchmarks.Benchmark; import com.yahoo.vespa.hosted.node.verification.hardware.benchmarks.BenchmarkResults; import com.yahoo.vespa.hosted.node.verification.hardware.benchmarks.CPUBenchmark; import com.yahoo.vespa.hosted.node.verification.hardware.benchmarks.DiskBenchmark; import com.yahoo.vespa.hosted.node.verification.hardware.benchmarks.MemoryBenchmark; +import io.airlift.command.Command; -import java.io.IOException; -import java.net.URL; import java.util.ArrayList; import java.util.Arrays; import java.util.List; -import java.util.logging.Level; -import java.util.logging.Logger; /** * Benchmarks different hardware components and creates report */ -public class HardwareBenchmarker { +@Command(name = "benchmark", description = "Run node benchmarks") +public class HardwareBenchmarker extends Main.VerifierCommand { - private static final Logger logger = Logger.getLogger(HardwareBenchmarker.class.getName()); + @Override + public void run(HardwareDivergenceReport hardwareDivergenceReport, CommandExecutor commandExecutor) { + BenchmarkReport benchmarkReport = hardwareBenchmarks(commandExecutor); - public static boolean hardwareBenchmarks(CommandExecutor commandExecutor, List nodeInfoUrls) throws IOException { + hardwareDivergenceReport.setBenchmarkReport(benchmarkReport); + } + + private BenchmarkReport hardwareBenchmarks(CommandExecutor commandExecutor) { BenchmarkResults benchmarkResults = new BenchmarkResults(); List benchmarks = new ArrayList<>(Arrays.asList( new DiskBenchmark(benchmarkResults, commandExecutor), @@ -36,25 +38,6 @@ public class HardwareBenchmarker { for (Benchmark benchmark : benchmarks) { benchmark.doBenchmark(); } - BenchmarkReport benchmarkReport = BenchmarkResultInspector.makeBenchmarkReport(benchmarkResults); - Reporter.reportBenchmarkResults(benchmarkReport, nodeInfoUrls); - return benchmarkReport.isAllBenchmarksOK(); - } - - public static void main(String[] args) throws IOException { - LogSetup.initVespaLogging("hardware-benchmarker"); - CommandExecutor commandExecutor = new CommandExecutor(); - List nodeInfoUrls; - if (args.length == 0) { - throw new IllegalStateException("Expected config server URL as parameter"); - } - try { - nodeInfoUrls = HostURLGenerator.generateNodeInfoUrl(commandExecutor, args[0]); - HardwareBenchmarker.hardwareBenchmarks(commandExecutor, nodeInfoUrls); - } catch (IOException e) { - logger.log(Level.WARNING, e.getMessage()); - } - + return BenchmarkResultInspector.makeBenchmarkReport(benchmarkResults); } - } diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/SpecVerifier.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/SpecVerifier.java index 0e425413c1c..c07cefb4405 100644 --- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/SpecVerifier.java +++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/SpecVerifier.java @@ -1,23 +1,21 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.node.verification.spec; -import com.yahoo.log.LogSetup; +import com.google.common.base.Strings; +import com.yahoo.vespa.defaults.Defaults; +import com.yahoo.vespa.hosted.node.verification.Main; import com.yahoo.vespa.hosted.node.verification.commons.CommandExecutor; -import com.yahoo.vespa.hosted.node.verification.commons.HostURLGenerator; import com.yahoo.vespa.hosted.node.verification.commons.noderepo.IPAddressVerifier; import com.yahoo.vespa.hosted.node.verification.commons.noderepo.NodeJsonConverter; -import com.yahoo.vespa.hosted.node.verification.commons.noderepo.NodeRepoInfoRetriever; import com.yahoo.vespa.hosted.node.verification.commons.noderepo.NodeSpec; -import com.yahoo.vespa.hosted.node.verification.commons.report.Reporter; +import com.yahoo.vespa.hosted.node.verification.commons.report.HardwareDivergenceReport; import com.yahoo.vespa.hosted.node.verification.commons.report.SpecVerificationReport; import com.yahoo.vespa.hosted.node.verification.spec.retrievers.HardwareInfo; import com.yahoo.vespa.hosted.node.verification.spec.retrievers.HardwareInfoRetriever; +import io.airlift.command.Command; +import io.airlift.command.Option; -import java.io.IOException; -import java.net.URL; -import java.util.List; -import java.util.logging.Level; -import java.util.logging.Logger; +import java.util.Optional; /** * Creates two HardwareInfo objects, one with spec from node repository and one from spec retrieved at the node. @@ -26,44 +24,47 @@ import java.util.logging.Logger; * @author olaaun * @author sgrostad */ -public class SpecVerifier { +@Command(name = "specification", description = "Verify that node's actual hardware and configuration matches the expected") +public class SpecVerifier extends Main.VerifierCommand { - private static final Logger logger = Logger.getLogger(SpecVerifier.class.getName()); + @Option(name = {"-d", "--disk"}, required = true, description = "Expected disk size in GB") + protected double diskAvailableGb; - public static boolean verifySpec(CommandExecutor commandExecutor, List nodeInfoUrls) throws IOException { - NodeSpec nodeSpec = getNodeRepositoryJSON(nodeInfoUrls); + @Option(name = {"-m", "--memory"}, required = true, description = "Expected main memory size in GB") + protected double mainMemoryAvailableGb; + + @Option(name = {"-c", "--cpu_cores"}, required = true, description = "Expected number of CPU cores") + protected double cpuCores; + + @Option(name = {"-s", "--is_ssd"}, required = true, description = "Set to true if disk is SSD", allowedValues = {"true", "false"}) + protected String fastDisk; + + @Option(name = {"-i", "--ips"}, description = "Comma separated list of IP addresses assigned to this node") + protected String ipAddresses; + + @Override + public void run(HardwareDivergenceReport hardwareDivergenceReport, CommandExecutor commandExecutor) { + String[] ips = Optional.ofNullable(ipAddresses) + .filter(s -> !Strings.isNullOrEmpty(s)) + .map(s -> s.split(",")) + .orElse(new String[0]); + + NodeSpec nodeSpec = new NodeSpec(diskAvailableGb, mainMemoryAvailableGb, cpuCores, Boolean.valueOf(fastDisk), ips); + SpecVerificationReport specVerificationReport = verifySpec(nodeSpec, commandExecutor); + + hardwareDivergenceReport.setSpecVerificationReport(specVerificationReport); + } + + private SpecVerificationReport verifySpec(NodeSpec nodeSpec, CommandExecutor commandExecutor) { VerifierSettings verifierSettings = new VerifierSettings(nodeSpec); HardwareInfo actualHardware = HardwareInfoRetriever.retrieve(commandExecutor, verifierSettings); - SpecVerificationReport specVerificationReport = makeVerificationReport(actualHardware, nodeSpec); - Reporter.reportSpecVerificationResults(specVerificationReport, nodeInfoUrls); - return specVerificationReport.isValidSpec(); + return makeVerificationReport(actualHardware, nodeSpec); } - protected static SpecVerificationReport makeVerificationReport(HardwareInfo actualHardware, NodeSpec nodeSpec) { + private static SpecVerificationReport makeVerificationReport(HardwareInfo actualHardware, NodeSpec nodeSpec) { SpecVerificationReport specVerificationReport = HardwareNodeComparator.compare(NodeJsonConverter.convertJsonModelToHardwareInfo(nodeSpec), actualHardware); - IPAddressVerifier ipAddressVerifier = new IPAddressVerifier(); + IPAddressVerifier ipAddressVerifier = new IPAddressVerifier(Defaults.getDefaults().vespaHostname()); ipAddressVerifier.reportFaultyIpAddresses(nodeSpec, specVerificationReport); return specVerificationReport; } - - protected static NodeSpec getNodeRepositoryJSON(List nodeInfoUrls) throws IOException { - NodeSpec nodeSpec = NodeRepoInfoRetriever.retrieve(nodeInfoUrls); - return nodeSpec; - } - - public static void main(String[] args) { - LogSetup.initVespaLogging("spec-verifier"); - CommandExecutor commandExecutor = new CommandExecutor(); - List nodeInfoUrls; - if (args.length == 0) { - throw new IllegalStateException("Expected config server URL as parameter"); - } - try { - nodeInfoUrls = HostURLGenerator.generateNodeInfoUrl(commandExecutor, args[0]); - SpecVerifier.verifySpec(commandExecutor, nodeInfoUrls); - } catch (IOException e) { - logger.log(Level.WARNING, e.getMessage()); - } - } - } -- cgit v1.2.3