diff options
author | Jon Bratseth <bratseth@yahoo-inc.com> | 2017-08-18 16:12:57 +0200 |
---|---|---|
committer | Jon Bratseth <bratseth@yahoo-inc.com> | 2017-08-18 16:12:57 +0200 |
commit | 2d4a99bebf3c4a054556a82e9a5d44bdfc283b4c (patch) | |
tree | 1682ea0e7822ad4a97b03c0842c704d14ce656a9 | |
parent | 9adb3c178a4608ec3d6931ef0a3a0e012927722a (diff) | |
parent | 5c7f956078eea4e70149ca74485380e0391fd1c7 (diff) |
Merge with master
87 files changed, 1506 insertions, 359 deletions
diff --git a/README.md b/README.md index 5988bad6787..ff67eca709b 100644 --- a/README.md +++ b/README.md @@ -13,22 +13,19 @@ Code licensed under the Apache 2.0 license. See [LICENSE](LICENSE) for terms. ## Get started developing -### Set up build environment +### Setup build environment C++ building is supported on CentOS 7. +We recommend using the following environment: [Create C++ dev environment on CentOS using VirtualBox and Vagrant](vagrant/README.md). +You can also setup CentOS 7 natively and install the following build dependencies: -#### Install required build dependencies - sudo yum -y install epel-release centos-release-scl yum-utils sudo yum-config-manager --add-repo https://copr.fedorainfracloud.org/coprs/g/vespa/vespa/repo/epel-7/group_vespa-vespa-epel-7.repo - sudo yum -y --enablerepo=epel-testing install devtoolset-6-gcc-c++ devtoolset-6-libatomic-devel devtoolset-6-binutils \ - Judy-devel cmake3 ccache lz4-devel zlib-devel maven libicu-devel llvm3.9-devel \ - llvm3.9-static java-1.8.0-openjdk-devel openssl-devel rpm-build make \ - vespa-boost-devel vespa-libtorrent-devel vespa-zookeeper-c-client-devel vespa-cppunit-devel -or use the prebuilt docker image - - # TODO: Add docker command + sudo yum -y install epel-release centos-release-scl yum-utils + sudo yum -y install ccache \ + rpm-build + yum-builddep -y <vespa-source>/dist/vespa.spec ### Build Java modules -Java modules can be built on any environment having Java and Maven: +Java modules can be built on any environment having Java 8 and Maven: sh bootstrap.sh mvn install diff --git a/application/pom.xml b/application/pom.xml index 8f301c1a884..3fd9e8054e7 100644 --- a/application/pom.xml +++ b/application/pom.xml @@ -67,7 +67,6 @@ <dependency> <groupId>junit</groupId> <artifactId>junit</artifactId> - <version>4.10</version> <scope>test</scope> </dependency> <dependency> diff --git a/config_test/pom.xml b/config_test/pom.xml index ae909a02603..ad96b5135c3 100644 --- a/config_test/pom.xml +++ b/config_test/pom.xml @@ -11,7 +11,7 @@ <dependency> <groupId>junit</groupId> <artifactId>junit</artifactId> - <version>4.11</version> + <version>4.12</version> <scope>test</scope> </dependency> <dependency> diff --git a/docker/ci/build-and-test.sh b/docker/ci/build-and-test.sh index cfebe09877b..0f0b8be94df 100644 --- a/docker/ci/build-and-test.sh +++ b/docker/ci/build-and-test.sh @@ -1,7 +1,6 @@ #!/bin/bash # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. set -e -set -m if [ $# -ne 4 ]; then echo "Usage: $0 <source dir> <build dir> <log dir> <num threads>" @@ -14,24 +13,10 @@ BUILD_DIR=$2 LOG_DIR=$3 NUM_THREADS=$4 -function bootstrap { - source /opt/rh/devtoolset-6/enable || true - export MAVEN_OPTS="-Xms128m -Xmx512m" - cd "${SOURCE_DIR}" && sh ./bootstrap.sh full - cd "${BUILD_DIR}" && cmake3 \ - -DCMAKE_INSTALL_PREFIX=/opt/vespa \ - -DJAVA_HOME=/usr/lib/jvm/java-openjdk \ - -DEXTRA_LINK_DIRECTORY="/opt/vespa-boost/lib;/opt/vespa-libtorrent/lib;/opt/vespa-zookeeper-c-client/lib;/opt/vespa-cppunit/lib;/usr/lib64/llvm3.9/lib" \ - -DEXTRA_INCLUDE_DIRECTORY="/opt/vespa-boost/include;/opt/vespa-libtorrent/include;/opt/vespa-zookeeper-c-client/include;/opt/vespa-cppunit/include;/usr/include/llvm3.9" \ - -DCMAKE_INSTALL_RPATH="/opt/vespa/lib64;/opt/vespa-boost/lib;/opt/vespa-libtorrent/lib;/opt/vespa-zookeeper-c-client/lib;/opt/vespa-cppunit/lib;/usr/lib/jvm/java-1.8.0/jre/lib/amd64/server;/usr/include/llvm3.9" \ - -DCMAKE_BUILD_RPATH=/opt/vespa/lib64 \ - -DVALGRIND_UNIT_TESTS=no \ - "${SOURCE_DIR}" -} - function build_java { cd "${SOURCE_DIR}" - mvn install -nsu -B -T ${NUM_THREADS} -V # Should ideally split out test phase, but some unit tests fails on 'mvn test' + export MAVEN_OPTS="-Xms128m -Xmx512m" + mvn install -nsu -B -T ${NUM_THREADS} # Should ideally split out test phase, but some unit tests fails on 'mvn test' } function build_cpp { @@ -40,32 +25,6 @@ function build_cpp { ctest3 -j ${NUM_THREADS} } -bootstrap - -pids=() -set -o pipefail - -# Java build and test -# Should be waited for first, because it takes much shorter time than C++ build. -build_java 2>&1 | tee ${LOG_DIR}/java.log & -pids+=($!) - -# C++ build and test -build_cpp 2>&1 | tee ${LOG_DIR}/cpp.log & -pids+=($!) - -EXIT_CODE=0 -for pid in "${pids[@]}"; do - wait $pid || EXIT_CODE=$? - echo "$pid finished with status ${EXIT_CODE}" - if [[ "${EXIT_CODE}" != 0 ]]; then - echo "Exiting and killing remaining jobs." - break; - fi -done -set +o pipefail - -# Kill any remaining jobs, ignoring error when no jobs are running -kill $(jobs -p) 2>/dev/null || true - -exit ${EXIT_CODE} +bash ${SOURCE_DIR}/bootstrap-cpp.sh ${SOURCE_DIR} ${BUILD_DIR} +build_java 2>&1 | tee ${LOG_DIR}/java.log +build_cpp 2>&1 | tee ${LOG_DIR}/cpp.log diff --git a/docker/ci/vespa-ci-internal.sh b/docker/ci/vespa-ci-internal.sh index 8ba77a56a69..5067aecaa6a 100755 --- a/docker/ci/vespa-ci-internal.sh +++ b/docker/ci/vespa-ci-internal.sh @@ -26,5 +26,5 @@ git -c advice.detachedHead=false checkout ${GIT_COMMIT} NUM_THREADS=$(($(nproc --all) * 2)) bash "${MAPPED_DIR}/docker/ci/build-and-test.sh" "${SOURCE_DIR}" "${BUILD_DIR}" "${LOG_DIR}" ${NUM_THREADS} -sudo cp "${LOG_DIR}/java.log" "${MAPPED_DIR}/docker/logs/${LOG_PREFIX}-java.log" -sudo cp "${LOG_DIR}/cpp.log" "${MAPPED_DIR}/docker/logs/${LOG_PREFIX}-cpp.log" +cp "${LOG_DIR}/java.log" "${MAPPED_DIR}/docker/logs/${LOG_PREFIX}-java.log" +cp "${LOG_DIR}/cpp.log" "${MAPPED_DIR}/docker/logs/${LOG_PREFIX}-cpp.log" diff --git a/documentapi/src/main/java/com/yahoo/documentapi/SyncParameters.java b/documentapi/src/main/java/com/yahoo/documentapi/SyncParameters.java index 5a4b4abbf1e..a7bbf1243ae 100755 --- a/documentapi/src/main/java/com/yahoo/documentapi/SyncParameters.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/SyncParameters.java @@ -1,11 +1,42 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.documentapi; +import java.time.temporal.TemporalAmount; +import java.util.Optional; + /** * Parameters for creating a synchronous session * + * @author bjorncs * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> */ public class SyncParameters extends Parameters { - // empty + private final TemporalAmount defaultTimeout; + + public SyncParameters() { + this(null); + } + + private SyncParameters(TemporalAmount defaultTimeout) { + this.defaultTimeout = defaultTimeout; + } + + public Optional<TemporalAmount> defaultTimeout() { + return Optional.ofNullable(defaultTimeout); + } + + public static class Builder { + private TemporalAmount defaultTimeout; + + /** + * Set default timeout for all messagebus operations. + */ + public void setDefaultTimeout(TemporalAmount defaultTimeout) { + this.defaultTimeout = defaultTimeout; + } + + public SyncParameters build() { + return new SyncParameters(defaultTimeout); + } + } } diff --git a/documentapi/src/main/java/com/yahoo/documentapi/SyncSession.java b/documentapi/src/main/java/com/yahoo/documentapi/SyncSession.java index 2a985212449..e04fb7a4e07 100755 --- a/documentapi/src/main/java/com/yahoo/documentapi/SyncSession.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/SyncSession.java @@ -5,16 +5,17 @@ import com.yahoo.document.Document; import com.yahoo.document.DocumentId; import com.yahoo.document.DocumentPut; import com.yahoo.document.DocumentRemove; -import com.yahoo.document.DocumentType; import com.yahoo.document.DocumentUpdate; -import com.yahoo.document.TestAndSetCondition; import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol; +import java.time.temporal.TemporalAmount; + /** * <p>A session for synchronous access to a document repository. This class * provides simple document access where throughput is not a concern.</p> * * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + * @author bjorncs */ public interface SyncSession extends Session { @@ -60,6 +61,41 @@ public interface SyncSession extends Session { Document get(DocumentId id, String fieldSet, DocumentProtocol.Priority priority); /** + * <p>Gets a document with timeout.</p> + * + * @param id The id of the document to get. + * @param timeout Timeout. If timeout is null, an unspecified default will be used. + * @return The known document having this id, or null if there is no + * document having this id. + * @throws UnsupportedOperationException Thrown if this access does not + * support retrieving. + * @throws RuntimeException If operation times out. + */ + // TODO Vespa 7: Remove default implementation. Consider removing get() overloads without timeout. + default Document get(DocumentId id, TemporalAmount timeout) { + return get(id); + } + + /** + * <p>Gets a document with timeout. </p> + * + * @param id The id of the document to get. + * @param fieldSet A comma-separated list of fields to retrieve + * @param priority The priority with which to perform this operation. + * @param timeout Timeout. If timeout is null, an unspecified default will be used. + * @return The known document having this id, or null if there is no + * document having this id. + * @throws UnsupportedOperationException Thrown if this access does not + * support retrieving. + * * @throws RuntimeException If operation times out. + */ + // TODO Vespa 7: Remove default implementation. Consider removing get() overloads without timeout. + default Document get(DocumentId id, String fieldSet, DocumentProtocol.Priority priority, + TemporalAmount timeout) { + return get(id, fieldSet, priority); + } + + /** * <p>Removes a document if it is present and condition is fulfilled.</p> * @param documentRemove document to delete * @return true If the document with this id was removed, false otherwise. diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusSyncSession.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusSyncSession.java index 2d0f1d9db82..f2b1816a410 100755 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusSyncSession.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusSyncSession.java @@ -12,21 +12,32 @@ import com.yahoo.documentapi.Response; import com.yahoo.documentapi.Result; import com.yahoo.documentapi.SyncParameters; import com.yahoo.documentapi.SyncSession; -import com.yahoo.documentapi.messagebus.protocol.*; import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol; +import com.yahoo.documentapi.messagebus.protocol.GetDocumentMessage; +import com.yahoo.documentapi.messagebus.protocol.GetDocumentReply; +import com.yahoo.documentapi.messagebus.protocol.PutDocumentMessage; +import com.yahoo.documentapi.messagebus.protocol.RemoveDocumentMessage; +import com.yahoo.documentapi.messagebus.protocol.RemoveDocumentReply; +import com.yahoo.documentapi.messagebus.protocol.UpdateDocumentMessage; +import com.yahoo.documentapi.messagebus.protocol.UpdateDocumentReply; import com.yahoo.messagebus.Message; import com.yahoo.messagebus.MessageBus; import com.yahoo.messagebus.Reply; import com.yahoo.messagebus.ReplyHandler; +import java.time.temporal.ChronoUnit; +import java.time.temporal.TemporalAmount; + /** * An implementation of the SyncSession interface running over message bus. * * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + * @author bjorncs */ public class MessageBusSyncSession implements MessageBusSession, SyncSession, ReplyHandler { - private MessageBusAsyncSession session; + private final MessageBusAsyncSession session; + private final TemporalAmount defaultTimeout; /** * Creates a new sync session running on message bus logic. @@ -37,6 +48,7 @@ public class MessageBusSyncSession implements MessageBusSession, SyncSession, Re */ MessageBusSyncSession(SyncParameters syncParams, MessageBus bus, MessageBusParams mbusParams) { session = new MessageBusAsyncSession(new AsyncParameters(), bus, mbusParams, this); + defaultTimeout = syncParams.defaultTimeout().orElse(null); } @Override @@ -72,6 +84,13 @@ public class MessageBusSyncSession implements MessageBusSession, SyncSession, Re * @return The reply received. */ public Reply syncSend(Message msg) { + return syncSend(msg, defaultTimeout); + } + + private Reply syncSend(Message msg, TemporalAmount timeout) { + if (timeout != null) { + msg.setTimeRemaining(timeout.get(ChronoUnit.MILLIS)); + } try { RequestMonitor monitor = new RequestMonitor(); msg.setContext(monitor); @@ -107,15 +126,26 @@ public class MessageBusSyncSession implements MessageBusSession, SyncSession, Re @Override public Document get(DocumentId id) { - return get(id, "[all]", DocumentProtocol.Priority.NORMAL_1); + return get(id, null); } @Override public Document get(DocumentId id, String fieldSet, DocumentProtocol.Priority pri) { + return get(id, fieldSet, pri, null); + } + + @Override + public Document get(DocumentId id, TemporalAmount timeout) { + return get(id, "[all]", DocumentProtocol.Priority.NORMAL_1, timeout); + } + + @Override + public Document get(DocumentId id, String fieldSet, DocumentProtocol.Priority pri, + TemporalAmount timeout) { GetDocumentMessage msg = new GetDocumentMessage(id, fieldSet); msg.setPriority(pri); - Reply reply = syncSend(msg); + Reply reply = syncSend(msg, timeout != null ? timeout : defaultTimeout); if (reply.hasErrors()) { throw new DocumentAccessException(MessageBusAsyncSession.getErrorMessage(reply)); } diff --git a/node-maintainer/README.md b/node-maintainer/README.md index 7743df2b0c2..4a32f207fdf 100644 --- a/node-maintainer/README.md +++ b/node-maintainer/README.md @@ -16,4 +16,5 @@ SpecVerifier: HardwareBenchmarker: - sudo java -cp node-maintainer-jar-with-dependencies.jar com.yahoo.vespa.hosted.node.verification.hardware.HardwareBenchmarker cfg.1.hostname,cfg.2.hostname,... -Both programs have README explaining closer what it does.
\ No newline at end of file +In "verification" you can find a TODO file where we have listed things we did not have time to implement. +Both programs have README explaining closer what it does. diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/maintainer/FileHelper.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/maintainer/FileHelper.java index 2274baf632e..db84f6af496 100644 --- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/maintainer/FileHelper.java +++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/maintainer/FileHelper.java @@ -20,7 +20,6 @@ import java.util.stream.Collectors; /** * @author freva */ - public class FileHelper { private static final Logger logger = Logger.getLogger(FileHelper.class.getSimpleName()); diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/TODO.md b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/TODO.md new file mode 100644 index 00000000000..846c52ff645 --- /dev/null +++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/TODO.md @@ -0,0 +1,30 @@ +#TODO +Here we have listed things we did not have time to do during the summer, but that we think can be implemented later. + +##Spec +- The net interface speed is at the moment approved if it is over 1 000MB/s. Since some nodes are supposed to have +10 000MB/s a feature could be to either add information about interface speed in node repo or use the flavor to decide +if the interface speed is correct or not. +- In HardwareNodeComparator the spec found in node repo and on the node are compared. We set a threshold on 5%, meaning +if a the value from the node is more than 5% away from what is said in node repo (+-), then we say it is a bad value. +If this threshold is too high or too low, it has to be changed in this class. + +##Benchmark +- BenchmarkResultInspector is the class that decides whether a benchmark result is ok or if the result should be + reported. The values that decides this are not given very much thought and it could be an idea to check these. +- Benchmark is not running on docker hosts since there is not yet found a solution to only start benchmarking at reboot. +Spec verification runs every hour on node-admin, but since node-admin reboots too often, benchmarking is at the moment +not running here. + +##Reporting / Node repo +- Since HardwareDivergenceReport is printed as a json string and then reported to noderepo as a string, a feature could +be to not store HardwareDivergence as a string in node repo, as it is now, but as a json. This will be mostly changes +outside of this code, but then the member variable HardwareDivergence in NodeRepoJsonModel have to be change from string +to HardwareDivergenceReport which will cause some work in the Reporter class. +- No actions are now taken if there is anything wrong with the node. The information is only uploaded to node repo. +Here it is room for improvements and some possible solutions are: + - Continue only reporting to node repo, but have a program that scans through all nodes and makes a list of those + with errors in HardwareDivergence. + - In addition to uploading the report to node repo, have actions based on what kind of errors and what kind of state + the node has. + - Automatically create Jira tickets when something is reported wrong on a node.
\ No newline at end of file diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/CommandExecutor.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/CommandExecutor.java index f783af4c273..21e54c8c010 100644 --- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/CommandExecutor.java +++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/CommandExecutor.java @@ -10,16 +10,19 @@ import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.StringReader; import java.util.ArrayList; +import java.util.List; /** - * Created by olaa on 03/07/2017. * Wrapper for executing terminal commands + * + * @author olaaun + * @author sgrostad */ public class CommandExecutor { - public ArrayList<String> executeCommand(String command) throws IOException { + public List<String> executeCommand(String command) throws IOException { ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); - ArrayList<String> results = new ArrayList<>(); + List<String> results = new ArrayList<>(); writeToOutputStream(outputStream, command); writeOutputStreamToResults(outputStream, results); return results; @@ -35,7 +38,7 @@ public class CommandExecutor { executor.execute(cmdLine); } - private void writeOutputStreamToResults(ByteArrayOutputStream outputStream, ArrayList<String> results) throws IOException { + private void writeOutputStreamToResults(ByteArrayOutputStream outputStream, List<String> results) throws IOException { String out = outputStream.toString(); BufferedReader br = new BufferedReader(new StringReader(out)); String line; diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/HostURLGenerator.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/HostURLGenerator.java index 6a6a1b4157a..7e8b198428f 100644 --- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/HostURLGenerator.java +++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/HostURLGenerator.java @@ -1,16 +1,17 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.node.verification.commons; -import com.yahoo.vespa.hosted.node.verification.commons.CommandExecutor; - import java.io.IOException; import java.net.MalformedURLException; import java.net.URL; import java.util.ArrayList; +import java.util.List; /** - * Created by olaa on 14/07/2017. * Makes the URL used to retrieve the JSON from the node repository with information about the node's spec. + * + * @author olaaun + * @author sgrostad */ public class HostURLGenerator { @@ -20,8 +21,8 @@ public class HostURLGenerator { private static final String PARSE_ALL_HOSTNAMES_REGEX = ","; private static final String PROTOCOL_REGEX = "^(https?|file)://.*$"; - public static ArrayList<URL> generateNodeInfoUrl(CommandExecutor commandExecutor, String commaSeparatedUrls) throws IOException { - ArrayList<URL> nodeInfoUrls = new ArrayList<>(); + public static List<URL> generateNodeInfoUrl(CommandExecutor commandExecutor, String commaSeparatedUrls) throws IOException { + List<URL> nodeInfoUrls = new ArrayList<>(); String[] configServerHostNames = commaSeparatedUrls.split(PARSE_ALL_HOSTNAMES_REGEX); String nodeHostName = generateNodeHostName(commandExecutor); for (String configServerHostName : configServerHostNames) { @@ -36,7 +37,7 @@ public class HostURLGenerator { } protected static String getEnvironmentVariable(CommandExecutor commandExecutor) throws IOException { - ArrayList<String> output = commandExecutor.executeCommand("hostname"); + List<String> output = commandExecutor.executeCommand("hostname"); if (output.size() == 1) { return output.get(0); } diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/noderepo/IPAddressVerifier.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/noderepo/IPAddressVerifier.java index 483c07002dd..db4eb6e73cd 100644 --- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/noderepo/IPAddressVerifier.java +++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/noderepo/IPAddressVerifier.java @@ -12,14 +12,17 @@ import javax.naming.directory.InitialDirContext; import java.util.ArrayList; import java.util.Enumeration; import java.util.Hashtable; +import java.util.List; import java.util.logging.Level; import java.util.logging.Logger; /** * Created by olaa on 14/07/2017. * Verifies that the IP addresses of a node points to the correct hostname + * + * @author olaaun + * @author sgrostad */ - public class IPAddressVerifier { private static final Logger logger = Logger.getLogger(IPAddressVerifier.class.getName()); @@ -33,7 +36,7 @@ public class IPAddressVerifier { public String[] getFaultyIpAddresses(NodeRepoJsonModel jsonModel) { String expectedHostname = jsonModel.getHostname(); - ArrayList<String> faultyIpAddresses = new ArrayList<>(); + List<String> faultyIpAddresses = new ArrayList<>(); if (expectedHostname == null || expectedHostname.equals("")) return new String[0]; if (!isValidIpv4(jsonModel.getIpv4Address(), expectedHostname)) { diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/noderepo/NodeJsonConverter.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/noderepo/NodeJsonConverter.java index ef6d706a1a4..5cf4bede3f1 100644 --- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/noderepo/NodeJsonConverter.java +++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/noderepo/NodeJsonConverter.java @@ -4,8 +4,10 @@ package com.yahoo.vespa.hosted.node.verification.commons.noderepo; import com.yahoo.vespa.hosted.node.verification.spec.retrievers.HardwareInfo; /** - * Created by olaa on 07/07/2017. * Converts a NodeRepoJsonModel object to a HardwareInfo object. + * + * @author olaaun + * @author sgrostad */ public class NodeJsonConverter { diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/noderepo/NodeRepoInfoRetriever.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/noderepo/NodeRepoInfoRetriever.java index 4d261334d35..57506e44961 100644 --- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/noderepo/NodeRepoInfoRetriever.java +++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/noderepo/NodeRepoInfoRetriever.java @@ -6,18 +6,21 @@ import com.fasterxml.jackson.databind.ObjectMapper; import java.io.IOException; import java.net.URL; import java.util.ArrayList; +import java.util.List; import java.util.logging.Level; import java.util.logging.Logger; /** - * Created by olaa on 04/07/2017. * Parse JSON from node repository and stores information as a NodeRepoJsonModel object. + * + * @author olaaun + * @author sgrostad */ public class NodeRepoInfoRetriever { private static final Logger logger = Logger.getLogger(NodeRepoInfoRetriever.class.getName()); - public static NodeRepoJsonModel retrieve(ArrayList<URL> nodeInfoUrls) throws IOException { + public static NodeRepoJsonModel retrieve(List<URL> nodeInfoUrls) throws IOException { NodeRepoJsonModel nodeRepoJsonModel; ObjectMapper objectMapper = new ObjectMapper(); for (URL nodeInfoURL : nodeInfoUrls) { diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/noderepo/NodeRepoJsonModel.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/noderepo/NodeRepoJsonModel.java index 043cccb681c..5d7ff0df8d1 100644 --- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/noderepo/NodeRepoJsonModel.java +++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/noderepo/NodeRepoJsonModel.java @@ -13,12 +13,14 @@ import java.net.InetAddress; import java.util.stream.Stream; /** - * Created by olaa on 05/07/2017. * Object with the information node repositories has about the node. + * + * @author olaaun + * @author sgrostad */ - @JsonIgnoreProperties(ignoreUnknown = true) public class NodeRepoJsonModel { + @JsonProperty("minDiskAvailableGb") private double minDiskAvailableGb; @JsonProperty("minMainMemoryAvailableGb") diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/parser/OutputParser.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/parser/OutputParser.java index 88e1d22cc0e..5da5abd4ad2 100644 --- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/parser/OutputParser.java +++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/parser/OutputParser.java @@ -2,19 +2,22 @@ package com.yahoo.vespa.hosted.node.verification.commons.parser; import java.util.ArrayList; +import java.util.List; import java.util.regex.Pattern; /** - * Created by sgrostad on 17/07/2017. * Parses terminal command output, and returns results based on ParseInstructions + * + * @author sgrostad + * @author olaaaun */ public class OutputParser { - public static ArrayList<ParseResult> parseOutput(ParseInstructions parseInstructions, ArrayList<String> commandOutput) { - ArrayList<ParseResult> results = new ArrayList<>(); + public static List<ParseResult> parseOutput(ParseInstructions parseInstructions, List<String> commandOutput) { + List<ParseResult> results = new ArrayList<>(); int searchElementIndex = parseInstructions.getSearchElementIndex(); int valueElementIndex = parseInstructions.getValueElementIndex(); - ArrayList<String> searchWords = parseInstructions.getSearchWords(); + List<String> searchWords = parseInstructions.getSearchWords(); for (String line : commandOutput) { String[] lineSplit = line.trim().split(parseInstructions.getSplitRegex()); if (lineSplit.length <= Math.max(searchElementIndex, valueElementIndex)) { @@ -30,15 +33,15 @@ public class OutputParser { return results; } - public static ParseResult parseSingleOutput(ParseInstructions parseInstructions, ArrayList<String> commandOutput) { - ArrayList<ParseResult> parseResults = parseOutput(parseInstructions, commandOutput); + public static ParseResult parseSingleOutput(ParseInstructions parseInstructions, List<String> commandOutput) { + List<ParseResult> parseResults = parseOutput(parseInstructions, commandOutput); if (parseResults.size() == 0) { return new ParseResult("invalid", "invalid"); } return parseResults.get(0); } - private static boolean matchingSearchWord(ArrayList<String> searchWords, String searchWordCandidate) { + private static boolean matchingSearchWord(List<String> searchWords, String searchWordCandidate) { return searchWords.stream().anyMatch(w -> Pattern.compile(w).matcher(searchWordCandidate).matches()); } diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/parser/ParseInstructions.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/parser/ParseInstructions.java index 6fb962c8232..262bfcba0b9 100644 --- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/parser/ParseInstructions.java +++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/parser/ParseInstructions.java @@ -1,19 +1,22 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.node.verification.commons.parser; -import java.util.ArrayList; +import java.util.List; /** - * Created by sgrostad on 17/07/2017. + * Contains instructions of how a command line output should be parsed + * + * @author sgrostad + * @author olaaaun */ public class ParseInstructions { private final int searchElementIndex; private final int valueElementIndex; private final String splitRegex; - private final ArrayList<String> searchWords; + private final List<String> searchWords; - public ParseInstructions(int searchElementIndex, int returnElementNum, String splitRegex, ArrayList<String> searchWords) { + public ParseInstructions(int searchElementIndex, int returnElementNum, String splitRegex, List<String> searchWords) { this.searchElementIndex = searchElementIndex; this.valueElementIndex = returnElementNum; this.splitRegex = splitRegex; @@ -32,7 +35,7 @@ public class ParseInstructions { return splitRegex; } - public ArrayList<String> getSearchWords() { + public List<String> getSearchWords() { return searchWords; } diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/parser/ParseResult.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/parser/ParseResult.java index df16897066b..a89cfd89f31 100644 --- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/parser/ParseResult.java +++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/parser/ParseResult.java @@ -4,7 +4,10 @@ package com.yahoo.vespa.hosted.node.verification.commons.parser; import java.util.Objects; /** - * Created by sgrostad on 17/07/2017. + * Contains the results from parsing a command line output + * + * @author sgrostad + * @author olaaun */ public class ParseResult { diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/report/BenchmarkReport.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/report/BenchmarkReport.java index 0cfe3c86b92..ce4e3685a25 100644 --- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/report/BenchmarkReport.java +++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/report/BenchmarkReport.java @@ -8,8 +8,10 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; /** - * Created by sgrostad on 12/07/2017. * JSON-mapped class for reporting benchmark results to node repo + * + * @author sgrostad + * @author olaaun */ @JsonInclude(JsonInclude.Include.NON_NULL) public class BenchmarkReport { @@ -62,7 +64,7 @@ public class BenchmarkReport { try { String jsonReport = om.writeValueAsString(this); return jsonReport.length() == 2; - } catch (JsonProcessingException e){ + } catch (JsonProcessingException e) { e.printStackTrace(); return false; } diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/report/HardwareDivergenceReport.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/report/HardwareDivergenceReport.java index 4536649ae25..11830c24cef 100644 --- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/report/HardwareDivergenceReport.java +++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/report/HardwareDivergenceReport.java @@ -5,6 +5,12 @@ import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; +/** + * JSON-wrapped report for node repo + * + * @author sgrostad + * @author olaaun + */ @JsonInclude(JsonInclude.Include.NON_NULL) public class HardwareDivergenceReport { @@ -15,10 +21,9 @@ public class HardwareDivergenceReport { BenchmarkReport benchmarkReport; public void setSpecVerificationReport(SpecVerificationReport specVerificationReport) { - if (specVerificationReport.isValidSpec()){ + if (specVerificationReport.isValidSpec()) { this.specVerificationReport = null; - } - else { + } else { this.specVerificationReport = specVerificationReport; } } @@ -26,15 +31,14 @@ public class HardwareDivergenceReport { public void setBenchmarkReport(BenchmarkReport benchmarkReport) { if (benchmarkReport.isAllBenchmarksOK()) { this.benchmarkReport = null; - } - else { + } else { this.benchmarkReport = benchmarkReport; } } @JsonIgnore - public boolean isHardwareDivergenceReportEmpty(){ - if (specVerificationReport == null && benchmarkReport == null){ + public boolean isHardwareDivergenceReportEmpty() { + if (specVerificationReport == null && benchmarkReport == null) { return true; } return false; diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/report/ReportSender.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/report/Reporter.java index 45a4b8cacb8..11835cab14b 100644 --- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/report/ReportSender.java +++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/report/Reporter.java @@ -2,45 +2,50 @@ package com.yahoo.vespa.hosted.node.verification.commons.report; import com.fasterxml.jackson.databind.ObjectMapper; - import com.yahoo.vespa.hosted.node.verification.commons.noderepo.NodeRepoInfoRetriever; import com.yahoo.vespa.hosted.node.verification.commons.noderepo.NodeRepoJsonModel; import java.io.IOException; import java.net.URL; import java.util.ArrayList; +import java.util.List; import java.util.logging.Level; import java.util.logging.Logger; -public class ReportSender { +/** + * Responsible for printing hardware divergence report to standard out + * + * @author sgrostad + * @author olaaun + */ +public class Reporter { - private static final Logger logger = Logger.getLogger(ReportSender.class.getName()); + private static final Logger logger = Logger.getLogger(Reporter.class.getName()); private static void printHardwareDivergenceReport(HardwareDivergenceReport hardwareDivergenceReport) throws IOException { ObjectMapper om = new ObjectMapper(); String report; - if (hardwareDivergenceReport.isHardwareDivergenceReportEmpty()){ + if (hardwareDivergenceReport.isHardwareDivergenceReportEmpty()) { report = "null"; - } - else { + } else { report = om.writeValueAsString(hardwareDivergenceReport); } System.out.print(report); } - public static void reportBenchmarkResults(BenchmarkReport benchmarkReport, ArrayList<URL> nodeInfoUrls) throws IOException { + public static void reportBenchmarkResults(BenchmarkReport benchmarkReport, List<URL> nodeInfoUrls) throws IOException { HardwareDivergenceReport hardwareDivergenceReport = generateHardwareDivergenceReport(nodeInfoUrls); hardwareDivergenceReport.setBenchmarkReport(benchmarkReport); printHardwareDivergenceReport(hardwareDivergenceReport); } - public static void reportSpecVerificationResults(SpecVerificationReport specVerificationReport, ArrayList<URL> nodeInfoUrls) throws IOException { + public static void reportSpecVerificationResults(SpecVerificationReport specVerificationReport, List<URL> nodeInfoUrls) throws IOException { HardwareDivergenceReport hardwareDivergenceReport = generateHardwareDivergenceReport(nodeInfoUrls); hardwareDivergenceReport.setSpecVerificationReport(specVerificationReport); printHardwareDivergenceReport(hardwareDivergenceReport); } - private static HardwareDivergenceReport generateHardwareDivergenceReport(ArrayList<URL> nodeInfoUrls) throws IOException { + private static HardwareDivergenceReport generateHardwareDivergenceReport(List<URL> nodeInfoUrls) throws IOException { NodeRepoJsonModel nodeRepoJsonModel = NodeRepoInfoRetriever.retrieve(nodeInfoUrls); ObjectMapper om = new ObjectMapper(); if (nodeRepoJsonModel.getHardwareDivergence() == null || nodeRepoJsonModel.getHardwareDivergence().equals("null")) { @@ -49,10 +54,10 @@ public class ReportSender { try { HardwareDivergenceReport hardwareDivergenceReport = om.readValue(nodeRepoJsonModel.getHardwareDivergence(), HardwareDivergenceReport.class); return hardwareDivergenceReport; - } - catch (IOException e){ + } catch (IOException e) { logger.log(Level.WARNING, "Failed to parse hardware divergence report from node repo. Report:\n" + nodeRepoJsonModel.getHardwareDivergence(), e.getMessage()); return new HardwareDivergenceReport(); } } + } diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/report/SpecVerificationReport.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/report/SpecVerificationReport.java index 3eca8f38e54..ef99208a7ce 100644 --- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/report/SpecVerificationReport.java +++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/report/SpecVerificationReport.java @@ -13,6 +13,9 @@ import com.yahoo.vespa.hosted.node.verification.spec.retrievers.HardwareInfo; * Stores results of caomparing node repo spec and actual hardware info. * In case of divergent values, set the corresponding attribute to the actual hardware info value. * Attributes of equal value remain null. + * + * @author sgrostad + * @author olaaun */ @JsonInclude(JsonInclude.Include.NON_NULL) public class SpecVerificationReport { @@ -66,8 +69,7 @@ public class SpecVerificationReport { try { String jsonReport = om.writeValueAsString(this); return jsonReport.length() == 2; - } - catch (JsonProcessingException e){ + } catch (JsonProcessingException e) { e.printStackTrace(); return false; } diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/BenchmarkResultInspector.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/BenchmarkResultInspector.java index 2cd73b46cee..4989c574ed4 100644 --- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/BenchmarkResultInspector.java +++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/BenchmarkResultInspector.java @@ -1,9 +1,15 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.node.verification.hardware; -import com.yahoo.vespa.hosted.node.verification.hardware.benchmarks.BenchmarkResults; import com.yahoo.vespa.hosted.node.verification.commons.report.BenchmarkReport; +import com.yahoo.vespa.hosted.node.verification.hardware.benchmarks.BenchmarkResults; +/** + * Responsible for checking the benchmarks results, and adding unreasonable results to BenchmarkReport + * + * @author sgrostad + * @author olaaun + */ public class BenchmarkResultInspector { private static final double CPU_FREQUENCY_LOWER_LIMIT = 0.5; diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/HardwareBenchmarker.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/HardwareBenchmarker.java index e589ace6988..7e151a6a87e 100644 --- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/HardwareBenchmarker.java +++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/HardwareBenchmarker.java @@ -4,18 +4,19 @@ package com.yahoo.vespa.hosted.node.verification.hardware; import com.yahoo.log.LogSetup; import com.yahoo.vespa.hosted.node.verification.commons.CommandExecutor; import com.yahoo.vespa.hosted.node.verification.commons.HostURLGenerator; -import com.yahoo.vespa.hosted.node.verification.commons.report.ReportSender; +import com.yahoo.vespa.hosted.node.verification.commons.report.BenchmarkReport; +import com.yahoo.vespa.hosted.node.verification.commons.report.Reporter; import com.yahoo.vespa.hosted.node.verification.hardware.benchmarks.Benchmark; import com.yahoo.vespa.hosted.node.verification.hardware.benchmarks.BenchmarkResults; import com.yahoo.vespa.hosted.node.verification.hardware.benchmarks.CPUBenchmark; import com.yahoo.vespa.hosted.node.verification.hardware.benchmarks.DiskBenchmark; import com.yahoo.vespa.hosted.node.verification.hardware.benchmarks.MemoryBenchmark; -import com.yahoo.vespa.hosted.node.verification.commons.report.BenchmarkReport; import java.io.IOException; import java.net.URL; import java.util.ArrayList; import java.util.Arrays; +import java.util.List; import java.util.logging.Level; import java.util.logging.Logger; @@ -26,9 +27,9 @@ public class HardwareBenchmarker { private static final Logger logger = Logger.getLogger(HardwareBenchmarker.class.getName()); - public static boolean hardwareBenchmarks(CommandExecutor commandExecutor, ArrayList<URL> nodeInfoUrls) throws IOException { + public static boolean hardwareBenchmarks(CommandExecutor commandExecutor, List<URL> nodeInfoUrls) throws IOException { BenchmarkResults benchmarkResults = new BenchmarkResults(); - ArrayList<Benchmark> benchmarks = new ArrayList<>(Arrays.asList( + List<Benchmark> benchmarks = new ArrayList<>(Arrays.asList( new DiskBenchmark(benchmarkResults, commandExecutor), new CPUBenchmark(benchmarkResults, commandExecutor), new MemoryBenchmark(benchmarkResults, commandExecutor))); @@ -36,14 +37,14 @@ public class HardwareBenchmarker { benchmark.doBenchmark(); } BenchmarkReport benchmarkReport = BenchmarkResultInspector.makeBenchmarkReport(benchmarkResults); - ReportSender.reportBenchmarkResults(benchmarkReport, nodeInfoUrls); + Reporter.reportBenchmarkResults(benchmarkReport, nodeInfoUrls); return benchmarkReport.isAllBenchmarksOK(); } public static void main(String[] args) throws IOException { LogSetup.initVespaLogging("hardware-benchmarker"); CommandExecutor commandExecutor = new CommandExecutor(); - ArrayList<URL> nodeInfoUrls; + List<URL> nodeInfoUrls; if (args.length == 0) { throw new IllegalStateException("Expected config server URL as parameter"); } diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/README.md b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/README.md index e9fa4901814..ff580db7f73 100644 --- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/README.md +++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/README.md @@ -3,6 +3,11 @@ Verification of behaviour and performance of hardware. Benchmarks cpu frequency, A report is sent to the node repository if any of the results are below an accepted threshold. ## Code Walkthrough -The main class, HardwareBenchmarker, calls every benchmark in the benchmark package. The results of these benchmarks are passed through -the BenchmarkResultInspector, which creates a BenchmarkReport containing the values below the accepted threshold. ReportSender is then called such that -the node repository is updated with the new benchmark results.
\ No newline at end of file +The main class, HardwareBenchmarker, calls every benchmark in the benchmark package. + +The results of these benchmarks are passed through +the BenchmarkResultInspector, which creates a BenchmarkReport containing the values below the accepted threshold. + +ReportSender is then called such that +the old HardwareDivergence report is retrieved from node repo and updated with the new results. ReportSender prints the new HardwareReport such that it +can be updated in node repo by chef or other.
\ No newline at end of file diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/Benchmark.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/Benchmark.java index c1791ef45bd..1e5b512f492 100644 --- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/Benchmark.java +++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/Benchmark.java @@ -2,10 +2,14 @@ package com.yahoo.vespa.hosted.node.verification.hardware.benchmarks; /** - * Created by sgrostad on 11/07/2017. + * @author sgrostad + * @author olaaun */ public interface Benchmark { + /** + * Should perform benchmark for some part of the hardware, and store the result in BenchmarkResults instance passed to class + */ void doBenchmark(); } diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/BenchmarkResults.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/BenchmarkResults.java index 999af15da09..dfa436460e1 100644 --- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/BenchmarkResults.java +++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/BenchmarkResults.java @@ -2,9 +2,12 @@ package com.yahoo.vespa.hosted.node.verification.hardware.benchmarks; /** - * Created by sgrostad on 11/07/2017. * Stores results from benchmarks + * + * @author sgrostad + * @author olaaun */ +// TODO: This should be immutable public class BenchmarkResults { private double cpuCyclesPerSec; @@ -12,7 +15,6 @@ public class BenchmarkResults { private double memoryWriteSpeedGBs; private double memoryReadSpeedGBs; - public double getMemoryWriteSpeedGBs() { return memoryWriteSpeedGBs; } diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/CPUBenchmark.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/CPUBenchmark.java index e70385bb3a2..a55a92ba8a3 100644 --- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/CPUBenchmark.java +++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/CPUBenchmark.java @@ -9,11 +9,15 @@ import com.yahoo.vespa.hosted.node.verification.commons.parser.ParseResult; import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; +import java.util.List; import java.util.logging.Level; import java.util.logging.Logger; /** - * Created by sgrostad on 11/07/2017. + * Responsible for benchmarking CPU clock frequency, and storing the result in a BenchmarkResults instance + * + * @author sgrostad + * @author olaaun */ public class CPUBenchmark implements Benchmark { @@ -33,31 +37,32 @@ public class CPUBenchmark implements Benchmark { this.commandExecutor = commandExecutor; } + @Override public void doBenchmark() { try { - ArrayList<String> commandOutput = commandExecutor.executeCommand(CPU_BENCHMARK_COMMAND); - ArrayList<ParseResult> parseResults = parseCpuCyclesPerSec(commandOutput); + List<String> commandOutput = commandExecutor.executeCommand(CPU_BENCHMARK_COMMAND); + List<ParseResult> parseResults = parseCpuCyclesPerSec(commandOutput); setCpuCyclesPerSec(parseResults); } catch (IOException e) { logger.log(Level.WARNING, "Failed to perform CPU benchmark", e); } } - protected ArrayList<ParseResult> parseCpuCyclesPerSec(ArrayList<String> commandOutput) { - ArrayList<String> searchWords = new ArrayList<>(Arrays.asList(CYCLES_SEARCH_WORD, SECONDS_SEARCH_WORD)); + protected List<ParseResult> parseCpuCyclesPerSec(List<String> commandOutput) { + List<String> searchWords = new ArrayList<>(Arrays.asList(CYCLES_SEARCH_WORD, SECONDS_SEARCH_WORD)); ParseInstructions parseInstructions = new ParseInstructions(SEARCH_ELEMENT_INDEX, RETURN_ELEMENT_INDEX, SPLIT_REGEX_STRING, searchWords); return OutputParser.parseOutput(parseInstructions, commandOutput); } - protected void setCpuCyclesPerSec(ArrayList<ParseResult> parseResults) { + protected void setCpuCyclesPerSec(List<ParseResult> parseResults) { double cpuCyclesPerSec = getCyclesPerSecond(parseResults); if (cpuCyclesPerSec > 0) { benchmarkResults.setCpuCyclesPerSec(cpuCyclesPerSec); } } - protected double getCyclesPerSecond(ArrayList<ParseResult> parseResults) { + protected double getCyclesPerSecond(List<ParseResult> parseResults) { double cycles = -1; double seconds = -1; for (ParseResult parseResult : parseResults) { diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/DiskBenchmark.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/DiskBenchmark.java index ff8e1f21612..2a4c8d9b694 100644 --- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/DiskBenchmark.java +++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/DiskBenchmark.java @@ -9,12 +9,16 @@ import com.yahoo.vespa.hosted.node.verification.commons.parser.ParseResult; import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; +import java.util.List; import java.util.logging.Level; import java.util.logging.Logger; /** - * Created by olaa on 10/07/2017. + * Responsible for benchmarking disk write speed, and storing the result in a BenchmarkResults instance + * + * @author olaaun + * @author sgrostad */ public class DiskBenchmark implements Benchmark { @@ -34,9 +38,10 @@ public class DiskBenchmark implements Benchmark { this.commandExecutor = commandExecutor; } + @Override public void doBenchmark() { try { - ArrayList<String> commandOutput = commandExecutor.executeCommand(DISK_BENCHMARK_COMMAND); + List<String> commandOutput = commandExecutor.executeCommand(DISK_BENCHMARK_COMMAND); ParseResult parseResult = parseDiskSpeed(commandOutput); setDiskSpeed(parseResult); } catch (IOException e) { @@ -44,8 +49,8 @@ public class DiskBenchmark implements Benchmark { } } - protected ParseResult parseDiskSpeed(ArrayList<String> commandOutput) { - ArrayList<String> searchWords = new ArrayList<>(Arrays.asList(KILO_BYTE_SEARCH_WORD, MEGA_BYTE_SEARCH_WORD, GIGA_BYTE_SEARCH_WORD)); + protected ParseResult parseDiskSpeed(List<String> commandOutput) { + List<String> searchWords = new ArrayList<>(Arrays.asList(KILO_BYTE_SEARCH_WORD, MEGA_BYTE_SEARCH_WORD, GIGA_BYTE_SEARCH_WORD)); ParseInstructions parseInstructions = new ParseInstructions(SEARCH_ELEMENT_INDEX, RETURN_ELEMENT_INDEX, SPLIT_REGEX_STRING, searchWords); return OutputParser.parseSingleOutput(parseInstructions, commandOutput); } diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/MemoryBenchmark.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/MemoryBenchmark.java index 0aa4bc28eb1..b701d9a02f8 100644 --- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/MemoryBenchmark.java +++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/MemoryBenchmark.java @@ -9,11 +9,15 @@ import com.yahoo.vespa.hosted.node.verification.commons.parser.ParseResult; import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; +import java.util.List; import java.util.logging.Level; import java.util.logging.Logger; /** - * Created by sgrostad on 11/07/2017. + * Responsible for benchmarking memory read/write speed, and storing the result in a BenchmarkResults instance + * + * @author sgrostad + * @author olaaun */ public class MemoryBenchmark implements Benchmark { @@ -36,10 +40,11 @@ public class MemoryBenchmark implements Benchmark { this.commandExecutor = commandExecutor; } + @Override public void doBenchmark() { try { setupMountPoint(); - ArrayList<String> commandOutput = commandExecutor.executeCommand(MEM_BENCHMARK_WRITE_SPEED); + List<String> commandOutput = commandExecutor.executeCommand(MEM_BENCHMARK_WRITE_SPEED); ParseResult parseResult = parseMemorySpeed(commandOutput); updateMemoryWriteSpeed(parseResult.getValue()); commandOutput = commandExecutor.executeCommand(MEM_BENCHMARK_READ_SPEED); @@ -70,8 +75,8 @@ public class MemoryBenchmark implements Benchmark { } } - protected ParseResult parseMemorySpeed(ArrayList<String> commandOutput) { - ArrayList<String> searchWords = new ArrayList<>(Arrays.asList(READ_AND_WRITE_SEARCH_WORD)); + protected ParseResult parseMemorySpeed(List<String> commandOutput) { + List<String> searchWords = new ArrayList<>(Arrays.asList(READ_AND_WRITE_SEARCH_WORD)); ParseInstructions parseInstructions = new ParseInstructions(SEARCH_ELEMENT_INDEX, RETURN_ELEMENT_INDEX, SPLIT_REGEX_STRING, searchWords); return OutputParser.parseSingleOutput(parseInstructions, commandOutput); } diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/HardwareNodeComparator.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/HardwareNodeComparator.java index 460fd31db18..88bd0539ccd 100644 --- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/HardwareNodeComparator.java +++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/HardwareNodeComparator.java @@ -6,8 +6,10 @@ import com.yahoo.vespa.hosted.node.verification.spec.retrievers.HardwareInfo; import com.yahoo.vespa.hosted.node.verification.spec.retrievers.HardwareInfo.DiskType; /** - * Created by olaa on 04/07/2017. * Compares two HardwareInfo objects and stores divergent values in a SpecVerificationReport + * + * @author olaaun + * @author sgrostad */ public class HardwareNodeComparator { @@ -74,7 +76,7 @@ public class HardwareNodeComparator { } } - private static boolean outsideThreshold(double value1, double value2 , double thresholdPercentage) { + private static boolean outsideThreshold(double value1, double value2, double thresholdPercentage) { double lowerThresholdPercentage = 1 - thresholdPercentage; double upperThresholdPercentage = 1 + thresholdPercentage; return value1 < lowerThresholdPercentage * value2 || value1 > upperThresholdPercentage * value2; diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/README.md b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/README.md index ff87c7358f1..26856e078eb 100644 --- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/README.md +++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/README.md @@ -10,5 +10,5 @@ It then retrieves all the hardware information at the node with the "retrievers" "HardwareInfo" object. SpecVerifier then uses HardwareNodeComparator to compare spec from node repo and the node itself. It generates a -SpecVerificationReport and uses ReportSender in "commons" to generate a full json report that can update node repo -with the new results. +SpecVerificationReport and uses Reporter in "commons" to retrieve the old HardwareDivergence report from node repo and update it. Reporter then prints the new +updated HardwareDivergence report such that it can be updated in node repo by chef or other. diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/SpecVerifier.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/SpecVerifier.java index 1c96e685aab..e4cae4d7e9e 100644 --- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/SpecVerifier.java +++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/SpecVerifier.java @@ -4,11 +4,11 @@ package com.yahoo.vespa.hosted.node.verification.spec; import com.yahoo.log.LogSetup; import com.yahoo.vespa.hosted.node.verification.commons.CommandExecutor; import com.yahoo.vespa.hosted.node.verification.commons.HostURLGenerator; -import com.yahoo.vespa.hosted.node.verification.commons.report.ReportSender; import com.yahoo.vespa.hosted.node.verification.commons.noderepo.IPAddressVerifier; import com.yahoo.vespa.hosted.node.verification.commons.noderepo.NodeJsonConverter; import com.yahoo.vespa.hosted.node.verification.commons.noderepo.NodeRepoInfoRetriever; import com.yahoo.vespa.hosted.node.verification.commons.noderepo.NodeRepoJsonModel; +import com.yahoo.vespa.hosted.node.verification.commons.report.Reporter; import com.yahoo.vespa.hosted.node.verification.commons.report.SpecVerificationReport; import com.yahoo.vespa.hosted.node.verification.spec.retrievers.HardwareInfo; import com.yahoo.vespa.hosted.node.verification.spec.retrievers.HardwareInfoRetriever; @@ -16,25 +16,27 @@ import com.yahoo.vespa.hosted.node.verification.spec.retrievers.HardwareInfoRetr import java.io.IOException; import java.net.URL; import java.util.ArrayList; +import java.util.List; import java.util.logging.Level; import java.util.logging.Logger; /** - * Created by olaa on 14/07/2017. * Creates two HardwareInfo objects, one with spec from node repository and one from spec retrieved at the node. * Compares the objects and returns the result. + * + * @author olaaun + * @author sgrostad */ public class SpecVerifier { private static final Logger logger = Logger.getLogger(SpecVerifier.class.getName()); - private static final String VIRTUAL_ENVIRONMENT = "VIRTUAL_MACHINE"; - public static boolean verifySpec(CommandExecutor commandExecutor, ArrayList<URL> nodeInfoUrls) throws IOException { + public static boolean verifySpec(CommandExecutor commandExecutor, List<URL> nodeInfoUrls) throws IOException { NodeRepoJsonModel nodeRepoJsonModel = getNodeRepositoryJSON(nodeInfoUrls); VerifierSettings verifierSettings = new VerifierSettings(nodeRepoJsonModel); HardwareInfo actualHardware = HardwareInfoRetriever.retrieve(commandExecutor, verifierSettings); SpecVerificationReport specVerificationReport = makeVerificationReport(actualHardware, nodeRepoJsonModel); - ReportSender.reportSpecVerificationResults(specVerificationReport, nodeInfoUrls); + Reporter.reportSpecVerificationResults(specVerificationReport, nodeInfoUrls); return specVerificationReport.isValidSpec(); } @@ -45,15 +47,15 @@ public class SpecVerifier { return specVerificationReport; } - protected static NodeRepoJsonModel getNodeRepositoryJSON(ArrayList<URL> nodeInfoUrls) throws IOException { + protected static NodeRepoJsonModel getNodeRepositoryJSON(List<URL> nodeInfoUrls) throws IOException { NodeRepoJsonModel nodeRepoJsonModel = NodeRepoInfoRetriever.retrieve(nodeInfoUrls); return nodeRepoJsonModel; } - public static void main(String[] args) { + public static void main(String[] args) { LogSetup.initVespaLogging("spec-verifier"); CommandExecutor commandExecutor = new CommandExecutor(); - ArrayList<URL> nodeInfoUrls; + List<URL> nodeInfoUrls; if (args.length == 0) { throw new IllegalStateException("Expected config server URL as parameter"); } diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/VerifierSettings.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/VerifierSettings.java index ba045e67766..2b5a19ce818 100644 --- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/VerifierSettings.java +++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/VerifierSettings.java @@ -4,19 +4,20 @@ package com.yahoo.vespa.hosted.node.verification.spec; import com.yahoo.vespa.hosted.node.verification.commons.noderepo.NodeRepoJsonModel; /** - * Created by sgrostad on 07/08/2017. * Contains information on what spec should be verified or not. + * + * @author sgrostad + * @author olaaun */ - public class VerifierSettings { private final boolean checkIPv6; - public VerifierSettings(){ + public VerifierSettings() { this.checkIPv6 = true; } - public VerifierSettings(NodeRepoJsonModel nodeRepoJsonModel){ + public VerifierSettings(NodeRepoJsonModel nodeRepoJsonModel) { checkIPv6 = nodeRepoJsonModel.getIpv6Address() != null; } diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/CPURetriever.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/CPURetriever.java index b02f37533da..b8ae5d89b55 100644 --- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/CPURetriever.java +++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/CPURetriever.java @@ -9,11 +9,15 @@ import com.yahoo.vespa.hosted.node.verification.commons.parser.ParseResult; import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; +import java.util.List; import java.util.logging.Level; import java.util.logging.Logger; /** - * Created by olaa on 30/06/2017. + * Retrieves number of CPU cores, and stores the result in a HardwareInfo instance + * + * @author olaaun + * @author sgrostad */ public class CPURetriever implements HardwareRetriever { @@ -31,28 +35,29 @@ public class CPURetriever implements HardwareRetriever { this.commandExecutor = commandExecutor; } + @Override public void updateInfo() { try { - ArrayList<String> commandOutput = commandExecutor.executeCommand(CPU_INFO_COMMAND); - ArrayList<ParseResult> parseResults = parseCPUInfoFile(commandOutput); + List<String> commandOutput = commandExecutor.executeCommand(CPU_INFO_COMMAND); + List<ParseResult> parseResults = parseCPUInfoFile(commandOutput); setCpuCores(parseResults); } catch (IOException e) { logger.log(Level.WARNING, "Failed to retrieve CPU info", e); } } - protected ArrayList<ParseResult> parseCPUInfoFile(ArrayList<String> commandOutput) { - ArrayList<String> searchWords = new ArrayList<>(Arrays.asList(SEARCH_WORD)); + protected List<ParseResult> parseCPUInfoFile(List<String> commandOutput) { + List<String> searchWords = new ArrayList<>(Arrays.asList(SEARCH_WORD)); ParseInstructions parseInstructions = new ParseInstructions(SEARCH_ELEMENT_INDEX, RETURN_ELEMENT_INDEX, REGEX_SPLIT, searchWords); - ArrayList<ParseResult> parseResults = OutputParser.parseOutput(parseInstructions, commandOutput); + List<ParseResult> parseResults = OutputParser.parseOutput(parseInstructions, commandOutput); return parseResults; } - protected void setCpuCores(ArrayList<ParseResult> parseResults) { + protected void setCpuCores(List<ParseResult> parseResults) { hardwareInfo.setMinCpuCores(countCpuCores(parseResults)); } - protected int countCpuCores(ArrayList<ParseResult> parseResults) { + protected int countCpuCores(List<ParseResult> parseResults) { return parseResults.size(); } diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/DiskRetriever.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/DiskRetriever.java index 58d4810299b..8058c9438d7 100644 --- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/DiskRetriever.java +++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/DiskRetriever.java @@ -10,13 +10,18 @@ import com.yahoo.vespa.hosted.node.verification.spec.retrievers.HardwareInfo.Dis import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; +import java.util.List; import java.util.logging.Level; import java.util.logging.Logger; /** - * Created by olaa on 30/06/2017. + * Retrieves disk space and type, and stores the result in a HardwareInfo instance + * + * @author olaaun + * @author sgrostad */ public class DiskRetriever implements HardwareRetriever { + private static final String DISK_CHECK_TYPE = "lsblk -d -o name,rota"; private static final String DISK_CHECK_SIZE = "pvdisplay --units G | grep 'PV Size'"; private static final String DISK_NAME = "sda"; @@ -37,6 +42,7 @@ public class DiskRetriever implements HardwareRetriever { this.commandExecutor = commandExecutor; } + @Override public void updateInfo() { updateDiskType(); updateDiskSize(); @@ -44,7 +50,7 @@ public class DiskRetriever implements HardwareRetriever { protected void updateDiskType() { try { - ArrayList<String> commandOutput = commandExecutor.executeCommand(DISK_CHECK_TYPE); + List<String> commandOutput = commandExecutor.executeCommand(DISK_CHECK_TYPE); ParseResult parseResult = parseDiskType(commandOutput); setDiskType(parseResult); } catch (IOException e) { @@ -54,16 +60,16 @@ public class DiskRetriever implements HardwareRetriever { protected void updateDiskSize() { try { - ArrayList<String> commandOutput = commandExecutor.executeCommand(DISK_CHECK_SIZE); - ArrayList<ParseResult> parseResult = parseDiskSize(commandOutput); + List<String> commandOutput = commandExecutor.executeCommand(DISK_CHECK_SIZE); + List<ParseResult> parseResult = parseDiskSize(commandOutput); setDiskSize(parseResult); } catch (IOException e) { logger.log(Level.WARNING, "Failed to retrieve disk size", e); } } - protected ParseResult parseDiskType(ArrayList<String> commandOutput) throws IOException { - ArrayList<String> searchWords = new ArrayList<>(Arrays.asList(DISK_NAME)); + protected ParseResult parseDiskType(List<String> commandOutput) throws IOException { + List<String> searchWords = new ArrayList<>(Arrays.asList(DISK_NAME)); ParseInstructions parseInstructions = new ParseInstructions(DISK_TYPE_SEARCH_ELEMENT_INDEX, DISK_TYPE_RETURN_ELEMENT_INDEX, DISK_TYPE_REGEX_SPLIT, searchWords); ParseResult parseResult = OutputParser.parseSingleOutput(parseInstructions, commandOutput); if (!parseResult.getSearchWord().equals(DISK_NAME)) { @@ -83,13 +89,13 @@ public class DiskRetriever implements HardwareRetriever { } } - protected ArrayList<ParseResult> parseDiskSize(ArrayList<String> commandOutput) { - ArrayList<String> searchWords = new ArrayList<>(Arrays.asList(DISK_SIZE_SEARCH_WORD)); + protected List<ParseResult> parseDiskSize(List<String> commandOutput) { + List<String> searchWords = new ArrayList<>(Arrays.asList(DISK_SIZE_SEARCH_WORD)); ParseInstructions parseInstructions = new ParseInstructions(DISK_SIZE_SEARCH_ELEMENT_INDEX, DISK_SIZE_RETURN_ELEMENT_INDEX, DISK_SIZE_REGEX_SPLIT, searchWords); return OutputParser.parseOutput(parseInstructions, commandOutput); } - protected void setDiskSize(ArrayList<ParseResult> parseResults) { + protected void setDiskSize(List<ParseResult> parseResults) { double diskSize = 0; try { for (ParseResult parseResult : parseResults) { diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/HardwareInfo.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/HardwareInfo.java index 8d726c507ba..1c17d73523b 100644 --- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/HardwareInfo.java +++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/HardwareInfo.java @@ -2,10 +2,12 @@ package com.yahoo.vespa.hosted.node.verification.spec.retrievers; /** - * Created by olaa on 04/07/2017. * All information the different retrievers retrieve is stored as a HardwareInfo object. + * + * @author olaaun + * @author sgrostad */ - +// TODO: This should be immutable public class HardwareInfo { private double minDiskAvailableGb; @@ -17,7 +19,6 @@ public class HardwareInfo { private double interfaceSpeedMbs; private DiskType diskType; - public double getInterfaceSpeedMbs() { return interfaceSpeedMbs; } diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/HardwareInfoRetriever.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/HardwareInfoRetriever.java index e3235590169..ba29f07baeb 100644 --- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/HardwareInfoRetriever.java +++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/HardwareInfoRetriever.java @@ -5,16 +5,19 @@ import com.yahoo.vespa.hosted.node.verification.commons.CommandExecutor; import com.yahoo.vespa.hosted.node.verification.spec.VerifierSettings; import java.util.ArrayList; +import java.util.List; /** - * Created by olaa on 30/06/2017. * Makes a HardwareInfo object and calls all the retrievers for this object. + * + * @author olaaun + * @author sgrostad */ public class HardwareInfoRetriever { public static HardwareInfo retrieve(CommandExecutor commandExecutor, VerifierSettings verifierSettings) { HardwareInfo hardwareInfo = new HardwareInfo(); - ArrayList<HardwareRetriever> infoList = new ArrayList<>(); + List<HardwareRetriever> infoList = new ArrayList<>(); infoList.add(new CPURetriever(hardwareInfo, commandExecutor)); infoList.add(new MemoryRetriever(hardwareInfo, commandExecutor)); infoList.add(new DiskRetriever(hardwareInfo, commandExecutor)); diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/HardwareRetriever.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/HardwareRetriever.java index 90e00560e59..d77342db982 100644 --- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/HardwareRetriever.java +++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/HardwareRetriever.java @@ -2,10 +2,14 @@ package com.yahoo.vespa.hosted.node.verification.spec.retrievers; /** - * Created by olaa on 30/06/2017. + * @author olaaun + * @author sgrostad */ public interface HardwareRetriever { + /** + * Should retrieve spec from some part of the hardware, and store the result in hardwareinfo instance passed to class + */ void updateInfo(); } diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/MemoryRetriever.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/MemoryRetriever.java index 2a64e0f4b21..c6655e13797 100644 --- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/MemoryRetriever.java +++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/MemoryRetriever.java @@ -9,11 +9,15 @@ import com.yahoo.vespa.hosted.node.verification.commons.parser.ParseResult; import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; +import java.util.List; import java.util.logging.Level; import java.util.logging.Logger; /** - * Created by olaa on 30/06/2017. + * Retrieves memory size, and stores the result in a HardwareInfo instance + * + * @author olaaun + * @author sgrostad */ public class MemoryRetriever implements HardwareRetriever { @@ -31,10 +35,10 @@ public class MemoryRetriever implements HardwareRetriever { this.commandExecutor = commandExecutor; } - + @Override public void updateInfo() { try { - ArrayList<String> commandOutput = commandExecutor.executeCommand(MEMORY_INFO_COMMAND); + List<String> commandOutput = commandExecutor.executeCommand(MEMORY_INFO_COMMAND); ParseResult parseResult = parseMemInfoFile(commandOutput); updateMemoryInfo(parseResult); } catch (IOException e) { @@ -42,8 +46,8 @@ public class MemoryRetriever implements HardwareRetriever { } } - protected ParseResult parseMemInfoFile(ArrayList<String> commandOutput) throws IOException { - ArrayList<String> searchWords = new ArrayList<>(Arrays.asList(SEARCH_WORD)); + protected ParseResult parseMemInfoFile(List<String> commandOutput) throws IOException { + List<String> searchWords = new ArrayList<>(Arrays.asList(SEARCH_WORD)); ParseInstructions parseInstructions = new ParseInstructions(SEARCH_ELEMENT_INDEX, RETURN_ELEMENT_INDEX, REGEX_SPLIT, searchWords); ParseResult parseResult = OutputParser.parseSingleOutput(parseInstructions, commandOutput); if (!parseResult.getSearchWord().matches(SEARCH_WORD)) { diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/NetRetriever.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/NetRetriever.java index 746279915bf..f30d455f339 100644 --- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/NetRetriever.java +++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/NetRetriever.java @@ -11,11 +11,16 @@ import org.apache.commons.exec.ExecuteException; import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; +import java.util.List; import java.util.logging.Level; import java.util.logging.Logger; /** - * Created by olaa on 30/06/2017. + * Retrieves IPv4/IPv6 interface, and checks interface speed. If node should have IPv6, tries to ping6. + * The results are stored in a HardwareInfo instance + * + * @author olaaun + * @author sgrostad */ public class NetRetriever implements HardwareRetriever { @@ -48,8 +53,9 @@ public class NetRetriever implements HardwareRetriever { this.verifierSettings = verifierSettings; } + @Override public void updateInfo() { - ArrayList<ParseResult> parseResults = findInterface(); + List<ParseResult> parseResults = findInterface(); findInterfaceSpeed(parseResults); if (verifierSettings.isCheckIPv6()) { testPingResponse(parseResults); @@ -57,10 +63,10 @@ public class NetRetriever implements HardwareRetriever { updateHardwareInfoWithNet(parseResults); } - protected ArrayList<ParseResult> findInterface() { - ArrayList<ParseResult> parseResults = new ArrayList<>(); + protected List<ParseResult> findInterface() { + List<ParseResult> parseResults = new ArrayList<>(); try { - ArrayList<String> commandOutput = commandExecutor.executeCommand(NET_FIND_INTERFACE); + List<String> commandOutput = commandExecutor.executeCommand(NET_FIND_INTERFACE); parseResults = parseNetInterface(commandOutput); } catch (IOException e) { @@ -69,16 +75,16 @@ public class NetRetriever implements HardwareRetriever { return parseResults; } - protected ArrayList<ParseResult> parseNetInterface(ArrayList<String> commandOutput) { - ArrayList<String> searchWords = new ArrayList<>(Arrays.asList(SEARCH_WORD_INTERFACE_IP4, SEARCH_WORD_INTERFACE_IPV6)); + protected List<ParseResult> parseNetInterface(List<String> commandOutput) { + List<String> searchWords = new ArrayList<>(Arrays.asList(SEARCH_WORD_INTERFACE_IP4, SEARCH_WORD_INTERFACE_IPV6)); ParseInstructions parseInstructions = new ParseInstructions(INTERFACE_SEARCH_ELEMENT_INDEX, INTERFACE_RETURN_ELEMENT_INDEX, INTERFACE_NAME_REGEX_SPLIT, searchWords); - ArrayList<ParseResult> parseResults = OutputParser.parseOutput(parseInstructions, commandOutput); + List<ParseResult> parseResults = OutputParser.parseOutput(parseInstructions, commandOutput); return parseResults; } - protected void findInterfaceSpeed(ArrayList<ParseResult> parseResults) { + protected void findInterfaceSpeed(List<ParseResult> parseResults) { try { - ArrayList<String> commandOutput = commandExecutor.executeCommand(NET_CHECK_INTERFACE_SPEED); + List<String> commandOutput = commandExecutor.executeCommand(NET_CHECK_INTERFACE_SPEED); ParseResult parseResult = parseInterfaceSpeed(commandOutput); parseResults.add(parseResult); } catch (IOException e) { @@ -86,8 +92,8 @@ public class NetRetriever implements HardwareRetriever { } } - protected ParseResult parseInterfaceSpeed(ArrayList<String> commandOutput) throws IOException { - ArrayList<String> searchWords = new ArrayList<>(Arrays.asList(SEARCH_WORD_INTERFACE_SPEED)); + protected ParseResult parseInterfaceSpeed(List<String> commandOutput) throws IOException { + List<String> searchWords = new ArrayList<>(Arrays.asList(SEARCH_WORD_INTERFACE_SPEED)); ParseInstructions parseInstructions = new ParseInstructions(INTERFACE_SPEED_SEARCH_ELEMENT_INDEX, INTERFACE_SPEED_RETURN_ELEMENT_INDEX, INTERFACE_SPEED_REGEX_SPLIT, searchWords); ParseResult parseResult = OutputParser.parseSingleOutput(parseInstructions, commandOutput); if (!parseResult.getSearchWord().matches(SEARCH_WORD_INTERFACE_SPEED)) { @@ -96,9 +102,9 @@ public class NetRetriever implements HardwareRetriever { return parseResult; } - protected void testPingResponse(ArrayList<ParseResult> parseResults) { + protected void testPingResponse(List<ParseResult> parseResults) { try { - ArrayList<String> commandOutput = commandExecutor.executeCommand(PING_NET_COMMAND); + List<String> commandOutput = commandExecutor.executeCommand(PING_NET_COMMAND); parseResults.add(parsePingResponse(commandOutput)); } catch (ExecuteException e) { logger.log(Level.WARNING, "Failed to execute ping6", e); @@ -107,8 +113,8 @@ public class NetRetriever implements HardwareRetriever { } } - protected ParseResult parsePingResponse(ArrayList<String> commandOutput) throws IOException { - ArrayList<String> searchWords = new ArrayList<>(Arrays.asList(PING_SEARCH_WORD)); + protected ParseResult parsePingResponse(List<String> commandOutput) throws IOException { + List<String> searchWords = new ArrayList<>(Arrays.asList(PING_SEARCH_WORD)); ParseInstructions parseInstructions = new ParseInstructions(PING_SEARCH_ELEMENT_INDEX, PING_RETURN_ELEMENT_INDEX, PING_SPLIT_REGEX_STRING, searchWords); ParseResult parseResult = OutputParser.parseSingleOutput(parseInstructions, commandOutput); if (!parseResult.getSearchWord().matches(PING_SEARCH_WORD)) { @@ -117,7 +123,7 @@ public class NetRetriever implements HardwareRetriever { return new ParseResult(PING_SEARCH_WORD, parseResult.getValue()); } - protected void updateHardwareInfoWithNet(ArrayList<ParseResult> parseResults) { + protected void updateHardwareInfoWithNet(List<ParseResult> parseResults) { hardwareInfo.setIpv6Interface(false); hardwareInfo.setIpv4Interface(false); for (ParseResult parseResult : parseResults) { diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/commons/CommandExecutorTest.java b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/commons/CommandExecutorTest.java index ce4156b5960..5f5aca825c0 100644 --- a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/commons/CommandExecutorTest.java +++ b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/commons/CommandExecutorTest.java @@ -5,15 +5,16 @@ import org.junit.Before; import org.junit.Test; import java.io.IOException; -import java.util.ArrayList; import java.util.List; import static java.util.Arrays.asList; import static org.junit.Assert.assertEquals; /** - * Created by sgrostad on 12/07/2017. + * @author sgrostad + * @author olaaun */ + public class CommandExecutorTest { private CommandExecutor commandExecutor; @@ -26,7 +27,7 @@ public class CommandExecutorTest { @Test public void test_if_executeAString_reads_testReadFile_correct() throws IOException { String command = "cat src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/resources/testReadFile.txt"; - ArrayList<String> commandOutput = commandExecutor.executeCommand(command); + List<String> commandOutput = commandExecutor.executeCommand(command); List<String> expectedOutput = asList("This test file tests apache commons exec", "Second line"); assertEquals(expectedOutput, commandOutput); } diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/commons/HostURLGeneratorTest.java b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/commons/HostURLGeneratorTest.java index 1519e4e52fa..773172de807 100644 --- a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/commons/HostURLGeneratorTest.java +++ b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/commons/HostURLGeneratorTest.java @@ -7,15 +7,17 @@ import org.junit.Test; import java.io.IOException; import java.net.URL; -import java.util.ArrayList; -import java.util.stream.Collectors; +import java.util.List; + import static org.junit.Assert.assertEquals; import static org.junit.Assert.fail; /** - * Created by olaa on 14/07/2017. + * @author sgrostad + * @author olaaun */ + public class HostURLGeneratorTest { private MockCommandExecutor mockCommandExecutor; @@ -35,7 +37,7 @@ public class HostURLGeneratorTest { @Test public void generateNodeInfoUrl_find_config_server_test_if_url_is_formatted_correctly() throws Exception { mockCommandExecutor.addCommand(CAT_NODE_HOST_NAME_PATH); - ArrayList<URL> urls = HostURLGenerator.generateNodeInfoUrl(mockCommandExecutor, CONFIG_SERVER_HOSTNAME_1 + "," + CONFIG_SERVER_HOSTNAME_2); + List<URL> urls = HostURLGenerator.generateNodeInfoUrl(mockCommandExecutor, CONFIG_SERVER_HOSTNAME_1 + "," + CONFIG_SERVER_HOSTNAME_2); String expectedUrl1 = CONFIG_SERVER_HOSTNAME_1 + NODE_HOSTNAME_PREFIX + EXPECTED_HOSTNAME; String expectedUrl2 = CONFIG_SERVER_HOSTNAME_2 + NODE_HOSTNAME_PREFIX + EXPECTED_HOSTNAME; assertEquals(expectedUrl1, urls.get(0).toString()); @@ -58,7 +60,7 @@ public class HostURLGeneratorTest { public void generateNodeInfoUrl_retrieve_config_server_as_parameter_test_if_url_is_formatted_correctly() throws Exception { mockCommandExecutor.addCommand(CAT_NODE_HOST_NAME_PATH); String configServerHostname = "cfg1.prod.region1"; - ArrayList<URL> actualUrls = HostURLGenerator.generateNodeInfoUrl(mockCommandExecutor, configServerHostname); + List<URL> actualUrls = HostURLGenerator.generateNodeInfoUrl(mockCommandExecutor, configServerHostname); String expectedUrl = CONFIG_SERVER_HOSTNAME_1 + NODE_HOSTNAME_PREFIX + EXPECTED_HOSTNAME; String actualUrl = actualUrls.get(0).toString(); assertEquals(expectedUrl, actualUrl); diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/commons/noderepo/IPAddressVerifierTest.java b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/commons/noderepo/IPAddressVerifierTest.java index d0b378af4b8..197b4836bc8 100644 --- a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/commons/noderepo/IPAddressVerifierTest.java +++ b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/commons/noderepo/IPAddressVerifierTest.java @@ -14,6 +14,11 @@ import static org.junit.Assert.assertEquals; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.spy; +/** + * @author sgrostad + * @author olaaun + */ + public class IPAddressVerifierTest { private IPAddressVerifier ipAddressVerifier = spy(new IPAddressVerifier()); diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/commons/noderepo/NodeJsonConverterTest.java b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/commons/noderepo/NodeJsonConverterTest.java index c23d29c4076..0940efd08d3 100644 --- a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/commons/noderepo/NodeJsonConverterTest.java +++ b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/commons/noderepo/NodeJsonConverterTest.java @@ -8,21 +8,24 @@ import java.io.File; import java.net.URL; import java.util.ArrayList; import java.util.Arrays; +import java.util.List; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; /** - * Created by olaa on 07/07/2017. + * @author sgrostad + * @author olaaun */ + public class NodeJsonConverterTest { private static final double DELTA = 0.1; @Test public void convertJsonModel_should_return_correct_HardwareInfo() throws Exception { - ArrayList<URL> urls = new ArrayList<>(Arrays.asList(new File("src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/nodeInfoTest.json").toURI().toURL())); + List<URL> urls = new ArrayList<>(Arrays.asList(new File("src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/nodeInfoTest.json").toURI().toURL())); NodeRepoJsonModel nodeRepoJsonModel = NodeRepoInfoRetriever.retrieve(urls); HardwareInfo hardwareInfo = NodeJsonConverter.convertJsonModelToHardwareInfo(nodeRepoJsonModel); double expectedMinDiskAvailable = 500.0; diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/commons/noderepo/NodeRepoInfoRetrieverTest.java b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/commons/noderepo/NodeRepoInfoRetrieverTest.java index 49792963a05..97bf3fdeb11 100644 --- a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/commons/noderepo/NodeRepoInfoRetrieverTest.java +++ b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/commons/noderepo/NodeRepoInfoRetrieverTest.java @@ -9,14 +9,20 @@ import java.net.MalformedURLException; import java.net.URL; import java.nio.file.Paths; import java.util.ArrayList; +import java.util.List; import static org.junit.Assert.assertEquals; import static org.junit.Assert.fail; +/** + * @author sgrostad + * @author olaaun + */ + public class NodeRepoInfoRetrieverTest { private NodeRepoInfoRetriever nodeRepoInfoRetriever; - private ArrayList<URL> urls; + private List<URL> urls; private static final double DELTA = 0.1; private static final String ABSOLUTE_PATH = Paths.get(".").toAbsolutePath().normalize().toString(); private static final String RESOURCE_PATH = "src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources"; diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/commons/parser/OutputParserTest.java b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/commons/parser/OutputParserTest.java index dc43fcd2e47..09286f4e61a 100644 --- a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/commons/parser/OutputParserTest.java +++ b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/commons/parser/OutputParserTest.java @@ -1,20 +1,20 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.node.verification.commons.parser; -import com.yahoo.vespa.hosted.node.verification.commons.parser.OutputParser; -import com.yahoo.vespa.hosted.node.verification.commons.parser.ParseInstructions; -import com.yahoo.vespa.hosted.node.verification.commons.parser.ParseResult; import org.junit.Before; import org.junit.Test; import java.util.ArrayList; import java.util.Arrays; +import java.util.List; import static org.junit.Assert.assertEquals; /** - * Created by sgrostad on 21/07/2017. + * @author sgrostad + * @author olaaun */ + public class OutputParserTest { private static final String RETURN_VALUE = "#returnValue#"; @@ -24,8 +24,8 @@ public class OutputParserTest { private static final String SEARCH_WORD_1 = "Parsing"; private static final String SEARCH_WORD_2 = "this"; private static final String REGEX_SEARCH_WORD = ".*S.*"; - private ArrayList<String> commandOutput; - private ArrayList<String> searchWords; + private List<String> commandOutput; + private List<String> searchWords; @Before public void setup() { @@ -36,7 +36,7 @@ public class OutputParserTest { public void parseOutput_searching_for_two_normal_words() { searchWords = new ArrayList<>(Arrays.asList(SEARCH_WORD_1, SEARCH_WORD_2)); ParseInstructions parseInstructions = new ParseInstructions(6, 8, " ", searchWords); - ArrayList<ParseResult> parseResults = OutputParser.parseOutput(parseInstructions, commandOutput); + List<ParseResult> parseResults = OutputParser.parseOutput(parseInstructions, commandOutput); ParseResult expectedParseResult1 = new ParseResult(SEARCH_WORD_1, RETURN_VALUE); ParseResult expectedParseResult2 = new ParseResult(SEARCH_WORD_2, RETURN_VALUE); assertEquals(expectedParseResult1, parseResults.get(0)); @@ -47,7 +47,7 @@ public class OutputParserTest { public void parseOutput_searching_for_two_normal_words_with_semicolon_as_line_split() { searchWords = new ArrayList<>(Arrays.asList(SEARCH_WORD_1, SEARCH_WORD_2)); ParseInstructions parseInstructions = new ParseInstructions(4, 5, ";", searchWords); - ArrayList<ParseResult> parseResults = OutputParser.parseOutput(parseInstructions, commandOutput); + List<ParseResult> parseResults = OutputParser.parseOutput(parseInstructions, commandOutput); ParseResult expectedParseResult1 = new ParseResult(SEARCH_WORD_1, RETURN_VALUE); ParseResult expectedParseResult2 = new ParseResult(SEARCH_WORD_2, RETURN_VALUE); assertEquals(expectedParseResult1, parseResults.get(0)); @@ -58,7 +58,7 @@ public class OutputParserTest { public void parseOutput_searching_for_word_containing_capital_s() { searchWords = new ArrayList<>(Arrays.asList(REGEX_SEARCH_WORD)); ParseInstructions parseInstructions = new ParseInstructions(1, 8, " ", searchWords); - ArrayList<ParseResult> parseResults = OutputParser.parseOutput(parseInstructions, commandOutput); + List<ParseResult> parseResults = OutputParser.parseOutput(parseInstructions, commandOutput); ParseResult expectedParseResult1 = new ParseResult("Should", RETURN_VALUE); ParseResult expectedParseResult2 = new ParseResult("thiS", RETURN_VALUE); assertEquals(expectedParseResult1, parseResults.get(0)); diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/commons/report/BenchmarkReportTest.java b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/commons/report/BenchmarkReportTest.java index 2963b57110c..931b42bb2a0 100644 --- a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/commons/report/BenchmarkReportTest.java +++ b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/commons/report/BenchmarkReportTest.java @@ -6,6 +6,11 @@ import org.junit.Test; import static org.junit.Assert.assertEquals; +/** + * @author sgrostad + * @author olaaun + */ + public class BenchmarkReportTest { private BenchmarkReport benchmarkReport = new BenchmarkReport(); diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/commons/report/ReportSenderTest.java b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/commons/report/ReportSenderTest.java deleted file mode 100644 index bc6d1d6d089..00000000000 --- a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/commons/report/ReportSenderTest.java +++ /dev/null @@ -1,70 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.verification.commons.report; - -import org.junit.After; -import org.junit.Before; -import org.junit.Test; - -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.PrintStream; -import java.net.URL; -import java.nio.file.Paths; -import java.util.ArrayList; -import java.util.Arrays; - -import static org.junit.Assert.assertEquals; - -public class ReportSenderTest { - - private final ByteArrayOutputStream println = new ByteArrayOutputStream(); - private static final String ABSOLUTE_PATH = Paths.get(".").toAbsolutePath().normalize().toString(); - private static final String RESOURCE_PATH = "src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/nodeInfoTest.json"; - private static final String URL_RESOURCE_PATH = "file://" + ABSOLUTE_PATH + "/" + RESOURCE_PATH; - private static ArrayList<URL> nodeInfoUrls; - - @Before - public void setup() throws IOException { - System.setOut(new PrintStream(println)); - URL nodeInfoUrlWithAlreadyExistingHardwareDivergence = new URL(URL_RESOURCE_PATH); - nodeInfoUrls = new ArrayList<>(Arrays.asList(nodeInfoUrlWithAlreadyExistingHardwareDivergence)); - } - - @After - public void cleanUpStream() { - System.setOut(System.out); - } - - @Test - public void reportBenchmarkResults_should_update_already_existing_hardwareDivergence_changing_existing_values() throws Exception { - BenchmarkReport benchmarkReport = new BenchmarkReport(); - double cpuCyclesPerSec = 0.3; - double memoryReadSpeedGBs = 0.1; - benchmarkReport.setCpuCyclesPerSec(cpuCyclesPerSec); - benchmarkReport.setMemoryReadSpeedGBs(memoryReadSpeedGBs); - String expectedReport = "{\"benchmarkReport\":{\"cpuCyclesPerSec\":0.3,\"memoryReadSpeedGBs\":0.1}}"; - ReportSender.reportBenchmarkResults(benchmarkReport,nodeInfoUrls); - assertEquals(expectedReport, println.toString()); - } - - @Test - public void reportBenchmarkResults_should_should_update_already_existing_hardwareDivergence_prints_null_when_empty_benchmarkReport() throws Exception { - BenchmarkReport benchmarkReport = new BenchmarkReport(); - String expectedReport = "null"; - ReportSender.reportBenchmarkResults(benchmarkReport, nodeInfoUrls); - assertEquals(expectedReport, println.toString()); - } - - @Test - public void reportSpecVerificationResults_should_update_already_existing_hardwareDivergence_adding_report_type() throws Exception { - SpecVerificationReport specVerificationReport = new SpecVerificationReport(); - double actualDiskSpaceAvailable = 150D; - boolean actualIpv6Connection = false; - specVerificationReport.setActualDiskSpaceAvailable(actualDiskSpaceAvailable); - specVerificationReport.setActualIpv6Connection(actualIpv6Connection); - String expectedReport = "{\"specVerificationReport\":{\"actualDiskSpaceAvailable\":150.0,\"actualIpv6Connection\":false},\"benchmarkReport\":{\"cpuCyclesPerSec\":0.5}}"; - ReportSender.reportSpecVerificationResults(specVerificationReport, nodeInfoUrls); - assertEquals(expectedReport, println.toString()); - } - -} diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/commons/report/ReporterTest.java b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/commons/report/ReporterTest.java new file mode 100644 index 00000000000..5f469db7f53 --- /dev/null +++ b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/commons/report/ReporterTest.java @@ -0,0 +1,104 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.verification.commons.report; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.PrintStream; +import java.net.URL; +import java.nio.file.Paths; +import java.util.List; +import java.util.ArrayList; +import java.util.Arrays; + +import static org.junit.Assert.assertEquals; + +/** + * @author sgrostad + * @author olaaun + */ + +public class ReporterTest { + + private final ByteArrayOutputStream println = new ByteArrayOutputStream(); + private static final String ABSOLUTE_PATH = Paths.get(".").toAbsolutePath().normalize().toString(); + private static final String RESOURCE_PATH_TO_VALID_HARDWARE_DIVERGENCE = "src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/nodeInfoTest.json"; + private static final String RESOURCE_PATH_TO_INVALID_HARDWARE_DIVERGENCE = "src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/nodeInfoNotInterpretableHardwareDivergence.json"; + private static final String RESOURCE_PATH_TO_EMPTY_HARDWARE_DIVERGENCE = "src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/nodeRepo.json"; + private static final String URL_VALID_RESOURCE_PATH = "file://" + ABSOLUTE_PATH + "/" + RESOURCE_PATH_TO_VALID_HARDWARE_DIVERGENCE; + private static final String URL_INVALID_RESOURCE_PATH = "file://" + ABSOLUTE_PATH + "/" + RESOURCE_PATH_TO_INVALID_HARDWARE_DIVERGENCE; + private static final String URL_EMPTY_RESOURCE_PATH = "file://" + ABSOLUTE_PATH + "/" + RESOURCE_PATH_TO_EMPTY_HARDWARE_DIVERGENCE; + private static List<URL> nodeInfoUrlsToValidHardwareDivergence; + private static List<URL> nodeInfoUrlsToNOTValidHardwareDivergence; + private static List<URL> nodeInfoUrlsWithNoHardwareDivergence; + + @Before + public void setup() throws IOException { + System.setOut(new PrintStream(println)); + URL nodeInfoUrlWithAlreadyExistingHardwareDivergence = new URL(URL_VALID_RESOURCE_PATH); + nodeInfoUrlsToValidHardwareDivergence = new ArrayList<>(Arrays.asList(nodeInfoUrlWithAlreadyExistingHardwareDivergence)); + URL nodeInfoUrlWithExistingButWrongHardwareDivergence = new URL(URL_INVALID_RESOURCE_PATH); + nodeInfoUrlsToNOTValidHardwareDivergence = new ArrayList<>(Arrays.asList(nodeInfoUrlWithExistingButWrongHardwareDivergence)); + URL nodeInfoUrlWithNoHardwareDivergence = new URL(URL_EMPTY_RESOURCE_PATH); + nodeInfoUrlsWithNoHardwareDivergence = new ArrayList<>(Arrays.asList(nodeInfoUrlWithNoHardwareDivergence)); + } + + @After + public void cleanUpStream() { + System.setOut(System.out); + } + + @Test + public void reportBenchmarkResults_should_update_already_existing_hardwareDivergence_changing_existing_values() throws Exception { + BenchmarkReport benchmarkReport = new BenchmarkReport(); + double cpuCyclesPerSec = 0.3; + double memoryReadSpeedGBs = 0.1; + benchmarkReport.setCpuCyclesPerSec(cpuCyclesPerSec); + benchmarkReport.setMemoryReadSpeedGBs(memoryReadSpeedGBs); + String expectedReport = "{\"benchmarkReport\":{\"cpuCyclesPerSec\":0.3,\"memoryReadSpeedGBs\":0.1}}"; + Reporter.reportBenchmarkResults(benchmarkReport, nodeInfoUrlsToValidHardwareDivergence); + assertEquals(expectedReport, println.toString()); + } + + @Test + public void reportBenchmarkResults_should_should_update_already_existing_hardwareDivergence_prints_null_when_empty_benchmarkReport() throws Exception { + BenchmarkReport benchmarkReport = new BenchmarkReport(); + String expectedReport = "null"; + Reporter.reportBenchmarkResults(benchmarkReport, nodeInfoUrlsToValidHardwareDivergence); + assertEquals(expectedReport, println.toString()); + } + + @Test + public void reportSpecVerificationResults_should_update_already_existing_hardwareDivergence_adding_report_type() throws Exception { + SpecVerificationReport specVerificationReport = new SpecVerificationReport(); + double actualDiskSpaceAvailable = 150D; + boolean actualIpv6Connection = false; + specVerificationReport.setActualDiskSpaceAvailable(actualDiskSpaceAvailable); + specVerificationReport.setActualIpv6Connection(actualIpv6Connection); + String expectedReport = "{\"specVerificationReport\":{\"actualDiskSpaceAvailable\":150.0,\"actualIpv6Connection\":false},\"benchmarkReport\":{\"cpuCyclesPerSec\":0.5}}"; + Reporter.reportSpecVerificationResults(specVerificationReport, nodeInfoUrlsToValidHardwareDivergence); + assertEquals(expectedReport, println.toString()); + } + + @Test + public void reportSpecVerificationResults_make_new_correct_hardwareDivergence_because_old_is_wrong() throws Exception { + SpecVerificationReport specVerificationReport = new SpecVerificationReport(); + double actualDiskSpaceAvailable = 150D; + specVerificationReport.setActualDiskSpaceAvailable(actualDiskSpaceAvailable); + String expectedReport = "{\"specVerificationReport\":{\"actualDiskSpaceAvailable\":150.0}}"; + Reporter.reportSpecVerificationResults(specVerificationReport, nodeInfoUrlsToNOTValidHardwareDivergence); + assertEquals(expectedReport, println.toString()); + } + + @Test + public void reportSpecVerificationResults_make_new_empty_hardwareDivergence_because_there_is_no_old() throws Exception { + SpecVerificationReport specVerificationReport = new SpecVerificationReport(); + String expectedReport = "null"; + Reporter.reportSpecVerificationResults(specVerificationReport, nodeInfoUrlsWithNoHardwareDivergence); + assertEquals(expectedReport, println.toString()); + } + +}
\ No newline at end of file diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/commons/report/SpecVerificationReportTest.java b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/commons/report/SpecVerificationReportTest.java index 0b19e1b758c..72629cc565d 100644 --- a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/commons/report/SpecVerificationReportTest.java +++ b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/commons/report/SpecVerificationReportTest.java @@ -2,13 +2,17 @@ package com.yahoo.vespa.hosted.node.verification.commons.report; import com.fasterxml.jackson.databind.ObjectMapper; -import com.yahoo.vespa.hosted.node.verification.commons.report.SpecVerificationReport; import com.yahoo.vespa.hosted.node.verification.mock.MockCommandExecutor; import com.yahoo.vespa.hosted.node.verification.spec.retrievers.HardwareInfo; import org.junit.Before; import org.junit.Test; -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; + +/** + * @author sgrostad + * @author olaaun + */ public class SpecVerificationReportTest { diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/BenchmarkResultInspectorTest.java b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/BenchmarkResultInspectorTest.java index b1a98acd78a..4c999dba35d 100644 --- a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/BenchmarkResultInspectorTest.java +++ b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/BenchmarkResultInspectorTest.java @@ -1,17 +1,18 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.node.verification.hardware; -import com.yahoo.vespa.hosted.node.verification.hardware.benchmarks.BenchmarkResults; import com.yahoo.vespa.hosted.node.verification.commons.report.BenchmarkReport; +import com.yahoo.vespa.hosted.node.verification.hardware.benchmarks.BenchmarkResults; import org.junit.Before; import org.junit.Test; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertTrue; +/** + * @author sgrostad + * @author olaaun + */ public class BenchmarkResultInspectorTest { diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/HardwareBenchmarkerTest.java b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/HardwareBenchmarkerTest.java index 7b4ad9b967b..6848fc045e2 100644 --- a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/HardwareBenchmarkerTest.java +++ b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/HardwareBenchmarkerTest.java @@ -12,6 +12,11 @@ import java.util.Arrays; import static org.junit.Assert.assertTrue; +/** + * @author sgrostad + * @author olaaun + */ + public class HardwareBenchmarkerTest { private MockCommandExecutor mockCommandExecutor; diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/CPUBenchmarkTest.java b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/CPUBenchmarkTest.java index b2655aca26e..04e9057b44e 100644 --- a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/CPUBenchmarkTest.java +++ b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/CPUBenchmarkTest.java @@ -9,14 +9,17 @@ import org.junit.Test; import java.io.IOException; import java.util.ArrayList; +import java.util.List; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; /** - * Created by sgrostad on 11/07/2017. + * @author sgrostad + * @author olaaun */ + public class CPUBenchmarkTest { private static final String cpuEuropeanDelimiters = "src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/resources/cpuCyclesWithDotsTimeWithCommaTest.txt"; @@ -56,8 +59,8 @@ public class CPUBenchmarkTest { @Test public void parseCpuCyclesPerSec_return_correct_ArrayList() throws IOException { - ArrayList<String> mockCommandOutput = MockCommandExecutor.readFromFile(cpuEuropeanDelimiters); - ArrayList<ParseResult> parseResults = cpu.parseCpuCyclesPerSec(mockCommandOutput); + List<String> mockCommandOutput = MockCommandExecutor.readFromFile(cpuEuropeanDelimiters); + List<ParseResult> parseResults = cpu.parseCpuCyclesPerSec(mockCommandOutput); ParseResult expectedParseCyclesResult = new ParseResult("cycles", "2.066.201.729"); ParseResult expectedParseSecondsResult = new ParseResult("seconds", "0,957617512"); assertEquals(expectedParseCyclesResult, parseResults.get(0)); @@ -66,7 +69,7 @@ public class CPUBenchmarkTest { @Test public void test_if_setCpuCyclesPerSec_reads_output_correctly() throws IOException { - ArrayList<ParseResult> parseResults = new ArrayList<>(); + List<ParseResult> parseResults = new ArrayList<>(); parseResults.add(new ParseResult("cycles", "2.066.201.729")); parseResults.add(new ParseResult("seconds", "0,957617512")); cpu.setCpuCyclesPerSec(parseResults); diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/DiskBenchmarkTest.java b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/DiskBenchmarkTest.java index 8bc0266e3b5..8254718f36d 100644 --- a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/DiskBenchmarkTest.java +++ b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/DiskBenchmarkTest.java @@ -7,14 +7,17 @@ import org.junit.Before; import org.junit.Test; import java.util.ArrayList; +import java.util.List; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; /** - * Created by sgrostad on 12/07/2017. + * @author sgrostad + * @author olaaun */ + public class DiskBenchmarkTest { private DiskBenchmark diskBenchmark; @@ -54,7 +57,7 @@ public class DiskBenchmarkTest { @Test public void parseDiskSpeed_valid_input() throws Exception { - ArrayList<String> mockCommandOutput = MockCommandExecutor.readFromFile(VALID_OUTPUT_FILE); + List<String> mockCommandOutput = MockCommandExecutor.readFromFile(VALID_OUTPUT_FILE); ParseResult parseResult = diskBenchmark.parseDiskSpeed(mockCommandOutput); ParseResult expectedParseResult = new ParseResult("MB/s", "243"); assertEquals(expectedParseResult, parseResult); @@ -62,7 +65,7 @@ public class DiskBenchmarkTest { @Test public void parseDiskSpeed_invalid_input() throws Exception { - ArrayList<String> mockCommandOutput = MockCommandExecutor.readFromFile(INVALID_OUTPUT_FILE); + List<String> mockCommandOutput = MockCommandExecutor.readFromFile(INVALID_OUTPUT_FILE); ParseResult parseResult = diskBenchmark.parseDiskSpeed(mockCommandOutput); ParseResult expectedParseResult = new ParseResult("invalid", "invalid"); assertEquals(expectedParseResult, parseResult); diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/MemoryBenchmarkTest.java b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/MemoryBenchmarkTest.java index b44fcd8903e..e2c5cdfa555 100644 --- a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/MemoryBenchmarkTest.java +++ b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/MemoryBenchmarkTest.java @@ -6,15 +6,17 @@ import com.yahoo.vespa.hosted.node.verification.mock.MockCommandExecutor; import org.junit.Before; import org.junit.Test; -import java.util.ArrayList; +import java.util.List; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; /** - * Created by olaa on 14/07/2017. + * @author sgrostad + * @author olaaun */ + public class MemoryBenchmarkTest { private MemoryBenchmark memoryBenchmark; @@ -48,7 +50,7 @@ public class MemoryBenchmarkTest { public void parseMemorySpeed_valid_output() throws Exception { Double expectedSpeed = 12.1; String mockOutput = "This is a test \n the memory speed to be found is " + expectedSpeed + " GB/s"; - ArrayList<String> mockCommandOutput = commandExecutor.outputFromString(mockOutput); + List<String> mockCommandOutput = commandExecutor.outputFromString(mockOutput); ParseResult parseResult = memoryBenchmark.parseMemorySpeed(mockCommandOutput); ParseResult expectedParseResult = new ParseResult("GB/s", expectedSpeed.toString()); assertEquals(expectedParseResult, parseResult); @@ -56,7 +58,7 @@ public class MemoryBenchmarkTest { @Test public void parseMemorySpeed_invalid_output() throws Exception { - ArrayList<String> mockCommandOutput = commandExecutor.outputFromString(""); + List<String> mockCommandOutput = commandExecutor.outputFromString(""); ParseResult parseResult = memoryBenchmark.parseMemorySpeed(mockCommandOutput); ParseResult expectedParseResult = new ParseResult("invalid", "invalid"); assertEquals(expectedParseResult, parseResult); diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/mock/MockCommandExecutor.java b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/mock/MockCommandExecutor.java index 83f542e7142..6e89b91aa94 100644 --- a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/mock/MockCommandExecutor.java +++ b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/mock/MockCommandExecutor.java @@ -8,10 +8,13 @@ import java.nio.file.Files; import java.nio.file.Paths; import java.util.ArrayList; import java.util.Arrays; +import java.util.List; /** - * Created by olaa on 17/07/2017. + * @author sgrostad + * @author olaaun */ + public class MockCommandExecutor extends CommandExecutor { private ArrayList<String> mockCommands; @@ -24,7 +27,7 @@ public class MockCommandExecutor extends CommandExecutor { } @Override - public ArrayList<String> executeCommand(String command) throws IOException { + public List<String> executeCommand(String command) throws IOException { String mockCommand = mockCommands.get(counter++); if (mockCommand.equals(DUMMY_COMMAND)) return null; return super.executeCommand(mockCommand); @@ -38,11 +41,11 @@ public class MockCommandExecutor extends CommandExecutor { mockCommands.add(DUMMY_COMMAND); } - public static ArrayList<String> readFromFile(String filepath) throws IOException { + public static List<String> readFromFile(String filepath) throws IOException { return new ArrayList<>(Arrays.asList(new String(Files.readAllBytes(Paths.get(filepath))).split("\n"))); } - public ArrayList<String> outputFromString(String output) { + public List<String> outputFromString(String output) { return new ArrayList<>(Arrays.asList(output.split("\n"))); } diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/HardwareNodeComparatorTest.java b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/HardwareNodeComparatorTest.java index 6113ce315c1..22d1a167a13 100644 --- a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/HardwareNodeComparatorTest.java +++ b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/HardwareNodeComparatorTest.java @@ -8,12 +8,12 @@ import org.junit.Before; import org.junit.Test; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; /** - * Created by olaa on 07/07/2017. + * @author sgrostad + * @author olaaun */ + public class HardwareNodeComparatorTest { private HardwareInfo actualHardware; diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/SpecVerifierTest.java b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/SpecVerifierTest.java index 61f52537e74..16e72521853 100644 --- a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/SpecVerifierTest.java +++ b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/SpecVerifierTest.java @@ -2,11 +2,11 @@ package com.yahoo.vespa.hosted.node.verification.spec; import com.fasterxml.jackson.databind.ObjectMapper; -import com.yahoo.vespa.hosted.node.verification.mock.MockCommandExecutor; import com.yahoo.vespa.hosted.node.verification.commons.noderepo.NodeRepoInfoRetriever; import com.yahoo.vespa.hosted.node.verification.commons.noderepo.NodeRepoJsonModel; -import com.yahoo.vespa.hosted.node.verification.spec.retrievers.HardwareInfo; import com.yahoo.vespa.hosted.node.verification.commons.report.SpecVerificationReport; +import com.yahoo.vespa.hosted.node.verification.mock.MockCommandExecutor; +import com.yahoo.vespa.hosted.node.verification.spec.retrievers.HardwareInfo; import org.junit.Before; import org.junit.Test; @@ -14,11 +14,17 @@ import java.io.File; import java.net.URL; import java.nio.file.Paths; import java.util.ArrayList; +import java.util.List; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; +/** + * @author sgrostad + * @author olaaun + */ + public class SpecVerifierTest { private MockCommandExecutor mockCommandExecutor; @@ -36,7 +42,7 @@ public class SpecVerifierTest { private static final String PING_RESPONSE = RESOURCE_PATH + "/validpingresponse"; private static final String INVALID_PING_RESPONSE = RESOURCE_PATH + "/pingresponse-all-packets-lost"; private static final double DELTA = 0.1; - ArrayList<URL> nodeInfoUrls; + List<URL> nodeInfoUrls; @Before public void setup() { diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/IPAddressVerifierTest.json b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/IPAddressVerifierTest.json index 4fcb5efed9e..a3bd4957467 100644 --- a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/IPAddressVerifierTest.json +++ b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/IPAddressVerifierTest.json @@ -77,6 +77,5 @@ "2001:db8:0:1234:0:567:8:1" ], "additionalIpAddresses": [ - ] }
\ No newline at end of file diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/nodeInfoNotInterpretableHardwareDivergence.json b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/nodeInfoNotInterpretableHardwareDivergence.json new file mode 100644 index 00000000000..0faf1873446 --- /dev/null +++ b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/nodeInfoNotInterpretableHardwareDivergence.json @@ -0,0 +1,82 @@ +{ + "url": "https://api.vespayahoo.com", + "id": "id.prod.yahoo.com", + "state": "failed", + "type": "host", + "hostname": "hostname", + "openStackId": "4a8689e1-3a8e-4f50-b99f-d33b245a9661", + "flavor": "C-2T/24/500", + "canonicalFlavor": "C-2B/24/500", + "minDiskAvailableGb": 500.0, + "minMainMemoryAvailableGb": 24.0, + "description": "BARE_METAL with 24.0 CPUs, 24.0 Gb memory and 500.0 Gb disk", + "minCpuCores": 24.0, + "cost": 70, + "fastDisk": false, + "environment": "BARE_METAL", + "owner": { + "tenant": "hosted-vespa", + "application": "routing", + "instance": "default" + }, + "membership": { + "clustertype": "container", + "clusterid": "node-admin", + "group": "0", + "index": 32, + "retired": false + }, + "restartGeneration": 0, + "currentRestartGeneration": 0, + "wantedDockerImage": "docker-registry.ops.yahoo.com:4443/vespa/ci:6.138.1", + "wantedVespaVersion": "6.138.1", + "rebootGeneration": 49, + "currentRebootGeneration": 49, + "vespaVersion": "6.138.18", + "currentDockerImage": "docker-registry.ops.yahoo.com:4443/vespa/ci:6.138.18", + "hostedVersion": "6.138.18", + "convergedStateVersion": "6.138.18", + "failCount": 44, + "hardwareFailure": true, + "hardwareFailureType": "disk_smart", + "hardwareFailureDescription": "disk_smart", + "wantToRetire": false, + "wantToDeprovision": false, + "history": [ + { + "event": "readied", + "at": 1501070910372, + "agent": "system" + }, + { + "event": "reserved", + "at": 1501070952692, + "agent": "application" + }, + { + "event": "activated", + "at": 1501070958698, + "agent": "application" + }, + { + "event": "rebooted", + "at": 1501656469859, + "agent": "system" + }, + { + "event": "down", + "at": 1502809344596, + "agent": "system" + }, + { + "event": "failed", + "at": 1502812976621, + "agent": "system" + } + ], + "ipAddresses": [ + ], + "additionalIpAddresses": [ + ], + "hardwareDivergence": "{\"hardwareDivergence\": null - it will not manage to parse this}" +}
\ No newline at end of file diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/nodeInfoTest.json b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/nodeInfoTest.json index 216991846b0..c557afb3256 100644 --- a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/nodeInfoTest.json +++ b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/nodeInfoTest.json @@ -45,6 +45,5 @@ "10.200.66.16" ], "additionalIpAddresses": [], - "hardwareDivergence": "{\"benchmarkReport\": {\"cpuCyclesPerSec\": 0.5}}" }
\ No newline at end of file diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/CPURetrieverTest.java b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/CPURetrieverTest.java index cda8066734d..d878d6614f7 100644 --- a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/CPURetrieverTest.java +++ b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/CPURetrieverTest.java @@ -8,12 +8,15 @@ import org.junit.Test; import java.io.IOException; import java.util.ArrayList; +import java.util.List; import static org.junit.Assert.assertEquals; /** - * Created by olaa on 03/07/2017. + * @author sgrostad + * @author olaaun */ + public class CPURetrieverTest { private static final String FILENAME = "src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/cpuinfoTest"; @@ -39,8 +42,8 @@ public class CPURetrieverTest { @Test public void parseCPUInfoFile_should_return_valid_ArrayList() throws IOException { - ArrayList<String> commandOutput = MockCommandExecutor.readFromFile(FILENAME); - ArrayList<ParseResult> ParseResults = cpu.parseCPUInfoFile(commandOutput); + List<String> commandOutput = MockCommandExecutor.readFromFile(FILENAME); + List<ParseResult> ParseResults = cpu.parseCPUInfoFile(commandOutput); String expectedSearchWord = "cpu MHz"; String expectedValue = "2493.821"; @@ -59,7 +62,7 @@ public class CPURetrieverTest { @Test public void setCpuCores_counts_cores_correctly() { - ArrayList<ParseResult> parseResults = new ArrayList<>(); + List<ParseResult> parseResults = new ArrayList<>(); parseResults.add(new ParseResult("cpu MHz", "2000")); parseResults.add(new ParseResult("cpu MHz", "2000")); parseResults.add(new ParseResult("cpu MHz", "2000")); diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/DiskRetrieverTest.java b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/DiskRetrieverTest.java index 6cd29ab8f64..de4bf946d59 100644 --- a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/DiskRetrieverTest.java +++ b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/DiskRetrieverTest.java @@ -8,14 +8,16 @@ import org.junit.Before; import org.junit.Test; import java.io.IOException; -import java.util.ArrayList; +import java.util.List; import static org.junit.Assert.assertEquals; import static org.junit.Assert.fail; /** - * Created by olaa on 06/07/2017. + * @author sgrostad + * @author olaaun */ + public class DiskRetrieverTest { private MockCommandExecutor commandExecutor; @@ -59,7 +61,7 @@ public class DiskRetrieverTest { @Test public void parseDiskType_should_find_fast_disk() throws Exception { diskRetriever = new DiskRetriever(hardwareInfo, commandExecutor); - ArrayList<String> mockOutput = commandExecutor.outputFromString("Name Rota \nsda 0"); + List<String> mockOutput = commandExecutor.outputFromString("Name Rota \nsda 0"); ParseResult parseResult = diskRetriever.parseDiskType(mockOutput); ParseResult expectedParseResult = new ParseResult("sda", "0"); assertEquals(expectedParseResult, parseResult); @@ -67,7 +69,7 @@ public class DiskRetrieverTest { @Test public void parseDiskType_should_not_find_fast_disk() throws Exception { - ArrayList<String> mockOutput = commandExecutor.outputFromString("Name Rota \nsda 1"); + List<String> mockOutput = commandExecutor.outputFromString("Name Rota \nsda 1"); ParseResult parseResult = diskRetriever.parseDiskType(mockOutput); ParseResult expectedParseResult = new ParseResult("sda", "1"); assertEquals(expectedParseResult, parseResult); @@ -75,7 +77,7 @@ public class DiskRetrieverTest { @Test public void parseDiskType_with_invalid_outputstream_does_not_contain_searchword_should_throw_exception() throws Exception { - ArrayList<String> mockOutput = commandExecutor.outputFromString("Name Rota"); + List<String> mockOutput = commandExecutor.outputFromString("Name Rota"); try { ParseResult parseResult = diskRetriever.parseDiskType(mockOutput); fail("Should have thrown IOException when outputstream doesn't contain search word"); @@ -89,8 +91,8 @@ public class DiskRetrieverTest { @Test public void parseDiskSize_should_find_size_from_file_and_insert_into_parseResult() throws Exception { String filepath = "src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/filesize"; - ArrayList<String> mockOutput = MockCommandExecutor.readFromFile(filepath); - ArrayList<ParseResult> parseResults = diskRetriever.parseDiskSize(mockOutput); + List<String> mockOutput = MockCommandExecutor.readFromFile(filepath); + List<ParseResult> parseResults = diskRetriever.parseDiskSize(mockOutput); ParseResult expectedParseResult1 = new ParseResult("Size", "799.65"); assertEquals(expectedParseResult1, parseResults.get(0)); ParseResult expectedParseResult2 = new ParseResult("Size", "960.19"); diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/HardwareInfoRetrieverTest.java b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/HardwareInfoRetrieverTest.java index ae131e0c4e8..552d57d2b59 100644 --- a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/HardwareInfoRetrieverTest.java +++ b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/HardwareInfoRetrieverTest.java @@ -6,10 +6,15 @@ import com.yahoo.vespa.hosted.node.verification.mock.MockCommandExecutor; import com.yahoo.vespa.hosted.node.verification.spec.VerifierSettings; import org.junit.Before; import org.junit.Test; + +import static org.junit.Assert.assertEquals; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.spy; -import static org.junit.Assert.assertEquals; +/** + * @author sgrostad + * @author olaaun + */ public class HardwareInfoRetrieverTest { diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/MemoryRetrieverTest.java b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/MemoryRetrieverTest.java index 9d20acbc762..6b0b5f6d24a 100644 --- a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/MemoryRetrieverTest.java +++ b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/MemoryRetrieverTest.java @@ -7,13 +7,15 @@ import org.junit.Before; import org.junit.Test; import java.io.IOException; -import java.util.ArrayList; +import java.util.List; import static org.junit.Assert.assertEquals; /** - * Created by sgrostad on 06/07/2017. + * @author sgrostad + * @author olaaun */ + public class MemoryRetrieverTest { private static final String FILENAME = "src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/meminfoTest"; @@ -39,7 +41,7 @@ public class MemoryRetrieverTest { @Test public void parseMemInfoFile_should_return_valid_parseResult() throws IOException { - ArrayList<String> commandOutput = MockCommandExecutor.readFromFile(FILENAME); + List<String> commandOutput = MockCommandExecutor.readFromFile(FILENAME); ParseResult parseResult = memory.parseMemInfoFile(commandOutput); ParseResult expectedParseResult = new ParseResult("MemTotal", "4042128 kB"); assertEquals(expectedParseResult, parseResult); diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/NetRetrieverTest.java b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/NetRetrieverTest.java index 893f23d6133..565712608f5 100644 --- a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/NetRetrieverTest.java +++ b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/NetRetrieverTest.java @@ -10,6 +10,7 @@ import org.junit.Test; import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; +import java.util.List; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -19,8 +20,10 @@ import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.spy; /** - * Created by sgrostad on 07/07/2017. + * @author sgrostad + * @author olaaun */ + public class NetRetrieverTest { private static final String RESOURCE_PATH = "src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/"; @@ -32,7 +35,7 @@ public class NetRetrieverTest { private HardwareInfo hardwareInfo; private MockCommandExecutor commandExecutor; private NetRetriever net; - private ArrayList<ParseResult> parseResults; + private List<ParseResult> parseResults; private VerifierSettings verifierSettings = spy(new VerifierSettings()); private static final double DELTA = 0.1; @@ -70,7 +73,7 @@ public class NetRetrieverTest { @Test public void parseNetInterface_get_ipv_from_ifconfig_testFile() throws IOException { - ArrayList<String> mockOutput = MockCommandExecutor.readFromFile(NET_FIND_INTERFACE); + List<String> mockOutput = MockCommandExecutor.readFromFile(NET_FIND_INTERFACE); parseResults = net.parseNetInterface(mockOutput); net.updateHardwareInfoWithNet(parseResults); assertTrue(hardwareInfo.getIpv4Interface()); @@ -79,7 +82,7 @@ public class NetRetrieverTest { @Test public void parseNetInterface_get_ipv_from_ifconfigNotIpv6_testFile() throws IOException { - ArrayList<String> mockOutput = MockCommandExecutor.readFromFile(NET_FIND_INTERFACE + "NoIpv6"); + List<String> mockOutput = MockCommandExecutor.readFromFile(NET_FIND_INTERFACE + "NoIpv6"); parseResults = net.parseNetInterface(mockOutput); ArrayList<ParseResult> expextedParseResults = new ArrayList<>(Arrays.asList( new ParseResult("inet", "inet"))); @@ -88,7 +91,7 @@ public class NetRetrieverTest { @Test public void parseInterfaceSpeed_get_interfaceSpeed_from_eth0_testFile() throws IOException { - ArrayList<String> mockOutput = MockCommandExecutor.readFromFile("src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/eth0"); + List<String> mockOutput = MockCommandExecutor.readFromFile("src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/eth0"); ParseResult parseResult = net.parseInterfaceSpeed(mockOutput); ParseResult expectedParseResult = new ParseResult("Speed", "1000Mb/s"); assertEquals(expectedParseResult, parseResult); @@ -117,7 +120,7 @@ public class NetRetrieverTest { @Test public void parsePingResponse_valid_ping_response_should_return_ipv6_connectivity() throws IOException { - ArrayList<String> mockCommandOutput = MockCommandExecutor.readFromFile(VALID_PING_RESPONSE); + List<String> mockCommandOutput = MockCommandExecutor.readFromFile(VALID_PING_RESPONSE); ParseResult parseResult = net.parsePingResponse(mockCommandOutput); String expectedPing = "0"; assertEquals(expectedPing, parseResult.getValue()); @@ -125,7 +128,7 @@ public class NetRetrieverTest { @Test public void parsePingResponse_invalid_ping_response_should_throw_IOException() throws IOException { - ArrayList<String> mockCommandOutput = MockCommandExecutor.readFromFile(INVALID_PING_RESPONSE); + List<String> mockCommandOutput = MockCommandExecutor.readFromFile(INVALID_PING_RESPONSE); try { ParseResult parseResult = net.parsePingResponse(mockCommandOutput); fail("Expected an IOException to be thrown"); @@ -589,12 +589,7 @@ <dependency> <groupId>junit</groupId> <artifactId>junit</artifactId> - <version>4.11</version> - </dependency> - <dependency> - <groupId>junit</groupId> - <artifactId>junit-dep</artifactId> - <version>4.11</version> + <version>4.12</version> </dependency> <dependency> <groupId>org.antlr</groupId> diff --git a/searchlib/CMakeLists.txt b/searchlib/CMakeLists.txt index c389cbdc8c6..0bd70592c3d 100644 --- a/searchlib/CMakeLists.txt +++ b/searchlib/CMakeLists.txt @@ -131,6 +131,7 @@ vespa_define_module( src/tests/features/element_similarity_feature src/tests/features/euclidean_distance src/tests/features/imported_dot_product + src/tests/features/internal_max_reduce_prod_join_feature src/tests/features/item_raw_score src/tests/features/native_dot_product src/tests/features/ranking_expression @@ -198,6 +199,7 @@ vespa_define_module( src/tests/queryeval/weak_and_scorers src/tests/queryeval/weighted_set_term src/tests/rankingexpression/feature_name_extractor + src/tests/rankingexpression/intrinsic_blueprint_adapter src/tests/ranksetup src/tests/ranksetup/verify_feature src/tests/sort diff --git a/searchlib/src/tests/features/internal_max_reduce_prod_join_feature/.gitignore b/searchlib/src/tests/features/internal_max_reduce_prod_join_feature/.gitignore new file mode 100644 index 00000000000..752721ef65a --- /dev/null +++ b/searchlib/src/tests/features/internal_max_reduce_prod_join_feature/.gitignore @@ -0,0 +1 @@ +searchlib_internal_max_reduce_prod_join_feature_test_app diff --git a/searchlib/src/tests/features/internal_max_reduce_prod_join_feature/CMakeLists.txt b/searchlib/src/tests/features/internal_max_reduce_prod_join_feature/CMakeLists.txt new file mode 100644 index 00000000000..65b8056cb54 --- /dev/null +++ b/searchlib/src/tests/features/internal_max_reduce_prod_join_feature/CMakeLists.txt @@ -0,0 +1,8 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(searchlib_internal_max_reduce_prod_join_feature_test_app TEST + SOURCES + internal_max_reduce_prod_join_feature_test.cpp + DEPENDS + searchlib +) +vespa_add_test(NAME searchlib_internal_max_reduce_prod_join_feature_test_app COMMAND searchlib_internal_max_reduce_prod_join_feature_test_app) diff --git a/searchlib/src/tests/features/internal_max_reduce_prod_join_feature/internal_max_reduce_prod_join_feature_test.cpp b/searchlib/src/tests/features/internal_max_reduce_prod_join_feature/internal_max_reduce_prod_join_feature_test.cpp new file mode 100644 index 00000000000..72e2240a12a --- /dev/null +++ b/searchlib/src/tests/features/internal_max_reduce_prod_join_feature/internal_max_reduce_prod_join_feature_test.cpp @@ -0,0 +1,173 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/vespalib/testkit/test_kit.h> + +#include <vespa/searchlib/attribute/attributefactory.h> +#include <vespa/searchlib/fef/test/ftlib.h> +#include <vespa/searchlib/features/internal_max_reduce_prod_join_feature.h> +#include <vespa/searchlib/attribute/attribute.h> + +using search::feature_t; +using namespace search::fef; +using namespace search::fef::test; +using namespace search::features; +using search::AttributeFactory; +using search::IntegerAttribute; +using search::FloatingPointAttribute; +using CollectionType = FieldInfo::CollectionType; +using DataType = FieldInfo::DataType; + +typedef search::attribute::Config AVC; +typedef search::attribute::BasicType AVBT; +typedef search::attribute::CollectionType AVCT; +typedef search::AttributeVector::SP AttributePtr; +typedef FtTestApp FTA; + +struct SetupFixture +{ + InternalMaxReduceProdJoinBlueprint blueprint; + IndexEnvironment indexEnv; + SetupFixture() + : blueprint(), + indexEnv() + { + addAttribute("long", CollectionType::SINGLE, DataType::INT64); + addAttribute("longarray", CollectionType::ARRAY, DataType::INT64); + addAttribute("intarray", CollectionType::ARRAY, DataType::INT32); + addAttribute("doublearray", CollectionType::ARRAY, DataType::DOUBLE); + } + + void addAttribute(const vespalib::string &name, const CollectionType &collType, const DataType &dataType) { + FieldInfo attrInfo(FieldType::ATTRIBUTE, collType, name, 0); + attrInfo.set_data_type(dataType); + indexEnv.getFields().push_back(attrInfo); + } +}; + +TEST_F("require that blueprint can be created", SetupFixture()) +{ + EXPECT_TRUE(FTA::assertCreateInstance(f.blueprint, "internalMaxReduceProdJoin")); +} + +TEST_F("require that setup fails if attribute does not exist", SetupFixture()) +{ + FTA::FT_SETUP_FAIL(f.blueprint, f.indexEnv, StringList().add("foo").add("bar")); +} + +TEST_F("require that setup fails if attribute is of wrong type", SetupFixture()) +{ + FTA::FT_SETUP_FAIL(f.blueprint, f.indexEnv, StringList().add("long").add("bar")); +} + +TEST_F("require that setup fails if attribute is of wrong array type", SetupFixture()) +{ + FTA::FT_SETUP_FAIL(f.blueprint, f.indexEnv, StringList().add("doublearray").add("bar")); +} + +TEST_F("require that setup succeeds with long array attribute", SetupFixture()) +{ + FTA::FT_SETUP_OK(f.blueprint, f.indexEnv, + StringList().add("longarray").add("query"), + StringList(), + StringList().add("scalar")); +} + +TEST_F("require that setup succeeds with int array attribute", SetupFixture()) +{ + FTA::FT_SETUP_OK(f.blueprint, f.indexEnv, + StringList().add("intarray").add("query"), + StringList(), + StringList().add("scalar")); +} + +struct ExecFixture +{ + BlueprintFactory factory; + FtFeatureTest test; + ExecFixture(const vespalib::string &feature) + : factory(), + test(factory, feature) + { + factory.addPrototype(std::make_shared<InternalMaxReduceProdJoinBlueprint>()); + setupAttributeVectors(); + setupQueryEnvironment(); + ASSERT_TRUE(test.setup()); + } + + void setupAttributeVectors() { + vespalib::string attrIntArray = "intarray"; + vespalib::string attrLongArray = "longarray"; + + test.getIndexEnv().getBuilder().addField(FieldType::ATTRIBUTE, CollectionType::ARRAY, DataType::INT64, attrLongArray); + test.getIndexEnv().getBuilder().addField(FieldType::ATTRIBUTE, CollectionType::ARRAY, DataType::INT32, attrIntArray); + + std::vector<AttributePtr> attrs; + attrs.push_back(AttributeFactory::createAttribute(attrLongArray, AVC(AVBT::INT64, AVCT::ARRAY))); + attrs.push_back(AttributeFactory::createAttribute(attrIntArray, AVC(AVBT::INT32, AVCT::ARRAY))); + for (const auto &attr : attrs) { + attr->addReservedDoc(); + attr->addDocs(1); + test.getIndexEnv().getAttributeMap().add(attr); + } + + IntegerAttribute *longArray = static_cast<IntegerAttribute *>(attrs[0].get()); + longArray->append(1, 1111, 0); + longArray->append(1, 2222, 0); + longArray->append(1, 78, 0); + + IntegerAttribute *intArray = static_cast<IntegerAttribute *>(attrs[1].get()); + intArray->append(1, 78, 0); + intArray->append(1, 1111, 0); + + for (const auto &attr : attrs) { + attr->commit(); + } + } + + void setupQueryEnvironment() { + test.getQueryEnv().getProperties().add("wset", "{1111:1234, 2222:2245}"); + test.getQueryEnv().getProperties().add("wsetnomatch", "{1:1000, 2:2000}"); + test.getQueryEnv().getProperties().add("array", "[1111,2222]"); + test.getQueryEnv().getProperties().add("negativewset", "{1111:-1000, 78:-42}"); + } + + bool evaluatesTo(feature_t expectedValue) { + return test.execute(expectedValue); + } + +}; + +TEST_F("require that executor returns correct result for long array", + ExecFixture("internalMaxReduceProdJoin(longarray,wset)")) +{ + EXPECT_FALSE(f.evaluatesTo(1234)); + EXPECT_TRUE(f.evaluatesTo(2245)); +} + +TEST_F("require that executor returns correct result for int array", + ExecFixture("internalMaxReduceProdJoin(intarray,wset)")) +{ + EXPECT_TRUE(f.evaluatesTo(1234)); + EXPECT_FALSE(f.evaluatesTo(2245)); +} + +TEST_F("require that executor returns 0 if no items match", + ExecFixture("internalMaxReduceProdJoin(longarray,wsetnomatch)")) +{ + EXPECT_TRUE(f.evaluatesTo(0.0)); +} + +TEST_F("require that executor return 0 if query is not a weighted set", + ExecFixture("internalMaxReduceProdJoin(longarray,array)")) +{ + EXPECT_TRUE(f.evaluatesTo(0.0)); +} + +TEST_F("require that executor supports negative numbers", + ExecFixture("internalMaxReduceProdJoin(intarray,negativewset)")) +{ + EXPECT_FALSE(f.evaluatesTo(-1000)); + EXPECT_TRUE(f.evaluatesTo(-42)); +} + +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchlib/src/tests/rankingexpression/intrinsic_blueprint_adapter/CMakeLists.txt b/searchlib/src/tests/rankingexpression/intrinsic_blueprint_adapter/CMakeLists.txt new file mode 100644 index 00000000000..d8d6ebe0504 --- /dev/null +++ b/searchlib/src/tests/rankingexpression/intrinsic_blueprint_adapter/CMakeLists.txt @@ -0,0 +1,8 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(searchlib_intrinsic_blueprint_adapter_test_app TEST + SOURCES + intrinsic_blueprint_adapter_test.cpp + DEPENDS + searchlib +) +vespa_add_test(NAME searchlib_intrinsic_blueprint_adapter_test_app COMMAND searchlib_intrinsic_blueprint_adapter_test_app) diff --git a/searchlib/src/tests/rankingexpression/intrinsic_blueprint_adapter/intrinsic_blueprint_adapter_test.cpp b/searchlib/src/tests/rankingexpression/intrinsic_blueprint_adapter/intrinsic_blueprint_adapter_test.cpp new file mode 100644 index 00000000000..b10da86dd8c --- /dev/null +++ b/searchlib/src/tests/rankingexpression/intrinsic_blueprint_adapter/intrinsic_blueprint_adapter_test.cpp @@ -0,0 +1,116 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/vespalib/testkit/test_kit.h> + +#include <vespa/searchlib/fef/blueprint.h> +#include <vespa/searchlib/features/rankingexpression/intrinsic_blueprint_adapter.h> +#include <vespa/searchlib/fef/test/indexenvironment.h> +#include <vespa/searchlib/fef/test/queryenvironment.h> +#include <set> + +using namespace search::features::rankingexpression; +using namespace search::fef::test; +using namespace search::fef; +using vespalib::Stash; +using vespalib::eval::ValueType; + +vespalib::string fail_setup = "fail_setup"; +vespalib::string extra_input = "extra_input"; +vespalib::string extra_output = "extra_output"; +vespalib::string no_output = "no_output"; +vespalib::string object_result = "object_result"; +vespalib::string error_result = "error_result"; + +struct MyExecutor : FeatureExecutor { + void execute(uint32_t) override {} +}; + +struct MyBlueprint : Blueprint { + std::set<vespalib::string> flags; + MyBlueprint() : Blueprint("my_bp"), flags() {} + MyBlueprint(const std::set<vespalib::string> &flags_in) : Blueprint("my_bp"), flags(flags_in) {} + void set(const vespalib::string &flag) { flags.insert(flag); } + bool is_set(const vespalib::string &flag) const { return (flags.count(flag) > 0); } + void visitDumpFeatures(const IIndexEnvironment &, IDumpFeatureVisitor &) const override {} + Blueprint::UP createInstance() const override { return std::make_unique<MyBlueprint>(flags); } + bool setup(const IIndexEnvironment &, const std::vector<vespalib::string> ¶ms) override { + EXPECT_EQUAL(getName(), "my_bp(foo,bar)"); + ASSERT_TRUE(params.size() == 2); + EXPECT_EQUAL(params[0], "foo"); + EXPECT_EQUAL(params[1], "bar"); + if (is_set(extra_input)) { + defineInput("my_input", AcceptInput::ANY); + } + if (!is_set(no_output)) { + if (is_set(error_result)) { + describeOutput("out", "my output", FeatureType::object(ValueType::error_type())); + } else { + if (is_set(object_result)) { + describeOutput("out", "my output", FeatureType::object(ValueType::double_type())); + } else { + describeOutput("out", "my output", FeatureType::number()); + } + } + if (is_set(extra_output)) { + describeOutput("extra", "my extra output", FeatureType::number()); + } + } + return !is_set(fail_setup); + } + FeatureExecutor &createExecutor(const IQueryEnvironment &, vespalib::Stash &stash) const override { + return stash.create<MyExecutor>(); + } +}; + +struct Fixture { + Stash stash; + IndexEnvironment idx_env; + QueryEnvironment query_env; + MyBlueprint blueprint; + Fixture() : stash(), idx_env(), query_env(&idx_env), blueprint() {} + IntrinsicExpression::UP create() const { + return IntrinsicBlueprintAdapter::try_create(blueprint, idx_env, {"foo", "bar"}); + } +}; + +TEST_F("require that blueprints can be used for intrinsic expressions", Fixture()) { + auto expression = f1.create(); + ASSERT_TRUE(bool(expression)); + EXPECT_TRUE(!expression->result_type().is_object()); + auto &executor = expression->create_executor(f1.query_env, f1.stash); + EXPECT_TRUE(dynamic_cast<MyExecutor*>(&executor) != nullptr); +} + +TEST_F("require that result type is propagated for intrinsic blueprints", Fixture()) { + f1.blueprint.set(object_result); + auto expression = f1.create(); + ASSERT_TRUE(bool(expression)); + EXPECT_TRUE(expression->result_type().is_object()); + EXPECT_TRUE(expression->result_type().type().is_double()); +} + +TEST_F("require that intrinsic blueprint adaption fails if blueprint setup fails", Fixture()) { + f1.blueprint.set(fail_setup); + EXPECT_TRUE(f1.create().get() == nullptr); +} + +TEST_F("require that intrinsic blueprint adaption fails if blueprint has inputs", Fixture()) { + f1.blueprint.set(extra_input); + EXPECT_TRUE(f1.create().get() == nullptr); +} + +TEST_F("require that intrinsic blueprint adaption fails if blueprint has more than one output", Fixture()) { + f1.blueprint.set(extra_output); + EXPECT_TRUE(f1.create().get() == nullptr); +} + +TEST_F("require that intrinsic blueprint adaption fails if blueprint has no result", Fixture()) { + f1.blueprint.set(no_output); + EXPECT_TRUE(f1.create().get() == nullptr); +} + +TEST_F("require that intrinsic blueprint adaption fails if blueprint has error typed output", Fixture()) { + f1.blueprint.set(error_result); + EXPECT_TRUE(f1.create().get() == nullptr); +} + +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchlib/src/vespa/searchlib/features/CMakeLists.txt b/searchlib/src/vespa/searchlib/features/CMakeLists.txt index 6b212bdee0f..9227b845966 100644 --- a/searchlib/src/vespa/searchlib/features/CMakeLists.txt +++ b/searchlib/src/vespa/searchlib/features/CMakeLists.txt @@ -25,6 +25,7 @@ vespa_add_library(searchlib_features OBJECT flow_completeness_feature.cpp foreachfeature.cpp freshnessfeature.cpp + internal_max_reduce_prod_join_feature.cpp item_raw_score_feature.cpp jarowinklerdistancefeature.cpp matchcountfeature.cpp diff --git a/searchlib/src/vespa/searchlib/features/internal_max_reduce_prod_join_feature.cpp b/searchlib/src/vespa/searchlib/features/internal_max_reduce_prod_join_feature.cpp new file mode 100644 index 00000000000..d07654c0a21 --- /dev/null +++ b/searchlib/src/vespa/searchlib/features/internal_max_reduce_prod_join_feature.cpp @@ -0,0 +1,246 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "internal_max_reduce_prod_join_feature.h" +#include "valuefeature.h" +#include "weighted_set_parser.h" + +#include <vespa/log/log.h> +#include <vespa/searchlib/attribute/attribute.h> +#include <vespa/searchlib/attribute/imported_attribute_vector.h> +#include <vespa/searchlib/attribute/multinumericattribute.h> +#include <vespa/searchlib/features/dotproductfeature.h> +#include <vespa/searchlib/fef/properties.h> +#include <vespa/searchlib/fef/featureexecutor.h> +#include <vespa/searchcommon/common/datatype.h> + +LOG_SETUP(".features.internalmaxreduceprodjoin"); + +using namespace search::attribute; +using namespace search::fef; + +using search::features::dotproduct::wset::IntegerVector; + +namespace search { +namespace features { + +/** + * Executor used when array can be accessed directly + */ +template <typename BaseType> +class RawExecutor : public FeatureExecutor { +protected: + const IAttributeVector *_attribute; + IntegerVector _queryVector; + +public: + RawExecutor(const IAttributeVector *attribute, const IntegerVector &queryVector); + void execute(uint32_t docId) override; +}; + +template <typename BaseType> +RawExecutor<BaseType>::RawExecutor(const IAttributeVector *attribute, + const IntegerVector &queryVector) : + FeatureExecutor(), + _attribute(attribute), + _queryVector(queryVector) +{ + _queryVector.syncMap(); +} + +template <typename A, typename V> +feature_t maxProduct(const A &array, size_t count, const V &query) +{ + feature_t val = -std::numeric_limits<double>::max(); + for (size_t i = 0; i < count; ++i) { + auto itr = query.getDimMap().find(array[i].value()); + if (itr != query.getDimMap().end()) { + feature_t v = itr->second; // weight from attribute is assumed to be 1.0 + if (v > val) { + val = v; + } + } + } + return val == -std::numeric_limits<double>::max() ? 0.0 : val; +} + +template <typename BaseType> +void +RawExecutor<BaseType>::execute(uint32_t docId) +{ + using A = IntegerAttributeTemplate<BaseType>; + const multivalue::Value<BaseType> *values(nullptr); + const A *iattr = dynamic_cast<const A *>(_attribute); + size_t count = iattr->getRawValues(docId, values); + outputs().set_number(0, maxProduct(values, count, _queryVector)); +} + +/** + * Executor when array can't be accessed directly + */ +template <typename BaseType> +class BufferedExecutor : public RawExecutor<BaseType> { +private: + WeightedIntegerContent _buffer; + +public: + BufferedExecutor(const IAttributeVector *attribute, const IntegerVector &queryVector); + void execute(uint32_t docId) override; +}; + +template <typename BaseType> +BufferedExecutor<BaseType>::BufferedExecutor(const IAttributeVector *attribute, const IntegerVector &queryVector) : + RawExecutor<BaseType>(attribute, queryVector), + _buffer() +{ +} + + +template <typename BaseType> +void +BufferedExecutor<BaseType>::execute(uint32_t docId) +{ + _buffer.fill(*(this->_attribute), docId); + this->outputs().set_number(0, maxProduct(_buffer, _buffer.size(), this->_queryVector)); +} + +/** + * Blueprint + */ +InternalMaxReduceProdJoinBlueprint::InternalMaxReduceProdJoinBlueprint() : + Blueprint("internalMaxReduceProdJoin") +{ +} + +InternalMaxReduceProdJoinBlueprint::~InternalMaxReduceProdJoinBlueprint() +{ +} + +void +InternalMaxReduceProdJoinBlueprint::visitDumpFeatures(const IIndexEnvironment &, + IDumpFeatureVisitor &) const +{ +} + +Blueprint::UP +InternalMaxReduceProdJoinBlueprint::createInstance() const +{ + return Blueprint::UP(new InternalMaxReduceProdJoinBlueprint()); +} + +ParameterDescriptions +InternalMaxReduceProdJoinBlueprint::getDescriptions() const +{ + return ParameterDescriptions().desc().attribute(ParameterCollection::ARRAY).string(); +} + +bool supportedAttributeType(Parameter param) { + const FieldInfo *attributeInfo = param.asField(); + if (attributeInfo == nullptr) { + return false; + } + if (attributeInfo->collection() != FieldInfo::CollectionType::ARRAY) { + return false; + } + if (attributeInfo->get_data_type() == FieldInfo::DataType::INT64) { + return true; + } + if (attributeInfo->get_data_type() == FieldInfo::DataType::INT32) { + return true; + } + return false; +} + +bool +InternalMaxReduceProdJoinBlueprint::setup(const IIndexEnvironment &env, const ParameterList ¶ms) +{ + if (!supportedAttributeType(params[0])) { + return false; + } + _attribute = params[0].getValue(); + _query = params[1].getValue(); + describeOutput("scalar", "Internal executor for optimized execution of reduce(join(A,Q,f(x,y)(x*y)),max)"); + env.hintAttributeAccess(_attribute); + return true; +} + +bool isImportedAttribute(const IAttributeVector &attribute) noexcept { + return dynamic_cast<const ImportedAttributeVector*>(&attribute) != nullptr; +} + +template<typename A> +bool supportsGetRawValues(const A &attr) noexcept { + try { + const multivalue::Value<typename A::BaseType> *tmp = nullptr; + attr.getRawValues(0, tmp); // Throws if unsupported + return true; + } catch (const std::runtime_error &e) { + (void) e; + return false; + } +} + +template <typename BaseType> +FeatureExecutor & +selectTypedExecutor(const IAttributeVector *attribute, const IntegerVector &vector, vespalib::Stash &stash) +{ + if (!isImportedAttribute(*attribute)) { + using A = IntegerAttributeTemplate<BaseType>; + using VT = multivalue::Value<BaseType>; + using ExactA = MultiValueNumericAttribute<A, VT>; + + const A *iattr = dynamic_cast<const A *>(attribute); + if (supportsGetRawValues(*iattr)) { + const ExactA *exactA = dynamic_cast<const ExactA *>(iattr); + if (exactA != nullptr) { + return stash.create<RawExecutor<BaseType>>(attribute, vector); + } + } + } + return stash.create<BufferedExecutor<BaseType>>(attribute, vector); +} + +FeatureExecutor & +selectExecutor(const IAttributeVector *attribute, const IntegerVector &vector, vespalib::Stash &stash) +{ + if (attribute->getCollectionType() == CollectionType::ARRAY) { + switch (attribute->getBasicType()) { + case BasicType::INT32: + return selectTypedExecutor<int32_t>(attribute, vector, stash); + case BasicType::INT64: + return selectTypedExecutor<int64_t>(attribute, vector, stash); + default: + break; + } + } + LOG(warning, "The attribute vector '%s' is not of type " + "array<int/long>, returning executor with default value.", attribute->getName().c_str()); + return stash.create<SingleZeroValueExecutor>(); +} + + +FeatureExecutor & +InternalMaxReduceProdJoinBlueprint::createExecutor(const IQueryEnvironment &env, vespalib::Stash &stash) const +{ + const IAttributeVector *attribute = env.getAttributeContext().getAttribute(_attribute); + if (attribute == nullptr) { + LOG(warning, "The attribute vector '%s' was not found in the attribute manager, " + "returning executor with default value.", + _attribute.c_str()); + return stash.create<SingleZeroValueExecutor>(); + } + Property prop = env.getProperties().lookup(_query); + if (prop.found() && !prop.get().empty()) { + IntegerVector vector; + WeightedSetParser::parse(prop.get(), vector); + if (!vector.getVector().empty()) { + return selectExecutor(attribute, vector, stash); + } + } + return stash.create<SingleZeroValueExecutor>(); +} + + +} +} + + diff --git a/searchlib/src/vespa/searchlib/features/internal_max_reduce_prod_join_feature.h b/searchlib/src/vespa/searchlib/features/internal_max_reduce_prod_join_feature.h new file mode 100644 index 00000000000..2c576b58345 --- /dev/null +++ b/searchlib/src/vespa/searchlib/features/internal_max_reduce_prod_join_feature.h @@ -0,0 +1,44 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vespa/searchlib/fef/blueprint.h> + +namespace search { +namespace features { + +/** + * Feature for the specific replacement of the expression: + * + * reduce( + * join( + * tensorFromLabels(attribute(A)), + * tensorFromWeightedset(query(Q)), + * f(x,y)(x*y) + * ), + * max + * ) + * + * where A is an array attribute of int or long type and Q is a query that parses as + * a weighted set. This expression is replaced with this feature to avoid incurring + * the cost of creating temporary tensors. + */ +class InternalMaxReduceProdJoinBlueprint : public fef::Blueprint { +private: + vespalib::string _attribute; + vespalib::string _query; + +public: + InternalMaxReduceProdJoinBlueprint(); + ~InternalMaxReduceProdJoinBlueprint(); + + fef::ParameterDescriptions getDescriptions() const override; + fef::Blueprint::UP createInstance() const override; + bool setup(const fef::IIndexEnvironment &env, const fef::ParameterList ¶ms) override; + fef::FeatureExecutor &createExecutor(const fef::IQueryEnvironment &env, vespalib::Stash &stash) const override; + void visitDumpFeatures(const fef::IIndexEnvironment &env, fef::IDumpFeatureVisitor &visitor) const override; + +}; + +} +} diff --git a/searchlib/src/vespa/searchlib/features/rankingexpression/CMakeLists.txt b/searchlib/src/vespa/searchlib/features/rankingexpression/CMakeLists.txt index 3ca8c36398c..68b4c4bb043 100644 --- a/searchlib/src/vespa/searchlib/features/rankingexpression/CMakeLists.txt +++ b/searchlib/src/vespa/searchlib/features/rankingexpression/CMakeLists.txt @@ -3,6 +3,7 @@ vespa_add_library(searchlib_features_rankingexpression OBJECT SOURCES expression_replacer.cpp feature_name_extractor.cpp + intrinsic_blueprint_adapter.cpp intrinsic_expression.cpp DEPENDS ) diff --git a/searchlib/src/vespa/searchlib/features/rankingexpression/intrinsic_blueprint_adapter.cpp b/searchlib/src/vespa/searchlib/features/rankingexpression/intrinsic_blueprint_adapter.cpp new file mode 100644 index 00000000000..92dbfb75eda --- /dev/null +++ b/searchlib/src/vespa/searchlib/features/rankingexpression/intrinsic_blueprint_adapter.cpp @@ -0,0 +1,75 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "intrinsic_blueprint_adapter.h" +#include <vespa/searchlib/fef/fef.h> + +using search::fef::Blueprint; +using search::fef::FeatureExecutor; +using search::fef::FeatureNameBuilder; +using search::fef::FeatureType; + +namespace search::features::rankingexpression { + +namespace { + +bool is_valid(const FeatureType *type) { + if (type == nullptr) { + return false; + } + if (!type->is_object()) { + return true; + } + return !type->type().is_error(); +} + +struct IntrinsicBlueprint : IntrinsicExpression { + Blueprint::UP blueprint; + FeatureType type; + IntrinsicBlueprint(Blueprint::UP blueprint_in, const FeatureType &type_in) + : blueprint(std::move(blueprint_in)), type(type_in) {} + FeatureType result_type() const override { return type; } + FeatureExecutor &create_executor(const QueryEnv &queryEnv, vespalib::Stash &stash) const override { + return blueprint->createExecutor(queryEnv, stash); + } +}; + +struct ResultTypeExtractor : Blueprint::DependencyHandler { + std::unique_ptr<FeatureType> result_type; + bool too_much; + ResultTypeExtractor() : result_type(), too_much(false) {} + const FeatureType &resolve_input(const vespalib::string &, Blueprint::AcceptInput) override { + too_much = true; + return FeatureType::number(); + } + void define_output(const vespalib::string &, const FeatureType &type) override { + too_much = (too_much || bool(result_type)); + result_type = std::make_unique<FeatureType>(type); + } + bool valid() const { return (is_valid(result_type.get()) && !too_much); } + const FeatureType &get() const { return *result_type; } +}; + +} // namespace search::features::rankingexpression::<unnamed> + +IntrinsicExpression::UP +IntrinsicBlueprintAdapter::try_create(const search::fef::Blueprint &proto, + const search::fef::IIndexEnvironment &env, + const std::vector<vespalib::string> ¶ms) +{ + FeatureNameBuilder name_builder; + ResultTypeExtractor result_type; + Blueprint::UP blueprint = proto.createInstance(); + name_builder.baseName(blueprint->getBaseName()); + for (const auto ¶m: params) { + name_builder.parameter(param); + } + blueprint->setName(name_builder.buildName()); + blueprint->attach_dependency_handler(result_type); + if (!blueprint->setup(env, params) || !result_type.valid()) { + return IntrinsicExpression::UP(nullptr); + } + blueprint->detach_dependency_handler(); + return std::make_unique<IntrinsicBlueprint>(std::move(blueprint), result_type.get()); +} + +} // namespace search::features::rankingexpression diff --git a/searchlib/src/vespa/searchlib/features/rankingexpression/intrinsic_blueprint_adapter.h b/searchlib/src/vespa/searchlib/features/rankingexpression/intrinsic_blueprint_adapter.h new file mode 100644 index 00000000000..01c6ced2391 --- /dev/null +++ b/searchlib/src/vespa/searchlib/features/rankingexpression/intrinsic_blueprint_adapter.h @@ -0,0 +1,25 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "intrinsic_expression.h" +#include <vespa/vespalib/stllike/string.h> +#include <vector> + +namespace search::fef { class Blueprint; } +namespace search::fef { class IIndexEnvironment; } + +namespace search::features::rankingexpression { + +/** + * Adapt a Blueprint with no inputs and a single output to the + * IntrinsicExpression interface. + **/ +struct IntrinsicBlueprintAdapter +{ + static IntrinsicExpression::UP try_create(const search::fef::Blueprint &proto, + const search::fef::IIndexEnvironment &env, + const std::vector<vespalib::string> ¶ms); +}; + +} // namespace search::features::rankingexpression diff --git a/searchlib/src/vespa/searchlib/fef/test/indexenvironmentbuilder.cpp b/searchlib/src/vespa/searchlib/fef/test/indexenvironmentbuilder.cpp index da27cb143cc..be78e8c45f8 100644 --- a/searchlib/src/vespa/searchlib/fef/test/indexenvironmentbuilder.cpp +++ b/searchlib/src/vespa/searchlib/fef/test/indexenvironmentbuilder.cpp @@ -1,5 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/searchcommon/common/datatype.h> #include "indexenvironmentbuilder.h" namespace search { @@ -17,8 +18,18 @@ IndexEnvironmentBuilder::addField(const FieldType &type, const FieldInfo::CollectionType &coll, const vespalib::string &name) { + return addField(type, coll, FieldInfo::DataType::DOUBLE, name); +} + +IndexEnvironmentBuilder & +IndexEnvironmentBuilder::addField(const FieldType &type, + const FieldInfo::CollectionType &coll, + const FieldInfo::DataType &dataType, + const vespalib::string &name) +{ uint32_t idx = _env.getFields().size(); FieldInfo field(type, coll, name, idx); + field.set_data_type(dataType); _env.getFields().push_back(field); return *this; } diff --git a/searchlib/src/vespa/searchlib/fef/test/indexenvironmentbuilder.h b/searchlib/src/vespa/searchlib/fef/test/indexenvironmentbuilder.h index fc7af0dc6aa..e6a81a46008 100644 --- a/searchlib/src/vespa/searchlib/fef/test/indexenvironmentbuilder.h +++ b/searchlib/src/vespa/searchlib/fef/test/indexenvironmentbuilder.h @@ -30,6 +30,19 @@ public: const FieldInfo::CollectionType &coll, const vespalib::string &name); + /** + * Add a field to the index environment with specified data type. + * + * @param type The type of field to add. + * @param coll Collection type + * @param dataType Collection base data type + * @param name The name of the field. + */ + IndexEnvironmentBuilder &addField(const FieldType &type, + const FieldInfo::CollectionType &coll, + const FieldInfo::DataType &dataType, + const vespalib::string &name); + /** Returns a reference to the index environment of this. */ IndexEnvironment &getIndexEnv() { return _env; } diff --git a/vagrant/README.md b/vagrant/README.md new file mode 100644 index 00000000000..7c69f8f22b0 --- /dev/null +++ b/vagrant/README.md @@ -0,0 +1,28 @@ + +# Create C++ dev environment on CentOS using VirtualBox and Vagrant + +## Prerequisites +* [Install VirtualBox](https://www.virtualbox.org/wiki/Downloads) +* [Install Vagrant](https://www.vagrantup.com/downloads.html) + +## Create dev environment + +### Change working directory to <vespa-source>/vagrant + cd <vespa-source>/vagrant + +### Start and provision the environment + vagrant up + +### Connect to machine via SSH +SSH agent forwarding is enabled to ensure easy interaction with GitHub inside the machine. + + vagrant ssh + +### Checkout vespa source inside machine +This is needed in order to compile and run tests fast on the local file system inside the machine. + + git clone git@github.com:vespa-engine/vespa.git + + +## Build C++ modules +Please follow the instructions described [here](../README.md). diff --git a/vagrant/Vagrantfile b/vagrant/Vagrantfile new file mode 100644 index 00000000000..a4cea6aa10f --- /dev/null +++ b/vagrant/Vagrantfile @@ -0,0 +1,38 @@ +# -*- mode: ruby -*- +# vi: set ft=ruby : + +# For a complete reference, please see the online documentation at https://docs.vagrantup.com. +Vagrant.configure("2") do |config| + + config.vm.box = "boxcutter/centos73-desktop" + + config.ssh.forward_agent = true + + config.vm.synced_folder "../dist", "/vagrant/dist" + + config.vm.provider "virtualbox" do |vb| + # Display the VirtualBox GUI when booting the machine + vb.gui = true + + vb.memory = "8192" + vb.cpus = 8 + end + + # Install required and nice-to-have packages + config.vm.provision "shell", inline: <<-SHELL + yum-config-manager --add-repo https://copr.fedorainfracloud.org/coprs/g/vespa/vespa/repo/epel-7/group_vespa-vespa-epel-7.repo + yum -y install epel-release + yum -y install centos-release-scl + yum -y install yum-utils + yum -y install git \ + ccache \ + rpm-build \ + valgrind \ + sudo \ + firefox \ + vim + yum-builddep -y /vagrant/dist/vespa.spec + echo -e "* soft nproc 409600\n* hard nproc 409600" > /etc/security/limits.d/99-nproc.conf + echo -e "* soft nofile 262144\n* hard nofile 262144" > /etc/security/limits.d/99-nofile.conf + SHELL +end diff --git a/vespa-hadoop/pom.xml b/vespa-hadoop/pom.xml index 4aa7d12e006..ea461e9783f 100644 --- a/vespa-hadoop/pom.xml +++ b/vespa-hadoop/pom.xml @@ -78,7 +78,6 @@ <dependency> <groupId>junit</groupId> <artifactId>junit</artifactId> - <version>4.12</version> <scope>test</scope> </dependency> |