summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorJon Bratseth <bratseth@yahoo-inc.com>2017-08-18 16:12:57 +0200
committerJon Bratseth <bratseth@yahoo-inc.com>2017-08-18 16:12:57 +0200
commit2d4a99bebf3c4a054556a82e9a5d44bdfc283b4c (patch)
tree1682ea0e7822ad4a97b03c0842c704d14ce656a9
parent9adb3c178a4608ec3d6931ef0a3a0e012927722a (diff)
parent5c7f956078eea4e70149ca74485380e0391fd1c7 (diff)
Merge with master
-rw-r--r--README.md19
-rw-r--r--application/pom.xml1
-rw-r--r--config_test/pom.xml2
-rw-r--r--docker/ci/build-and-test.sh51
-rwxr-xr-xdocker/ci/vespa-ci-internal.sh4
-rwxr-xr-xdocumentapi/src/main/java/com/yahoo/documentapi/SyncParameters.java33
-rwxr-xr-xdocumentapi/src/main/java/com/yahoo/documentapi/SyncSession.java40
-rwxr-xr-xdocumentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusSyncSession.java38
-rw-r--r--node-maintainer/README.md3
-rw-r--r--node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/maintainer/FileHelper.java1
-rw-r--r--node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/TODO.md30
-rw-r--r--node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/CommandExecutor.java11
-rw-r--r--node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/HostURLGenerator.java13
-rw-r--r--node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/noderepo/IPAddressVerifier.java7
-rw-r--r--node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/noderepo/NodeJsonConverter.java4
-rw-r--r--node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/noderepo/NodeRepoInfoRetriever.java7
-rw-r--r--node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/noderepo/NodeRepoJsonModel.java6
-rw-r--r--node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/parser/OutputParser.java17
-rw-r--r--node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/parser/ParseInstructions.java13
-rw-r--r--node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/parser/ParseResult.java5
-rw-r--r--node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/report/BenchmarkReport.java6
-rw-r--r--node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/report/HardwareDivergenceReport.java18
-rw-r--r--node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/report/Reporter.java (renamed from node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/report/ReportSender.java)27
-rw-r--r--node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/report/SpecVerificationReport.java6
-rw-r--r--node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/BenchmarkResultInspector.java8
-rw-r--r--node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/HardwareBenchmarker.java13
-rw-r--r--node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/README.md11
-rw-r--r--node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/Benchmark.java6
-rw-r--r--node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/BenchmarkResults.java6
-rw-r--r--node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/CPUBenchmark.java19
-rw-r--r--node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/DiskBenchmark.java13
-rw-r--r--node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/MemoryBenchmark.java13
-rw-r--r--node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/HardwareNodeComparator.java6
-rw-r--r--node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/README.md4
-rw-r--r--node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/SpecVerifier.java18
-rw-r--r--node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/VerifierSettings.java9
-rw-r--r--node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/CPURetriever.java21
-rw-r--r--node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/DiskRetriever.java24
-rw-r--r--node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/HardwareInfo.java7
-rw-r--r--node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/HardwareInfoRetriever.java7
-rw-r--r--node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/HardwareRetriever.java6
-rw-r--r--node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/MemoryRetriever.java14
-rw-r--r--node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/NetRetriever.java40
-rw-r--r--node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/commons/CommandExecutorTest.java7
-rw-r--r--node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/commons/HostURLGeneratorTest.java12
-rw-r--r--node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/commons/noderepo/IPAddressVerifierTest.java5
-rw-r--r--node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/commons/noderepo/NodeJsonConverterTest.java7
-rw-r--r--node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/commons/noderepo/NodeRepoInfoRetrieverTest.java8
-rw-r--r--node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/commons/parser/OutputParserTest.java18
-rw-r--r--node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/commons/report/BenchmarkReportTest.java5
-rw-r--r--node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/commons/report/ReportSenderTest.java70
-rw-r--r--node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/commons/report/ReporterTest.java104
-rw-r--r--node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/commons/report/SpecVerificationReportTest.java8
-rw-r--r--node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/BenchmarkResultInspectorTest.java9
-rw-r--r--node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/HardwareBenchmarkerTest.java5
-rw-r--r--node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/CPUBenchmarkTest.java11
-rw-r--r--node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/DiskBenchmarkTest.java9
-rw-r--r--node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/MemoryBenchmarkTest.java10
-rw-r--r--node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/mock/MockCommandExecutor.java11
-rw-r--r--node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/HardwareNodeComparatorTest.java6
-rw-r--r--node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/SpecVerifierTest.java12
-rw-r--r--node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/IPAddressVerifierTest.json1
-rw-r--r--node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/nodeInfoNotInterpretableHardwareDivergence.json82
-rw-r--r--node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/nodeInfoTest.json1
-rw-r--r--node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/CPURetrieverTest.java11
-rw-r--r--node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/DiskRetrieverTest.java16
-rw-r--r--node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/HardwareInfoRetrieverTest.java7
-rw-r--r--node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/MemoryRetrieverTest.java8
-rw-r--r--node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/NetRetrieverTest.java17
-rw-r--r--pom.xml7
-rw-r--r--searchlib/CMakeLists.txt2
-rw-r--r--searchlib/src/tests/features/internal_max_reduce_prod_join_feature/.gitignore1
-rw-r--r--searchlib/src/tests/features/internal_max_reduce_prod_join_feature/CMakeLists.txt8
-rw-r--r--searchlib/src/tests/features/internal_max_reduce_prod_join_feature/internal_max_reduce_prod_join_feature_test.cpp173
-rw-r--r--searchlib/src/tests/rankingexpression/intrinsic_blueprint_adapter/CMakeLists.txt8
-rw-r--r--searchlib/src/tests/rankingexpression/intrinsic_blueprint_adapter/intrinsic_blueprint_adapter_test.cpp116
-rw-r--r--searchlib/src/vespa/searchlib/features/CMakeLists.txt1
-rw-r--r--searchlib/src/vespa/searchlib/features/internal_max_reduce_prod_join_feature.cpp246
-rw-r--r--searchlib/src/vespa/searchlib/features/internal_max_reduce_prod_join_feature.h44
-rw-r--r--searchlib/src/vespa/searchlib/features/rankingexpression/CMakeLists.txt1
-rw-r--r--searchlib/src/vespa/searchlib/features/rankingexpression/intrinsic_blueprint_adapter.cpp75
-rw-r--r--searchlib/src/vespa/searchlib/features/rankingexpression/intrinsic_blueprint_adapter.h25
-rw-r--r--searchlib/src/vespa/searchlib/fef/test/indexenvironmentbuilder.cpp11
-rw-r--r--searchlib/src/vespa/searchlib/fef/test/indexenvironmentbuilder.h13
-rw-r--r--vagrant/README.md28
-rw-r--r--vagrant/Vagrantfile38
-rw-r--r--vespa-hadoop/pom.xml1
87 files changed, 1506 insertions, 359 deletions
diff --git a/README.md b/README.md
index 5988bad6787..ff67eca709b 100644
--- a/README.md
+++ b/README.md
@@ -13,22 +13,19 @@ Code licensed under the Apache 2.0 license. See [LICENSE](LICENSE) for terms.
## Get started developing
-### Set up build environment
+### Setup build environment
C++ building is supported on CentOS 7.
+We recommend using the following environment: [Create C++ dev environment on CentOS using VirtualBox and Vagrant](vagrant/README.md).
+You can also setup CentOS 7 natively and install the following build dependencies:
-#### Install required build dependencies
- sudo yum -y install epel-release centos-release-scl yum-utils
sudo yum-config-manager --add-repo https://copr.fedorainfracloud.org/coprs/g/vespa/vespa/repo/epel-7/group_vespa-vespa-epel-7.repo
- sudo yum -y --enablerepo=epel-testing install devtoolset-6-gcc-c++ devtoolset-6-libatomic-devel devtoolset-6-binutils \
- Judy-devel cmake3 ccache lz4-devel zlib-devel maven libicu-devel llvm3.9-devel \
- llvm3.9-static java-1.8.0-openjdk-devel openssl-devel rpm-build make \
- vespa-boost-devel vespa-libtorrent-devel vespa-zookeeper-c-client-devel vespa-cppunit-devel
-or use the prebuilt docker image
-
- # TODO: Add docker command
+ sudo yum -y install epel-release centos-release-scl yum-utils
+ sudo yum -y install ccache \
+ rpm-build
+ yum-builddep -y <vespa-source>/dist/vespa.spec
### Build Java modules
-Java modules can be built on any environment having Java and Maven:
+Java modules can be built on any environment having Java 8 and Maven:
sh bootstrap.sh
mvn install
diff --git a/application/pom.xml b/application/pom.xml
index 8f301c1a884..3fd9e8054e7 100644
--- a/application/pom.xml
+++ b/application/pom.xml
@@ -67,7 +67,6 @@
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
- <version>4.10</version>
<scope>test</scope>
</dependency>
<dependency>
diff --git a/config_test/pom.xml b/config_test/pom.xml
index ae909a02603..ad96b5135c3 100644
--- a/config_test/pom.xml
+++ b/config_test/pom.xml
@@ -11,7 +11,7 @@
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
- <version>4.11</version>
+ <version>4.12</version>
<scope>test</scope>
</dependency>
<dependency>
diff --git a/docker/ci/build-and-test.sh b/docker/ci/build-and-test.sh
index cfebe09877b..0f0b8be94df 100644
--- a/docker/ci/build-and-test.sh
+++ b/docker/ci/build-and-test.sh
@@ -1,7 +1,6 @@
#!/bin/bash
# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
set -e
-set -m
if [ $# -ne 4 ]; then
echo "Usage: $0 <source dir> <build dir> <log dir> <num threads>"
@@ -14,24 +13,10 @@ BUILD_DIR=$2
LOG_DIR=$3
NUM_THREADS=$4
-function bootstrap {
- source /opt/rh/devtoolset-6/enable || true
- export MAVEN_OPTS="-Xms128m -Xmx512m"
- cd "${SOURCE_DIR}" && sh ./bootstrap.sh full
- cd "${BUILD_DIR}" && cmake3 \
- -DCMAKE_INSTALL_PREFIX=/opt/vespa \
- -DJAVA_HOME=/usr/lib/jvm/java-openjdk \
- -DEXTRA_LINK_DIRECTORY="/opt/vespa-boost/lib;/opt/vespa-libtorrent/lib;/opt/vespa-zookeeper-c-client/lib;/opt/vespa-cppunit/lib;/usr/lib64/llvm3.9/lib" \
- -DEXTRA_INCLUDE_DIRECTORY="/opt/vespa-boost/include;/opt/vespa-libtorrent/include;/opt/vespa-zookeeper-c-client/include;/opt/vespa-cppunit/include;/usr/include/llvm3.9" \
- -DCMAKE_INSTALL_RPATH="/opt/vespa/lib64;/opt/vespa-boost/lib;/opt/vespa-libtorrent/lib;/opt/vespa-zookeeper-c-client/lib;/opt/vespa-cppunit/lib;/usr/lib/jvm/java-1.8.0/jre/lib/amd64/server;/usr/include/llvm3.9" \
- -DCMAKE_BUILD_RPATH=/opt/vespa/lib64 \
- -DVALGRIND_UNIT_TESTS=no \
- "${SOURCE_DIR}"
-}
-
function build_java {
cd "${SOURCE_DIR}"
- mvn install -nsu -B -T ${NUM_THREADS} -V # Should ideally split out test phase, but some unit tests fails on 'mvn test'
+ export MAVEN_OPTS="-Xms128m -Xmx512m"
+ mvn install -nsu -B -T ${NUM_THREADS} # Should ideally split out test phase, but some unit tests fails on 'mvn test'
}
function build_cpp {
@@ -40,32 +25,6 @@ function build_cpp {
ctest3 -j ${NUM_THREADS}
}
-bootstrap
-
-pids=()
-set -o pipefail
-
-# Java build and test
-# Should be waited for first, because it takes much shorter time than C++ build.
-build_java 2>&1 | tee ${LOG_DIR}/java.log &
-pids+=($!)
-
-# C++ build and test
-build_cpp 2>&1 | tee ${LOG_DIR}/cpp.log &
-pids+=($!)
-
-EXIT_CODE=0
-for pid in "${pids[@]}"; do
- wait $pid || EXIT_CODE=$?
- echo "$pid finished with status ${EXIT_CODE}"
- if [[ "${EXIT_CODE}" != 0 ]]; then
- echo "Exiting and killing remaining jobs."
- break;
- fi
-done
-set +o pipefail
-
-# Kill any remaining jobs, ignoring error when no jobs are running
-kill $(jobs -p) 2>/dev/null || true
-
-exit ${EXIT_CODE}
+bash ${SOURCE_DIR}/bootstrap-cpp.sh ${SOURCE_DIR} ${BUILD_DIR}
+build_java 2>&1 | tee ${LOG_DIR}/java.log
+build_cpp 2>&1 | tee ${LOG_DIR}/cpp.log
diff --git a/docker/ci/vespa-ci-internal.sh b/docker/ci/vespa-ci-internal.sh
index 8ba77a56a69..5067aecaa6a 100755
--- a/docker/ci/vespa-ci-internal.sh
+++ b/docker/ci/vespa-ci-internal.sh
@@ -26,5 +26,5 @@ git -c advice.detachedHead=false checkout ${GIT_COMMIT}
NUM_THREADS=$(($(nproc --all) * 2))
bash "${MAPPED_DIR}/docker/ci/build-and-test.sh" "${SOURCE_DIR}" "${BUILD_DIR}" "${LOG_DIR}" ${NUM_THREADS}
-sudo cp "${LOG_DIR}/java.log" "${MAPPED_DIR}/docker/logs/${LOG_PREFIX}-java.log"
-sudo cp "${LOG_DIR}/cpp.log" "${MAPPED_DIR}/docker/logs/${LOG_PREFIX}-cpp.log"
+cp "${LOG_DIR}/java.log" "${MAPPED_DIR}/docker/logs/${LOG_PREFIX}-java.log"
+cp "${LOG_DIR}/cpp.log" "${MAPPED_DIR}/docker/logs/${LOG_PREFIX}-cpp.log"
diff --git a/documentapi/src/main/java/com/yahoo/documentapi/SyncParameters.java b/documentapi/src/main/java/com/yahoo/documentapi/SyncParameters.java
index 5a4b4abbf1e..a7bbf1243ae 100755
--- a/documentapi/src/main/java/com/yahoo/documentapi/SyncParameters.java
+++ b/documentapi/src/main/java/com/yahoo/documentapi/SyncParameters.java
@@ -1,11 +1,42 @@
// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.documentapi;
+import java.time.temporal.TemporalAmount;
+import java.util.Optional;
+
/**
* Parameters for creating a synchronous session
*
+ * @author bjorncs
* @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
*/
public class SyncParameters extends Parameters {
- // empty
+ private final TemporalAmount defaultTimeout;
+
+ public SyncParameters() {
+ this(null);
+ }
+
+ private SyncParameters(TemporalAmount defaultTimeout) {
+ this.defaultTimeout = defaultTimeout;
+ }
+
+ public Optional<TemporalAmount> defaultTimeout() {
+ return Optional.ofNullable(defaultTimeout);
+ }
+
+ public static class Builder {
+ private TemporalAmount defaultTimeout;
+
+ /**
+ * Set default timeout for all messagebus operations.
+ */
+ public void setDefaultTimeout(TemporalAmount defaultTimeout) {
+ this.defaultTimeout = defaultTimeout;
+ }
+
+ public SyncParameters build() {
+ return new SyncParameters(defaultTimeout);
+ }
+ }
}
diff --git a/documentapi/src/main/java/com/yahoo/documentapi/SyncSession.java b/documentapi/src/main/java/com/yahoo/documentapi/SyncSession.java
index 2a985212449..e04fb7a4e07 100755
--- a/documentapi/src/main/java/com/yahoo/documentapi/SyncSession.java
+++ b/documentapi/src/main/java/com/yahoo/documentapi/SyncSession.java
@@ -5,16 +5,17 @@ import com.yahoo.document.Document;
import com.yahoo.document.DocumentId;
import com.yahoo.document.DocumentPut;
import com.yahoo.document.DocumentRemove;
-import com.yahoo.document.DocumentType;
import com.yahoo.document.DocumentUpdate;
-import com.yahoo.document.TestAndSetCondition;
import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol;
+import java.time.temporal.TemporalAmount;
+
/**
* <p>A session for synchronous access to a document repository. This class
* provides simple document access where throughput is not a concern.</p>
*
* @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ * @author bjorncs
*/
public interface SyncSession extends Session {
@@ -60,6 +61,41 @@ public interface SyncSession extends Session {
Document get(DocumentId id, String fieldSet, DocumentProtocol.Priority priority);
/**
+ * <p>Gets a document with timeout.</p>
+ *
+ * @param id The id of the document to get.
+ * @param timeout Timeout. If timeout is null, an unspecified default will be used.
+ * @return The known document having this id, or null if there is no
+ * document having this id.
+ * @throws UnsupportedOperationException Thrown if this access does not
+ * support retrieving.
+ * @throws RuntimeException If operation times out.
+ */
+ // TODO Vespa 7: Remove default implementation. Consider removing get() overloads without timeout.
+ default Document get(DocumentId id, TemporalAmount timeout) {
+ return get(id);
+ }
+
+ /**
+ * <p>Gets a document with timeout. </p>
+ *
+ * @param id The id of the document to get.
+ * @param fieldSet A comma-separated list of fields to retrieve
+ * @param priority The priority with which to perform this operation.
+ * @param timeout Timeout. If timeout is null, an unspecified default will be used.
+ * @return The known document having this id, or null if there is no
+ * document having this id.
+ * @throws UnsupportedOperationException Thrown if this access does not
+ * support retrieving.
+ * * @throws RuntimeException If operation times out.
+ */
+ // TODO Vespa 7: Remove default implementation. Consider removing get() overloads without timeout.
+ default Document get(DocumentId id, String fieldSet, DocumentProtocol.Priority priority,
+ TemporalAmount timeout) {
+ return get(id, fieldSet, priority);
+ }
+
+ /**
* <p>Removes a document if it is present and condition is fulfilled.</p>
* @param documentRemove document to delete
* @return true If the document with this id was removed, false otherwise.
diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusSyncSession.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusSyncSession.java
index 2d0f1d9db82..f2b1816a410 100755
--- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusSyncSession.java
+++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusSyncSession.java
@@ -12,21 +12,32 @@ import com.yahoo.documentapi.Response;
import com.yahoo.documentapi.Result;
import com.yahoo.documentapi.SyncParameters;
import com.yahoo.documentapi.SyncSession;
-import com.yahoo.documentapi.messagebus.protocol.*;
import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol;
+import com.yahoo.documentapi.messagebus.protocol.GetDocumentMessage;
+import com.yahoo.documentapi.messagebus.protocol.GetDocumentReply;
+import com.yahoo.documentapi.messagebus.protocol.PutDocumentMessage;
+import com.yahoo.documentapi.messagebus.protocol.RemoveDocumentMessage;
+import com.yahoo.documentapi.messagebus.protocol.RemoveDocumentReply;
+import com.yahoo.documentapi.messagebus.protocol.UpdateDocumentMessage;
+import com.yahoo.documentapi.messagebus.protocol.UpdateDocumentReply;
import com.yahoo.messagebus.Message;
import com.yahoo.messagebus.MessageBus;
import com.yahoo.messagebus.Reply;
import com.yahoo.messagebus.ReplyHandler;
+import java.time.temporal.ChronoUnit;
+import java.time.temporal.TemporalAmount;
+
/**
* An implementation of the SyncSession interface running over message bus.
*
* @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ * @author bjorncs
*/
public class MessageBusSyncSession implements MessageBusSession, SyncSession, ReplyHandler {
- private MessageBusAsyncSession session;
+ private final MessageBusAsyncSession session;
+ private final TemporalAmount defaultTimeout;
/**
* Creates a new sync session running on message bus logic.
@@ -37,6 +48,7 @@ public class MessageBusSyncSession implements MessageBusSession, SyncSession, Re
*/
MessageBusSyncSession(SyncParameters syncParams, MessageBus bus, MessageBusParams mbusParams) {
session = new MessageBusAsyncSession(new AsyncParameters(), bus, mbusParams, this);
+ defaultTimeout = syncParams.defaultTimeout().orElse(null);
}
@Override
@@ -72,6 +84,13 @@ public class MessageBusSyncSession implements MessageBusSession, SyncSession, Re
* @return The reply received.
*/
public Reply syncSend(Message msg) {
+ return syncSend(msg, defaultTimeout);
+ }
+
+ private Reply syncSend(Message msg, TemporalAmount timeout) {
+ if (timeout != null) {
+ msg.setTimeRemaining(timeout.get(ChronoUnit.MILLIS));
+ }
try {
RequestMonitor monitor = new RequestMonitor();
msg.setContext(monitor);
@@ -107,15 +126,26 @@ public class MessageBusSyncSession implements MessageBusSession, SyncSession, Re
@Override
public Document get(DocumentId id) {
- return get(id, "[all]", DocumentProtocol.Priority.NORMAL_1);
+ return get(id, null);
}
@Override
public Document get(DocumentId id, String fieldSet, DocumentProtocol.Priority pri) {
+ return get(id, fieldSet, pri, null);
+ }
+
+ @Override
+ public Document get(DocumentId id, TemporalAmount timeout) {
+ return get(id, "[all]", DocumentProtocol.Priority.NORMAL_1, timeout);
+ }
+
+ @Override
+ public Document get(DocumentId id, String fieldSet, DocumentProtocol.Priority pri,
+ TemporalAmount timeout) {
GetDocumentMessage msg = new GetDocumentMessage(id, fieldSet);
msg.setPriority(pri);
- Reply reply = syncSend(msg);
+ Reply reply = syncSend(msg, timeout != null ? timeout : defaultTimeout);
if (reply.hasErrors()) {
throw new DocumentAccessException(MessageBusAsyncSession.getErrorMessage(reply));
}
diff --git a/node-maintainer/README.md b/node-maintainer/README.md
index 7743df2b0c2..4a32f207fdf 100644
--- a/node-maintainer/README.md
+++ b/node-maintainer/README.md
@@ -16,4 +16,5 @@ SpecVerifier:
HardwareBenchmarker:
- sudo java -cp node-maintainer-jar-with-dependencies.jar com.yahoo.vespa.hosted.node.verification.hardware.HardwareBenchmarker cfg.1.hostname,cfg.2.hostname,...
-Both programs have README explaining closer what it does. \ No newline at end of file
+In "verification" you can find a TODO file where we have listed things we did not have time to implement.
+Both programs have README explaining closer what it does.
diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/maintainer/FileHelper.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/maintainer/FileHelper.java
index 2274baf632e..db84f6af496 100644
--- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/maintainer/FileHelper.java
+++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/maintainer/FileHelper.java
@@ -20,7 +20,6 @@ import java.util.stream.Collectors;
/**
* @author freva
*/
-
public class FileHelper {
private static final Logger logger = Logger.getLogger(FileHelper.class.getSimpleName());
diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/TODO.md b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/TODO.md
new file mode 100644
index 00000000000..846c52ff645
--- /dev/null
+++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/TODO.md
@@ -0,0 +1,30 @@
+#TODO
+Here we have listed things we did not have time to do during the summer, but that we think can be implemented later.
+
+##Spec
+- The net interface speed is at the moment approved if it is over 1 000MB/s. Since some nodes are supposed to have
+10 000MB/s a feature could be to either add information about interface speed in node repo or use the flavor to decide
+if the interface speed is correct or not.
+- In HardwareNodeComparator the spec found in node repo and on the node are compared. We set a threshold on 5%, meaning
+if a the value from the node is more than 5% away from what is said in node repo (+-), then we say it is a bad value.
+If this threshold is too high or too low, it has to be changed in this class.
+
+##Benchmark
+- BenchmarkResultInspector is the class that decides whether a benchmark result is ok or if the result should be
+ reported. The values that decides this are not given very much thought and it could be an idea to check these.
+- Benchmark is not running on docker hosts since there is not yet found a solution to only start benchmarking at reboot.
+Spec verification runs every hour on node-admin, but since node-admin reboots too often, benchmarking is at the moment
+not running here.
+
+##Reporting / Node repo
+- Since HardwareDivergenceReport is printed as a json string and then reported to noderepo as a string, a feature could
+be to not store HardwareDivergence as a string in node repo, as it is now, but as a json. This will be mostly changes
+outside of this code, but then the member variable HardwareDivergence in NodeRepoJsonModel have to be change from string
+to HardwareDivergenceReport which will cause some work in the Reporter class.
+- No actions are now taken if there is anything wrong with the node. The information is only uploaded to node repo.
+Here it is room for improvements and some possible solutions are:
+ - Continue only reporting to node repo, but have a program that scans through all nodes and makes a list of those
+ with errors in HardwareDivergence.
+ - In addition to uploading the report to node repo, have actions based on what kind of errors and what kind of state
+ the node has.
+ - Automatically create Jira tickets when something is reported wrong on a node. \ No newline at end of file
diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/CommandExecutor.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/CommandExecutor.java
index f783af4c273..21e54c8c010 100644
--- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/CommandExecutor.java
+++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/CommandExecutor.java
@@ -10,16 +10,19 @@ import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.StringReader;
import java.util.ArrayList;
+import java.util.List;
/**
- * Created by olaa on 03/07/2017.
* Wrapper for executing terminal commands
+ *
+ * @author olaaun
+ * @author sgrostad
*/
public class CommandExecutor {
- public ArrayList<String> executeCommand(String command) throws IOException {
+ public List<String> executeCommand(String command) throws IOException {
ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
- ArrayList<String> results = new ArrayList<>();
+ List<String> results = new ArrayList<>();
writeToOutputStream(outputStream, command);
writeOutputStreamToResults(outputStream, results);
return results;
@@ -35,7 +38,7 @@ public class CommandExecutor {
executor.execute(cmdLine);
}
- private void writeOutputStreamToResults(ByteArrayOutputStream outputStream, ArrayList<String> results) throws IOException {
+ private void writeOutputStreamToResults(ByteArrayOutputStream outputStream, List<String> results) throws IOException {
String out = outputStream.toString();
BufferedReader br = new BufferedReader(new StringReader(out));
String line;
diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/HostURLGenerator.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/HostURLGenerator.java
index 6a6a1b4157a..7e8b198428f 100644
--- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/HostURLGenerator.java
+++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/HostURLGenerator.java
@@ -1,16 +1,17 @@
// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.hosted.node.verification.commons;
-import com.yahoo.vespa.hosted.node.verification.commons.CommandExecutor;
-
import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.ArrayList;
+import java.util.List;
/**
- * Created by olaa on 14/07/2017.
* Makes the URL used to retrieve the JSON from the node repository with information about the node's spec.
+ *
+ * @author olaaun
+ * @author sgrostad
*/
public class HostURLGenerator {
@@ -20,8 +21,8 @@ public class HostURLGenerator {
private static final String PARSE_ALL_HOSTNAMES_REGEX = ",";
private static final String PROTOCOL_REGEX = "^(https?|file)://.*$";
- public static ArrayList<URL> generateNodeInfoUrl(CommandExecutor commandExecutor, String commaSeparatedUrls) throws IOException {
- ArrayList<URL> nodeInfoUrls = new ArrayList<>();
+ public static List<URL> generateNodeInfoUrl(CommandExecutor commandExecutor, String commaSeparatedUrls) throws IOException {
+ List<URL> nodeInfoUrls = new ArrayList<>();
String[] configServerHostNames = commaSeparatedUrls.split(PARSE_ALL_HOSTNAMES_REGEX);
String nodeHostName = generateNodeHostName(commandExecutor);
for (String configServerHostName : configServerHostNames) {
@@ -36,7 +37,7 @@ public class HostURLGenerator {
}
protected static String getEnvironmentVariable(CommandExecutor commandExecutor) throws IOException {
- ArrayList<String> output = commandExecutor.executeCommand("hostname");
+ List<String> output = commandExecutor.executeCommand("hostname");
if (output.size() == 1) {
return output.get(0);
}
diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/noderepo/IPAddressVerifier.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/noderepo/IPAddressVerifier.java
index 483c07002dd..db4eb6e73cd 100644
--- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/noderepo/IPAddressVerifier.java
+++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/noderepo/IPAddressVerifier.java
@@ -12,14 +12,17 @@ import javax.naming.directory.InitialDirContext;
import java.util.ArrayList;
import java.util.Enumeration;
import java.util.Hashtable;
+import java.util.List;
import java.util.logging.Level;
import java.util.logging.Logger;
/**
* Created by olaa on 14/07/2017.
* Verifies that the IP addresses of a node points to the correct hostname
+ *
+ * @author olaaun
+ * @author sgrostad
*/
-
public class IPAddressVerifier {
private static final Logger logger = Logger.getLogger(IPAddressVerifier.class.getName());
@@ -33,7 +36,7 @@ public class IPAddressVerifier {
public String[] getFaultyIpAddresses(NodeRepoJsonModel jsonModel) {
String expectedHostname = jsonModel.getHostname();
- ArrayList<String> faultyIpAddresses = new ArrayList<>();
+ List<String> faultyIpAddresses = new ArrayList<>();
if (expectedHostname == null || expectedHostname.equals(""))
return new String[0];
if (!isValidIpv4(jsonModel.getIpv4Address(), expectedHostname)) {
diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/noderepo/NodeJsonConverter.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/noderepo/NodeJsonConverter.java
index ef6d706a1a4..5cf4bede3f1 100644
--- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/noderepo/NodeJsonConverter.java
+++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/noderepo/NodeJsonConverter.java
@@ -4,8 +4,10 @@ package com.yahoo.vespa.hosted.node.verification.commons.noderepo;
import com.yahoo.vespa.hosted.node.verification.spec.retrievers.HardwareInfo;
/**
- * Created by olaa on 07/07/2017.
* Converts a NodeRepoJsonModel object to a HardwareInfo object.
+ *
+ * @author olaaun
+ * @author sgrostad
*/
public class NodeJsonConverter {
diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/noderepo/NodeRepoInfoRetriever.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/noderepo/NodeRepoInfoRetriever.java
index 4d261334d35..57506e44961 100644
--- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/noderepo/NodeRepoInfoRetriever.java
+++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/noderepo/NodeRepoInfoRetriever.java
@@ -6,18 +6,21 @@ import com.fasterxml.jackson.databind.ObjectMapper;
import java.io.IOException;
import java.net.URL;
import java.util.ArrayList;
+import java.util.List;
import java.util.logging.Level;
import java.util.logging.Logger;
/**
- * Created by olaa on 04/07/2017.
* Parse JSON from node repository and stores information as a NodeRepoJsonModel object.
+ *
+ * @author olaaun
+ * @author sgrostad
*/
public class NodeRepoInfoRetriever {
private static final Logger logger = Logger.getLogger(NodeRepoInfoRetriever.class.getName());
- public static NodeRepoJsonModel retrieve(ArrayList<URL> nodeInfoUrls) throws IOException {
+ public static NodeRepoJsonModel retrieve(List<URL> nodeInfoUrls) throws IOException {
NodeRepoJsonModel nodeRepoJsonModel;
ObjectMapper objectMapper = new ObjectMapper();
for (URL nodeInfoURL : nodeInfoUrls) {
diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/noderepo/NodeRepoJsonModel.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/noderepo/NodeRepoJsonModel.java
index 043cccb681c..5d7ff0df8d1 100644
--- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/noderepo/NodeRepoJsonModel.java
+++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/noderepo/NodeRepoJsonModel.java
@@ -13,12 +13,14 @@ import java.net.InetAddress;
import java.util.stream.Stream;
/**
- * Created by olaa on 05/07/2017.
* Object with the information node repositories has about the node.
+ *
+ * @author olaaun
+ * @author sgrostad
*/
-
@JsonIgnoreProperties(ignoreUnknown = true)
public class NodeRepoJsonModel {
+
@JsonProperty("minDiskAvailableGb")
private double minDiskAvailableGb;
@JsonProperty("minMainMemoryAvailableGb")
diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/parser/OutputParser.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/parser/OutputParser.java
index 88e1d22cc0e..5da5abd4ad2 100644
--- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/parser/OutputParser.java
+++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/parser/OutputParser.java
@@ -2,19 +2,22 @@
package com.yahoo.vespa.hosted.node.verification.commons.parser;
import java.util.ArrayList;
+import java.util.List;
import java.util.regex.Pattern;
/**
- * Created by sgrostad on 17/07/2017.
* Parses terminal command output, and returns results based on ParseInstructions
+ *
+ * @author sgrostad
+ * @author olaaaun
*/
public class OutputParser {
- public static ArrayList<ParseResult> parseOutput(ParseInstructions parseInstructions, ArrayList<String> commandOutput) {
- ArrayList<ParseResult> results = new ArrayList<>();
+ public static List<ParseResult> parseOutput(ParseInstructions parseInstructions, List<String> commandOutput) {
+ List<ParseResult> results = new ArrayList<>();
int searchElementIndex = parseInstructions.getSearchElementIndex();
int valueElementIndex = parseInstructions.getValueElementIndex();
- ArrayList<String> searchWords = parseInstructions.getSearchWords();
+ List<String> searchWords = parseInstructions.getSearchWords();
for (String line : commandOutput) {
String[] lineSplit = line.trim().split(parseInstructions.getSplitRegex());
if (lineSplit.length <= Math.max(searchElementIndex, valueElementIndex)) {
@@ -30,15 +33,15 @@ public class OutputParser {
return results;
}
- public static ParseResult parseSingleOutput(ParseInstructions parseInstructions, ArrayList<String> commandOutput) {
- ArrayList<ParseResult> parseResults = parseOutput(parseInstructions, commandOutput);
+ public static ParseResult parseSingleOutput(ParseInstructions parseInstructions, List<String> commandOutput) {
+ List<ParseResult> parseResults = parseOutput(parseInstructions, commandOutput);
if (parseResults.size() == 0) {
return new ParseResult("invalid", "invalid");
}
return parseResults.get(0);
}
- private static boolean matchingSearchWord(ArrayList<String> searchWords, String searchWordCandidate) {
+ private static boolean matchingSearchWord(List<String> searchWords, String searchWordCandidate) {
return searchWords.stream().anyMatch(w -> Pattern.compile(w).matcher(searchWordCandidate).matches());
}
diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/parser/ParseInstructions.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/parser/ParseInstructions.java
index 6fb962c8232..262bfcba0b9 100644
--- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/parser/ParseInstructions.java
+++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/parser/ParseInstructions.java
@@ -1,19 +1,22 @@
// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.hosted.node.verification.commons.parser;
-import java.util.ArrayList;
+import java.util.List;
/**
- * Created by sgrostad on 17/07/2017.
+ * Contains instructions of how a command line output should be parsed
+ *
+ * @author sgrostad
+ * @author olaaaun
*/
public class ParseInstructions {
private final int searchElementIndex;
private final int valueElementIndex;
private final String splitRegex;
- private final ArrayList<String> searchWords;
+ private final List<String> searchWords;
- public ParseInstructions(int searchElementIndex, int returnElementNum, String splitRegex, ArrayList<String> searchWords) {
+ public ParseInstructions(int searchElementIndex, int returnElementNum, String splitRegex, List<String> searchWords) {
this.searchElementIndex = searchElementIndex;
this.valueElementIndex = returnElementNum;
this.splitRegex = splitRegex;
@@ -32,7 +35,7 @@ public class ParseInstructions {
return splitRegex;
}
- public ArrayList<String> getSearchWords() {
+ public List<String> getSearchWords() {
return searchWords;
}
diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/parser/ParseResult.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/parser/ParseResult.java
index df16897066b..a89cfd89f31 100644
--- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/parser/ParseResult.java
+++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/parser/ParseResult.java
@@ -4,7 +4,10 @@ package com.yahoo.vespa.hosted.node.verification.commons.parser;
import java.util.Objects;
/**
- * Created by sgrostad on 17/07/2017.
+ * Contains the results from parsing a command line output
+ *
+ * @author sgrostad
+ * @author olaaun
*/
public class ParseResult {
diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/report/BenchmarkReport.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/report/BenchmarkReport.java
index 0cfe3c86b92..ce4e3685a25 100644
--- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/report/BenchmarkReport.java
+++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/report/BenchmarkReport.java
@@ -8,8 +8,10 @@ import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
/**
- * Created by sgrostad on 12/07/2017.
* JSON-mapped class for reporting benchmark results to node repo
+ *
+ * @author sgrostad
+ * @author olaaun
*/
@JsonInclude(JsonInclude.Include.NON_NULL)
public class BenchmarkReport {
@@ -62,7 +64,7 @@ public class BenchmarkReport {
try {
String jsonReport = om.writeValueAsString(this);
return jsonReport.length() == 2;
- } catch (JsonProcessingException e){
+ } catch (JsonProcessingException e) {
e.printStackTrace();
return false;
}
diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/report/HardwareDivergenceReport.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/report/HardwareDivergenceReport.java
index 4536649ae25..11830c24cef 100644
--- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/report/HardwareDivergenceReport.java
+++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/report/HardwareDivergenceReport.java
@@ -5,6 +5,12 @@ import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonProperty;
+/**
+ * JSON-wrapped report for node repo
+ *
+ * @author sgrostad
+ * @author olaaun
+ */
@JsonInclude(JsonInclude.Include.NON_NULL)
public class HardwareDivergenceReport {
@@ -15,10 +21,9 @@ public class HardwareDivergenceReport {
BenchmarkReport benchmarkReport;
public void setSpecVerificationReport(SpecVerificationReport specVerificationReport) {
- if (specVerificationReport.isValidSpec()){
+ if (specVerificationReport.isValidSpec()) {
this.specVerificationReport = null;
- }
- else {
+ } else {
this.specVerificationReport = specVerificationReport;
}
}
@@ -26,15 +31,14 @@ public class HardwareDivergenceReport {
public void setBenchmarkReport(BenchmarkReport benchmarkReport) {
if (benchmarkReport.isAllBenchmarksOK()) {
this.benchmarkReport = null;
- }
- else {
+ } else {
this.benchmarkReport = benchmarkReport;
}
}
@JsonIgnore
- public boolean isHardwareDivergenceReportEmpty(){
- if (specVerificationReport == null && benchmarkReport == null){
+ public boolean isHardwareDivergenceReportEmpty() {
+ if (specVerificationReport == null && benchmarkReport == null) {
return true;
}
return false;
diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/report/ReportSender.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/report/Reporter.java
index 45a4b8cacb8..11835cab14b 100644
--- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/report/ReportSender.java
+++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/report/Reporter.java
@@ -2,45 +2,50 @@
package com.yahoo.vespa.hosted.node.verification.commons.report;
import com.fasterxml.jackson.databind.ObjectMapper;
-
import com.yahoo.vespa.hosted.node.verification.commons.noderepo.NodeRepoInfoRetriever;
import com.yahoo.vespa.hosted.node.verification.commons.noderepo.NodeRepoJsonModel;
import java.io.IOException;
import java.net.URL;
import java.util.ArrayList;
+import java.util.List;
import java.util.logging.Level;
import java.util.logging.Logger;
-public class ReportSender {
+/**
+ * Responsible for printing hardware divergence report to standard out
+ *
+ * @author sgrostad
+ * @author olaaun
+ */
+public class Reporter {
- private static final Logger logger = Logger.getLogger(ReportSender.class.getName());
+ private static final Logger logger = Logger.getLogger(Reporter.class.getName());
private static void printHardwareDivergenceReport(HardwareDivergenceReport hardwareDivergenceReport) throws IOException {
ObjectMapper om = new ObjectMapper();
String report;
- if (hardwareDivergenceReport.isHardwareDivergenceReportEmpty()){
+ if (hardwareDivergenceReport.isHardwareDivergenceReportEmpty()) {
report = "null";
- }
- else {
+ } else {
report = om.writeValueAsString(hardwareDivergenceReport);
}
System.out.print(report);
}
- public static void reportBenchmarkResults(BenchmarkReport benchmarkReport, ArrayList<URL> nodeInfoUrls) throws IOException {
+ public static void reportBenchmarkResults(BenchmarkReport benchmarkReport, List<URL> nodeInfoUrls) throws IOException {
HardwareDivergenceReport hardwareDivergenceReport = generateHardwareDivergenceReport(nodeInfoUrls);
hardwareDivergenceReport.setBenchmarkReport(benchmarkReport);
printHardwareDivergenceReport(hardwareDivergenceReport);
}
- public static void reportSpecVerificationResults(SpecVerificationReport specVerificationReport, ArrayList<URL> nodeInfoUrls) throws IOException {
+ public static void reportSpecVerificationResults(SpecVerificationReport specVerificationReport, List<URL> nodeInfoUrls) throws IOException {
HardwareDivergenceReport hardwareDivergenceReport = generateHardwareDivergenceReport(nodeInfoUrls);
hardwareDivergenceReport.setSpecVerificationReport(specVerificationReport);
printHardwareDivergenceReport(hardwareDivergenceReport);
}
- private static HardwareDivergenceReport generateHardwareDivergenceReport(ArrayList<URL> nodeInfoUrls) throws IOException {
+ private static HardwareDivergenceReport generateHardwareDivergenceReport(List<URL> nodeInfoUrls) throws IOException {
NodeRepoJsonModel nodeRepoJsonModel = NodeRepoInfoRetriever.retrieve(nodeInfoUrls);
ObjectMapper om = new ObjectMapper();
if (nodeRepoJsonModel.getHardwareDivergence() == null || nodeRepoJsonModel.getHardwareDivergence().equals("null")) {
@@ -49,10 +54,10 @@ public class ReportSender {
try {
HardwareDivergenceReport hardwareDivergenceReport = om.readValue(nodeRepoJsonModel.getHardwareDivergence(), HardwareDivergenceReport.class);
return hardwareDivergenceReport;
- }
- catch (IOException e){
+ } catch (IOException e) {
logger.log(Level.WARNING, "Failed to parse hardware divergence report from node repo. Report:\n" + nodeRepoJsonModel.getHardwareDivergence(), e.getMessage());
return new HardwareDivergenceReport();
}
}
+
}
diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/report/SpecVerificationReport.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/report/SpecVerificationReport.java
index 3eca8f38e54..ef99208a7ce 100644
--- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/report/SpecVerificationReport.java
+++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/report/SpecVerificationReport.java
@@ -13,6 +13,9 @@ import com.yahoo.vespa.hosted.node.verification.spec.retrievers.HardwareInfo;
* Stores results of caomparing node repo spec and actual hardware info.
* In case of divergent values, set the corresponding attribute to the actual hardware info value.
* Attributes of equal value remain null.
+ *
+ * @author sgrostad
+ * @author olaaun
*/
@JsonInclude(JsonInclude.Include.NON_NULL)
public class SpecVerificationReport {
@@ -66,8 +69,7 @@ public class SpecVerificationReport {
try {
String jsonReport = om.writeValueAsString(this);
return jsonReport.length() == 2;
- }
- catch (JsonProcessingException e){
+ } catch (JsonProcessingException e) {
e.printStackTrace();
return false;
}
diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/BenchmarkResultInspector.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/BenchmarkResultInspector.java
index 2cd73b46cee..4989c574ed4 100644
--- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/BenchmarkResultInspector.java
+++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/BenchmarkResultInspector.java
@@ -1,9 +1,15 @@
// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.hosted.node.verification.hardware;
-import com.yahoo.vespa.hosted.node.verification.hardware.benchmarks.BenchmarkResults;
import com.yahoo.vespa.hosted.node.verification.commons.report.BenchmarkReport;
+import com.yahoo.vespa.hosted.node.verification.hardware.benchmarks.BenchmarkResults;
+/**
+ * Responsible for checking the benchmarks results, and adding unreasonable results to BenchmarkReport
+ *
+ * @author sgrostad
+ * @author olaaun
+ */
public class BenchmarkResultInspector {
private static final double CPU_FREQUENCY_LOWER_LIMIT = 0.5;
diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/HardwareBenchmarker.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/HardwareBenchmarker.java
index e589ace6988..7e151a6a87e 100644
--- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/HardwareBenchmarker.java
+++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/HardwareBenchmarker.java
@@ -4,18 +4,19 @@ package com.yahoo.vespa.hosted.node.verification.hardware;
import com.yahoo.log.LogSetup;
import com.yahoo.vespa.hosted.node.verification.commons.CommandExecutor;
import com.yahoo.vespa.hosted.node.verification.commons.HostURLGenerator;
-import com.yahoo.vespa.hosted.node.verification.commons.report.ReportSender;
+import com.yahoo.vespa.hosted.node.verification.commons.report.BenchmarkReport;
+import com.yahoo.vespa.hosted.node.verification.commons.report.Reporter;
import com.yahoo.vespa.hosted.node.verification.hardware.benchmarks.Benchmark;
import com.yahoo.vespa.hosted.node.verification.hardware.benchmarks.BenchmarkResults;
import com.yahoo.vespa.hosted.node.verification.hardware.benchmarks.CPUBenchmark;
import com.yahoo.vespa.hosted.node.verification.hardware.benchmarks.DiskBenchmark;
import com.yahoo.vespa.hosted.node.verification.hardware.benchmarks.MemoryBenchmark;
-import com.yahoo.vespa.hosted.node.verification.commons.report.BenchmarkReport;
import java.io.IOException;
import java.net.URL;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.List;
import java.util.logging.Level;
import java.util.logging.Logger;
@@ -26,9 +27,9 @@ public class HardwareBenchmarker {
private static final Logger logger = Logger.getLogger(HardwareBenchmarker.class.getName());
- public static boolean hardwareBenchmarks(CommandExecutor commandExecutor, ArrayList<URL> nodeInfoUrls) throws IOException {
+ public static boolean hardwareBenchmarks(CommandExecutor commandExecutor, List<URL> nodeInfoUrls) throws IOException {
BenchmarkResults benchmarkResults = new BenchmarkResults();
- ArrayList<Benchmark> benchmarks = new ArrayList<>(Arrays.asList(
+ List<Benchmark> benchmarks = new ArrayList<>(Arrays.asList(
new DiskBenchmark(benchmarkResults, commandExecutor),
new CPUBenchmark(benchmarkResults, commandExecutor),
new MemoryBenchmark(benchmarkResults, commandExecutor)));
@@ -36,14 +37,14 @@ public class HardwareBenchmarker {
benchmark.doBenchmark();
}
BenchmarkReport benchmarkReport = BenchmarkResultInspector.makeBenchmarkReport(benchmarkResults);
- ReportSender.reportBenchmarkResults(benchmarkReport, nodeInfoUrls);
+ Reporter.reportBenchmarkResults(benchmarkReport, nodeInfoUrls);
return benchmarkReport.isAllBenchmarksOK();
}
public static void main(String[] args) throws IOException {
LogSetup.initVespaLogging("hardware-benchmarker");
CommandExecutor commandExecutor = new CommandExecutor();
- ArrayList<URL> nodeInfoUrls;
+ List<URL> nodeInfoUrls;
if (args.length == 0) {
throw new IllegalStateException("Expected config server URL as parameter");
}
diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/README.md b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/README.md
index e9fa4901814..ff580db7f73 100644
--- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/README.md
+++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/README.md
@@ -3,6 +3,11 @@ Verification of behaviour and performance of hardware. Benchmarks cpu frequency,
A report is sent to the node repository if any of the results are below an accepted threshold.
## Code Walkthrough
-The main class, HardwareBenchmarker, calls every benchmark in the benchmark package. The results of these benchmarks are passed through
-the BenchmarkResultInspector, which creates a BenchmarkReport containing the values below the accepted threshold. ReportSender is then called such that
-the node repository is updated with the new benchmark results. \ No newline at end of file
+The main class, HardwareBenchmarker, calls every benchmark in the benchmark package.
+
+The results of these benchmarks are passed through
+the BenchmarkResultInspector, which creates a BenchmarkReport containing the values below the accepted threshold.
+
+ReportSender is then called such that
+the old HardwareDivergence report is retrieved from node repo and updated with the new results. ReportSender prints the new HardwareReport such that it
+can be updated in node repo by chef or other. \ No newline at end of file
diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/Benchmark.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/Benchmark.java
index c1791ef45bd..1e5b512f492 100644
--- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/Benchmark.java
+++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/Benchmark.java
@@ -2,10 +2,14 @@
package com.yahoo.vespa.hosted.node.verification.hardware.benchmarks;
/**
- * Created by sgrostad on 11/07/2017.
+ * @author sgrostad
+ * @author olaaun
*/
public interface Benchmark {
+ /**
+ * Should perform benchmark for some part of the hardware, and store the result in BenchmarkResults instance passed to class
+ */
void doBenchmark();
}
diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/BenchmarkResults.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/BenchmarkResults.java
index 999af15da09..dfa436460e1 100644
--- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/BenchmarkResults.java
+++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/BenchmarkResults.java
@@ -2,9 +2,12 @@
package com.yahoo.vespa.hosted.node.verification.hardware.benchmarks;
/**
- * Created by sgrostad on 11/07/2017.
* Stores results from benchmarks
+ *
+ * @author sgrostad
+ * @author olaaun
*/
+// TODO: This should be immutable
public class BenchmarkResults {
private double cpuCyclesPerSec;
@@ -12,7 +15,6 @@ public class BenchmarkResults {
private double memoryWriteSpeedGBs;
private double memoryReadSpeedGBs;
-
public double getMemoryWriteSpeedGBs() {
return memoryWriteSpeedGBs;
}
diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/CPUBenchmark.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/CPUBenchmark.java
index e70385bb3a2..a55a92ba8a3 100644
--- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/CPUBenchmark.java
+++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/CPUBenchmark.java
@@ -9,11 +9,15 @@ import com.yahoo.vespa.hosted.node.verification.commons.parser.ParseResult;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.List;
import java.util.logging.Level;
import java.util.logging.Logger;
/**
- * Created by sgrostad on 11/07/2017.
+ * Responsible for benchmarking CPU clock frequency, and storing the result in a BenchmarkResults instance
+ *
+ * @author sgrostad
+ * @author olaaun
*/
public class CPUBenchmark implements Benchmark {
@@ -33,31 +37,32 @@ public class CPUBenchmark implements Benchmark {
this.commandExecutor = commandExecutor;
}
+ @Override
public void doBenchmark() {
try {
- ArrayList<String> commandOutput = commandExecutor.executeCommand(CPU_BENCHMARK_COMMAND);
- ArrayList<ParseResult> parseResults = parseCpuCyclesPerSec(commandOutput);
+ List<String> commandOutput = commandExecutor.executeCommand(CPU_BENCHMARK_COMMAND);
+ List<ParseResult> parseResults = parseCpuCyclesPerSec(commandOutput);
setCpuCyclesPerSec(parseResults);
} catch (IOException e) {
logger.log(Level.WARNING, "Failed to perform CPU benchmark", e);
}
}
- protected ArrayList<ParseResult> parseCpuCyclesPerSec(ArrayList<String> commandOutput) {
- ArrayList<String> searchWords = new ArrayList<>(Arrays.asList(CYCLES_SEARCH_WORD, SECONDS_SEARCH_WORD));
+ protected List<ParseResult> parseCpuCyclesPerSec(List<String> commandOutput) {
+ List<String> searchWords = new ArrayList<>(Arrays.asList(CYCLES_SEARCH_WORD, SECONDS_SEARCH_WORD));
ParseInstructions parseInstructions = new ParseInstructions(SEARCH_ELEMENT_INDEX, RETURN_ELEMENT_INDEX, SPLIT_REGEX_STRING, searchWords);
return OutputParser.parseOutput(parseInstructions, commandOutput);
}
- protected void setCpuCyclesPerSec(ArrayList<ParseResult> parseResults) {
+ protected void setCpuCyclesPerSec(List<ParseResult> parseResults) {
double cpuCyclesPerSec = getCyclesPerSecond(parseResults);
if (cpuCyclesPerSec > 0) {
benchmarkResults.setCpuCyclesPerSec(cpuCyclesPerSec);
}
}
- protected double getCyclesPerSecond(ArrayList<ParseResult> parseResults) {
+ protected double getCyclesPerSecond(List<ParseResult> parseResults) {
double cycles = -1;
double seconds = -1;
for (ParseResult parseResult : parseResults) {
diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/DiskBenchmark.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/DiskBenchmark.java
index ff8e1f21612..2a4c8d9b694 100644
--- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/DiskBenchmark.java
+++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/DiskBenchmark.java
@@ -9,12 +9,16 @@ import com.yahoo.vespa.hosted.node.verification.commons.parser.ParseResult;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.List;
import java.util.logging.Level;
import java.util.logging.Logger;
/**
- * Created by olaa on 10/07/2017.
+ * Responsible for benchmarking disk write speed, and storing the result in a BenchmarkResults instance
+ *
+ * @author olaaun
+ * @author sgrostad
*/
public class DiskBenchmark implements Benchmark {
@@ -34,9 +38,10 @@ public class DiskBenchmark implements Benchmark {
this.commandExecutor = commandExecutor;
}
+ @Override
public void doBenchmark() {
try {
- ArrayList<String> commandOutput = commandExecutor.executeCommand(DISK_BENCHMARK_COMMAND);
+ List<String> commandOutput = commandExecutor.executeCommand(DISK_BENCHMARK_COMMAND);
ParseResult parseResult = parseDiskSpeed(commandOutput);
setDiskSpeed(parseResult);
} catch (IOException e) {
@@ -44,8 +49,8 @@ public class DiskBenchmark implements Benchmark {
}
}
- protected ParseResult parseDiskSpeed(ArrayList<String> commandOutput) {
- ArrayList<String> searchWords = new ArrayList<>(Arrays.asList(KILO_BYTE_SEARCH_WORD, MEGA_BYTE_SEARCH_WORD, GIGA_BYTE_SEARCH_WORD));
+ protected ParseResult parseDiskSpeed(List<String> commandOutput) {
+ List<String> searchWords = new ArrayList<>(Arrays.asList(KILO_BYTE_SEARCH_WORD, MEGA_BYTE_SEARCH_WORD, GIGA_BYTE_SEARCH_WORD));
ParseInstructions parseInstructions = new ParseInstructions(SEARCH_ELEMENT_INDEX, RETURN_ELEMENT_INDEX, SPLIT_REGEX_STRING, searchWords);
return OutputParser.parseSingleOutput(parseInstructions, commandOutput);
}
diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/MemoryBenchmark.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/MemoryBenchmark.java
index 0aa4bc28eb1..b701d9a02f8 100644
--- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/MemoryBenchmark.java
+++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/MemoryBenchmark.java
@@ -9,11 +9,15 @@ import com.yahoo.vespa.hosted.node.verification.commons.parser.ParseResult;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.List;
import java.util.logging.Level;
import java.util.logging.Logger;
/**
- * Created by sgrostad on 11/07/2017.
+ * Responsible for benchmarking memory read/write speed, and storing the result in a BenchmarkResults instance
+ *
+ * @author sgrostad
+ * @author olaaun
*/
public class MemoryBenchmark implements Benchmark {
@@ -36,10 +40,11 @@ public class MemoryBenchmark implements Benchmark {
this.commandExecutor = commandExecutor;
}
+ @Override
public void doBenchmark() {
try {
setupMountPoint();
- ArrayList<String> commandOutput = commandExecutor.executeCommand(MEM_BENCHMARK_WRITE_SPEED);
+ List<String> commandOutput = commandExecutor.executeCommand(MEM_BENCHMARK_WRITE_SPEED);
ParseResult parseResult = parseMemorySpeed(commandOutput);
updateMemoryWriteSpeed(parseResult.getValue());
commandOutput = commandExecutor.executeCommand(MEM_BENCHMARK_READ_SPEED);
@@ -70,8 +75,8 @@ public class MemoryBenchmark implements Benchmark {
}
}
- protected ParseResult parseMemorySpeed(ArrayList<String> commandOutput) {
- ArrayList<String> searchWords = new ArrayList<>(Arrays.asList(READ_AND_WRITE_SEARCH_WORD));
+ protected ParseResult parseMemorySpeed(List<String> commandOutput) {
+ List<String> searchWords = new ArrayList<>(Arrays.asList(READ_AND_WRITE_SEARCH_WORD));
ParseInstructions parseInstructions = new ParseInstructions(SEARCH_ELEMENT_INDEX, RETURN_ELEMENT_INDEX, SPLIT_REGEX_STRING, searchWords);
return OutputParser.parseSingleOutput(parseInstructions, commandOutput);
}
diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/HardwareNodeComparator.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/HardwareNodeComparator.java
index 460fd31db18..88bd0539ccd 100644
--- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/HardwareNodeComparator.java
+++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/HardwareNodeComparator.java
@@ -6,8 +6,10 @@ import com.yahoo.vespa.hosted.node.verification.spec.retrievers.HardwareInfo;
import com.yahoo.vespa.hosted.node.verification.spec.retrievers.HardwareInfo.DiskType;
/**
- * Created by olaa on 04/07/2017.
* Compares two HardwareInfo objects and stores divergent values in a SpecVerificationReport
+ *
+ * @author olaaun
+ * @author sgrostad
*/
public class HardwareNodeComparator {
@@ -74,7 +76,7 @@ public class HardwareNodeComparator {
}
}
- private static boolean outsideThreshold(double value1, double value2 , double thresholdPercentage) {
+ private static boolean outsideThreshold(double value1, double value2, double thresholdPercentage) {
double lowerThresholdPercentage = 1 - thresholdPercentage;
double upperThresholdPercentage = 1 + thresholdPercentage;
return value1 < lowerThresholdPercentage * value2 || value1 > upperThresholdPercentage * value2;
diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/README.md b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/README.md
index ff87c7358f1..26856e078eb 100644
--- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/README.md
+++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/README.md
@@ -10,5 +10,5 @@ It then retrieves all the hardware information at the node with the "retrievers"
"HardwareInfo" object.
SpecVerifier then uses HardwareNodeComparator to compare spec from node repo and the node itself. It generates a
-SpecVerificationReport and uses ReportSender in "commons" to generate a full json report that can update node repo
-with the new results.
+SpecVerificationReport and uses Reporter in "commons" to retrieve the old HardwareDivergence report from node repo and update it. Reporter then prints the new
+updated HardwareDivergence report such that it can be updated in node repo by chef or other.
diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/SpecVerifier.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/SpecVerifier.java
index 1c96e685aab..e4cae4d7e9e 100644
--- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/SpecVerifier.java
+++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/SpecVerifier.java
@@ -4,11 +4,11 @@ package com.yahoo.vespa.hosted.node.verification.spec;
import com.yahoo.log.LogSetup;
import com.yahoo.vespa.hosted.node.verification.commons.CommandExecutor;
import com.yahoo.vespa.hosted.node.verification.commons.HostURLGenerator;
-import com.yahoo.vespa.hosted.node.verification.commons.report.ReportSender;
import com.yahoo.vespa.hosted.node.verification.commons.noderepo.IPAddressVerifier;
import com.yahoo.vespa.hosted.node.verification.commons.noderepo.NodeJsonConverter;
import com.yahoo.vespa.hosted.node.verification.commons.noderepo.NodeRepoInfoRetriever;
import com.yahoo.vespa.hosted.node.verification.commons.noderepo.NodeRepoJsonModel;
+import com.yahoo.vespa.hosted.node.verification.commons.report.Reporter;
import com.yahoo.vespa.hosted.node.verification.commons.report.SpecVerificationReport;
import com.yahoo.vespa.hosted.node.verification.spec.retrievers.HardwareInfo;
import com.yahoo.vespa.hosted.node.verification.spec.retrievers.HardwareInfoRetriever;
@@ -16,25 +16,27 @@ import com.yahoo.vespa.hosted.node.verification.spec.retrievers.HardwareInfoRetr
import java.io.IOException;
import java.net.URL;
import java.util.ArrayList;
+import java.util.List;
import java.util.logging.Level;
import java.util.logging.Logger;
/**
- * Created by olaa on 14/07/2017.
* Creates two HardwareInfo objects, one with spec from node repository and one from spec retrieved at the node.
* Compares the objects and returns the result.
+ *
+ * @author olaaun
+ * @author sgrostad
*/
public class SpecVerifier {
private static final Logger logger = Logger.getLogger(SpecVerifier.class.getName());
- private static final String VIRTUAL_ENVIRONMENT = "VIRTUAL_MACHINE";
- public static boolean verifySpec(CommandExecutor commandExecutor, ArrayList<URL> nodeInfoUrls) throws IOException {
+ public static boolean verifySpec(CommandExecutor commandExecutor, List<URL> nodeInfoUrls) throws IOException {
NodeRepoJsonModel nodeRepoJsonModel = getNodeRepositoryJSON(nodeInfoUrls);
VerifierSettings verifierSettings = new VerifierSettings(nodeRepoJsonModel);
HardwareInfo actualHardware = HardwareInfoRetriever.retrieve(commandExecutor, verifierSettings);
SpecVerificationReport specVerificationReport = makeVerificationReport(actualHardware, nodeRepoJsonModel);
- ReportSender.reportSpecVerificationResults(specVerificationReport, nodeInfoUrls);
+ Reporter.reportSpecVerificationResults(specVerificationReport, nodeInfoUrls);
return specVerificationReport.isValidSpec();
}
@@ -45,15 +47,15 @@ public class SpecVerifier {
return specVerificationReport;
}
- protected static NodeRepoJsonModel getNodeRepositoryJSON(ArrayList<URL> nodeInfoUrls) throws IOException {
+ protected static NodeRepoJsonModel getNodeRepositoryJSON(List<URL> nodeInfoUrls) throws IOException {
NodeRepoJsonModel nodeRepoJsonModel = NodeRepoInfoRetriever.retrieve(nodeInfoUrls);
return nodeRepoJsonModel;
}
- public static void main(String[] args) {
+ public static void main(String[] args) {
LogSetup.initVespaLogging("spec-verifier");
CommandExecutor commandExecutor = new CommandExecutor();
- ArrayList<URL> nodeInfoUrls;
+ List<URL> nodeInfoUrls;
if (args.length == 0) {
throw new IllegalStateException("Expected config server URL as parameter");
}
diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/VerifierSettings.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/VerifierSettings.java
index ba045e67766..2b5a19ce818 100644
--- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/VerifierSettings.java
+++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/VerifierSettings.java
@@ -4,19 +4,20 @@ package com.yahoo.vespa.hosted.node.verification.spec;
import com.yahoo.vespa.hosted.node.verification.commons.noderepo.NodeRepoJsonModel;
/**
- * Created by sgrostad on 07/08/2017.
* Contains information on what spec should be verified or not.
+ *
+ * @author sgrostad
+ * @author olaaun
*/
-
public class VerifierSettings {
private final boolean checkIPv6;
- public VerifierSettings(){
+ public VerifierSettings() {
this.checkIPv6 = true;
}
- public VerifierSettings(NodeRepoJsonModel nodeRepoJsonModel){
+ public VerifierSettings(NodeRepoJsonModel nodeRepoJsonModel) {
checkIPv6 = nodeRepoJsonModel.getIpv6Address() != null;
}
diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/CPURetriever.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/CPURetriever.java
index b02f37533da..b8ae5d89b55 100644
--- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/CPURetriever.java
+++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/CPURetriever.java
@@ -9,11 +9,15 @@ import com.yahoo.vespa.hosted.node.verification.commons.parser.ParseResult;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.List;
import java.util.logging.Level;
import java.util.logging.Logger;
/**
- * Created by olaa on 30/06/2017.
+ * Retrieves number of CPU cores, and stores the result in a HardwareInfo instance
+ *
+ * @author olaaun
+ * @author sgrostad
*/
public class CPURetriever implements HardwareRetriever {
@@ -31,28 +35,29 @@ public class CPURetriever implements HardwareRetriever {
this.commandExecutor = commandExecutor;
}
+ @Override
public void updateInfo() {
try {
- ArrayList<String> commandOutput = commandExecutor.executeCommand(CPU_INFO_COMMAND);
- ArrayList<ParseResult> parseResults = parseCPUInfoFile(commandOutput);
+ List<String> commandOutput = commandExecutor.executeCommand(CPU_INFO_COMMAND);
+ List<ParseResult> parseResults = parseCPUInfoFile(commandOutput);
setCpuCores(parseResults);
} catch (IOException e) {
logger.log(Level.WARNING, "Failed to retrieve CPU info", e);
}
}
- protected ArrayList<ParseResult> parseCPUInfoFile(ArrayList<String> commandOutput) {
- ArrayList<String> searchWords = new ArrayList<>(Arrays.asList(SEARCH_WORD));
+ protected List<ParseResult> parseCPUInfoFile(List<String> commandOutput) {
+ List<String> searchWords = new ArrayList<>(Arrays.asList(SEARCH_WORD));
ParseInstructions parseInstructions = new ParseInstructions(SEARCH_ELEMENT_INDEX, RETURN_ELEMENT_INDEX, REGEX_SPLIT, searchWords);
- ArrayList<ParseResult> parseResults = OutputParser.parseOutput(parseInstructions, commandOutput);
+ List<ParseResult> parseResults = OutputParser.parseOutput(parseInstructions, commandOutput);
return parseResults;
}
- protected void setCpuCores(ArrayList<ParseResult> parseResults) {
+ protected void setCpuCores(List<ParseResult> parseResults) {
hardwareInfo.setMinCpuCores(countCpuCores(parseResults));
}
- protected int countCpuCores(ArrayList<ParseResult> parseResults) {
+ protected int countCpuCores(List<ParseResult> parseResults) {
return parseResults.size();
}
diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/DiskRetriever.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/DiskRetriever.java
index 58d4810299b..8058c9438d7 100644
--- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/DiskRetriever.java
+++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/DiskRetriever.java
@@ -10,13 +10,18 @@ import com.yahoo.vespa.hosted.node.verification.spec.retrievers.HardwareInfo.Dis
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.List;
import java.util.logging.Level;
import java.util.logging.Logger;
/**
- * Created by olaa on 30/06/2017.
+ * Retrieves disk space and type, and stores the result in a HardwareInfo instance
+ *
+ * @author olaaun
+ * @author sgrostad
*/
public class DiskRetriever implements HardwareRetriever {
+
private static final String DISK_CHECK_TYPE = "lsblk -d -o name,rota";
private static final String DISK_CHECK_SIZE = "pvdisplay --units G | grep 'PV Size'";
private static final String DISK_NAME = "sda";
@@ -37,6 +42,7 @@ public class DiskRetriever implements HardwareRetriever {
this.commandExecutor = commandExecutor;
}
+ @Override
public void updateInfo() {
updateDiskType();
updateDiskSize();
@@ -44,7 +50,7 @@ public class DiskRetriever implements HardwareRetriever {
protected void updateDiskType() {
try {
- ArrayList<String> commandOutput = commandExecutor.executeCommand(DISK_CHECK_TYPE);
+ List<String> commandOutput = commandExecutor.executeCommand(DISK_CHECK_TYPE);
ParseResult parseResult = parseDiskType(commandOutput);
setDiskType(parseResult);
} catch (IOException e) {
@@ -54,16 +60,16 @@ public class DiskRetriever implements HardwareRetriever {
protected void updateDiskSize() {
try {
- ArrayList<String> commandOutput = commandExecutor.executeCommand(DISK_CHECK_SIZE);
- ArrayList<ParseResult> parseResult = parseDiskSize(commandOutput);
+ List<String> commandOutput = commandExecutor.executeCommand(DISK_CHECK_SIZE);
+ List<ParseResult> parseResult = parseDiskSize(commandOutput);
setDiskSize(parseResult);
} catch (IOException e) {
logger.log(Level.WARNING, "Failed to retrieve disk size", e);
}
}
- protected ParseResult parseDiskType(ArrayList<String> commandOutput) throws IOException {
- ArrayList<String> searchWords = new ArrayList<>(Arrays.asList(DISK_NAME));
+ protected ParseResult parseDiskType(List<String> commandOutput) throws IOException {
+ List<String> searchWords = new ArrayList<>(Arrays.asList(DISK_NAME));
ParseInstructions parseInstructions = new ParseInstructions(DISK_TYPE_SEARCH_ELEMENT_INDEX, DISK_TYPE_RETURN_ELEMENT_INDEX, DISK_TYPE_REGEX_SPLIT, searchWords);
ParseResult parseResult = OutputParser.parseSingleOutput(parseInstructions, commandOutput);
if (!parseResult.getSearchWord().equals(DISK_NAME)) {
@@ -83,13 +89,13 @@ public class DiskRetriever implements HardwareRetriever {
}
}
- protected ArrayList<ParseResult> parseDiskSize(ArrayList<String> commandOutput) {
- ArrayList<String> searchWords = new ArrayList<>(Arrays.asList(DISK_SIZE_SEARCH_WORD));
+ protected List<ParseResult> parseDiskSize(List<String> commandOutput) {
+ List<String> searchWords = new ArrayList<>(Arrays.asList(DISK_SIZE_SEARCH_WORD));
ParseInstructions parseInstructions = new ParseInstructions(DISK_SIZE_SEARCH_ELEMENT_INDEX, DISK_SIZE_RETURN_ELEMENT_INDEX, DISK_SIZE_REGEX_SPLIT, searchWords);
return OutputParser.parseOutput(parseInstructions, commandOutput);
}
- protected void setDiskSize(ArrayList<ParseResult> parseResults) {
+ protected void setDiskSize(List<ParseResult> parseResults) {
double diskSize = 0;
try {
for (ParseResult parseResult : parseResults) {
diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/HardwareInfo.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/HardwareInfo.java
index 8d726c507ba..1c17d73523b 100644
--- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/HardwareInfo.java
+++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/HardwareInfo.java
@@ -2,10 +2,12 @@
package com.yahoo.vespa.hosted.node.verification.spec.retrievers;
/**
- * Created by olaa on 04/07/2017.
* All information the different retrievers retrieve is stored as a HardwareInfo object.
+ *
+ * @author olaaun
+ * @author sgrostad
*/
-
+// TODO: This should be immutable
public class HardwareInfo {
private double minDiskAvailableGb;
@@ -17,7 +19,6 @@ public class HardwareInfo {
private double interfaceSpeedMbs;
private DiskType diskType;
-
public double getInterfaceSpeedMbs() {
return interfaceSpeedMbs;
}
diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/HardwareInfoRetriever.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/HardwareInfoRetriever.java
index e3235590169..ba29f07baeb 100644
--- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/HardwareInfoRetriever.java
+++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/HardwareInfoRetriever.java
@@ -5,16 +5,19 @@ import com.yahoo.vespa.hosted.node.verification.commons.CommandExecutor;
import com.yahoo.vespa.hosted.node.verification.spec.VerifierSettings;
import java.util.ArrayList;
+import java.util.List;
/**
- * Created by olaa on 30/06/2017.
* Makes a HardwareInfo object and calls all the retrievers for this object.
+ *
+ * @author olaaun
+ * @author sgrostad
*/
public class HardwareInfoRetriever {
public static HardwareInfo retrieve(CommandExecutor commandExecutor, VerifierSettings verifierSettings) {
HardwareInfo hardwareInfo = new HardwareInfo();
- ArrayList<HardwareRetriever> infoList = new ArrayList<>();
+ List<HardwareRetriever> infoList = new ArrayList<>();
infoList.add(new CPURetriever(hardwareInfo, commandExecutor));
infoList.add(new MemoryRetriever(hardwareInfo, commandExecutor));
infoList.add(new DiskRetriever(hardwareInfo, commandExecutor));
diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/HardwareRetriever.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/HardwareRetriever.java
index 90e00560e59..d77342db982 100644
--- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/HardwareRetriever.java
+++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/HardwareRetriever.java
@@ -2,10 +2,14 @@
package com.yahoo.vespa.hosted.node.verification.spec.retrievers;
/**
- * Created by olaa on 30/06/2017.
+ * @author olaaun
+ * @author sgrostad
*/
public interface HardwareRetriever {
+ /**
+ * Should retrieve spec from some part of the hardware, and store the result in hardwareinfo instance passed to class
+ */
void updateInfo();
}
diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/MemoryRetriever.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/MemoryRetriever.java
index 2a64e0f4b21..c6655e13797 100644
--- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/MemoryRetriever.java
+++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/MemoryRetriever.java
@@ -9,11 +9,15 @@ import com.yahoo.vespa.hosted.node.verification.commons.parser.ParseResult;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.List;
import java.util.logging.Level;
import java.util.logging.Logger;
/**
- * Created by olaa on 30/06/2017.
+ * Retrieves memory size, and stores the result in a HardwareInfo instance
+ *
+ * @author olaaun
+ * @author sgrostad
*/
public class MemoryRetriever implements HardwareRetriever {
@@ -31,10 +35,10 @@ public class MemoryRetriever implements HardwareRetriever {
this.commandExecutor = commandExecutor;
}
-
+ @Override
public void updateInfo() {
try {
- ArrayList<String> commandOutput = commandExecutor.executeCommand(MEMORY_INFO_COMMAND);
+ List<String> commandOutput = commandExecutor.executeCommand(MEMORY_INFO_COMMAND);
ParseResult parseResult = parseMemInfoFile(commandOutput);
updateMemoryInfo(parseResult);
} catch (IOException e) {
@@ -42,8 +46,8 @@ public class MemoryRetriever implements HardwareRetriever {
}
}
- protected ParseResult parseMemInfoFile(ArrayList<String> commandOutput) throws IOException {
- ArrayList<String> searchWords = new ArrayList<>(Arrays.asList(SEARCH_WORD));
+ protected ParseResult parseMemInfoFile(List<String> commandOutput) throws IOException {
+ List<String> searchWords = new ArrayList<>(Arrays.asList(SEARCH_WORD));
ParseInstructions parseInstructions = new ParseInstructions(SEARCH_ELEMENT_INDEX, RETURN_ELEMENT_INDEX, REGEX_SPLIT, searchWords);
ParseResult parseResult = OutputParser.parseSingleOutput(parseInstructions, commandOutput);
if (!parseResult.getSearchWord().matches(SEARCH_WORD)) {
diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/NetRetriever.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/NetRetriever.java
index 746279915bf..f30d455f339 100644
--- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/NetRetriever.java
+++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/NetRetriever.java
@@ -11,11 +11,16 @@ import org.apache.commons.exec.ExecuteException;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.List;
import java.util.logging.Level;
import java.util.logging.Logger;
/**
- * Created by olaa on 30/06/2017.
+ * Retrieves IPv4/IPv6 interface, and checks interface speed. If node should have IPv6, tries to ping6.
+ * The results are stored in a HardwareInfo instance
+ *
+ * @author olaaun
+ * @author sgrostad
*/
public class NetRetriever implements HardwareRetriever {
@@ -48,8 +53,9 @@ public class NetRetriever implements HardwareRetriever {
this.verifierSettings = verifierSettings;
}
+ @Override
public void updateInfo() {
- ArrayList<ParseResult> parseResults = findInterface();
+ List<ParseResult> parseResults = findInterface();
findInterfaceSpeed(parseResults);
if (verifierSettings.isCheckIPv6()) {
testPingResponse(parseResults);
@@ -57,10 +63,10 @@ public class NetRetriever implements HardwareRetriever {
updateHardwareInfoWithNet(parseResults);
}
- protected ArrayList<ParseResult> findInterface() {
- ArrayList<ParseResult> parseResults = new ArrayList<>();
+ protected List<ParseResult> findInterface() {
+ List<ParseResult> parseResults = new ArrayList<>();
try {
- ArrayList<String> commandOutput = commandExecutor.executeCommand(NET_FIND_INTERFACE);
+ List<String> commandOutput = commandExecutor.executeCommand(NET_FIND_INTERFACE);
parseResults = parseNetInterface(commandOutput);
} catch (IOException e) {
@@ -69,16 +75,16 @@ public class NetRetriever implements HardwareRetriever {
return parseResults;
}
- protected ArrayList<ParseResult> parseNetInterface(ArrayList<String> commandOutput) {
- ArrayList<String> searchWords = new ArrayList<>(Arrays.asList(SEARCH_WORD_INTERFACE_IP4, SEARCH_WORD_INTERFACE_IPV6));
+ protected List<ParseResult> parseNetInterface(List<String> commandOutput) {
+ List<String> searchWords = new ArrayList<>(Arrays.asList(SEARCH_WORD_INTERFACE_IP4, SEARCH_WORD_INTERFACE_IPV6));
ParseInstructions parseInstructions = new ParseInstructions(INTERFACE_SEARCH_ELEMENT_INDEX, INTERFACE_RETURN_ELEMENT_INDEX, INTERFACE_NAME_REGEX_SPLIT, searchWords);
- ArrayList<ParseResult> parseResults = OutputParser.parseOutput(parseInstructions, commandOutput);
+ List<ParseResult> parseResults = OutputParser.parseOutput(parseInstructions, commandOutput);
return parseResults;
}
- protected void findInterfaceSpeed(ArrayList<ParseResult> parseResults) {
+ protected void findInterfaceSpeed(List<ParseResult> parseResults) {
try {
- ArrayList<String> commandOutput = commandExecutor.executeCommand(NET_CHECK_INTERFACE_SPEED);
+ List<String> commandOutput = commandExecutor.executeCommand(NET_CHECK_INTERFACE_SPEED);
ParseResult parseResult = parseInterfaceSpeed(commandOutput);
parseResults.add(parseResult);
} catch (IOException e) {
@@ -86,8 +92,8 @@ public class NetRetriever implements HardwareRetriever {
}
}
- protected ParseResult parseInterfaceSpeed(ArrayList<String> commandOutput) throws IOException {
- ArrayList<String> searchWords = new ArrayList<>(Arrays.asList(SEARCH_WORD_INTERFACE_SPEED));
+ protected ParseResult parseInterfaceSpeed(List<String> commandOutput) throws IOException {
+ List<String> searchWords = new ArrayList<>(Arrays.asList(SEARCH_WORD_INTERFACE_SPEED));
ParseInstructions parseInstructions = new ParseInstructions(INTERFACE_SPEED_SEARCH_ELEMENT_INDEX, INTERFACE_SPEED_RETURN_ELEMENT_INDEX, INTERFACE_SPEED_REGEX_SPLIT, searchWords);
ParseResult parseResult = OutputParser.parseSingleOutput(parseInstructions, commandOutput);
if (!parseResult.getSearchWord().matches(SEARCH_WORD_INTERFACE_SPEED)) {
@@ -96,9 +102,9 @@ public class NetRetriever implements HardwareRetriever {
return parseResult;
}
- protected void testPingResponse(ArrayList<ParseResult> parseResults) {
+ protected void testPingResponse(List<ParseResult> parseResults) {
try {
- ArrayList<String> commandOutput = commandExecutor.executeCommand(PING_NET_COMMAND);
+ List<String> commandOutput = commandExecutor.executeCommand(PING_NET_COMMAND);
parseResults.add(parsePingResponse(commandOutput));
} catch (ExecuteException e) {
logger.log(Level.WARNING, "Failed to execute ping6", e);
@@ -107,8 +113,8 @@ public class NetRetriever implements HardwareRetriever {
}
}
- protected ParseResult parsePingResponse(ArrayList<String> commandOutput) throws IOException {
- ArrayList<String> searchWords = new ArrayList<>(Arrays.asList(PING_SEARCH_WORD));
+ protected ParseResult parsePingResponse(List<String> commandOutput) throws IOException {
+ List<String> searchWords = new ArrayList<>(Arrays.asList(PING_SEARCH_WORD));
ParseInstructions parseInstructions = new ParseInstructions(PING_SEARCH_ELEMENT_INDEX, PING_RETURN_ELEMENT_INDEX, PING_SPLIT_REGEX_STRING, searchWords);
ParseResult parseResult = OutputParser.parseSingleOutput(parseInstructions, commandOutput);
if (!parseResult.getSearchWord().matches(PING_SEARCH_WORD)) {
@@ -117,7 +123,7 @@ public class NetRetriever implements HardwareRetriever {
return new ParseResult(PING_SEARCH_WORD, parseResult.getValue());
}
- protected void updateHardwareInfoWithNet(ArrayList<ParseResult> parseResults) {
+ protected void updateHardwareInfoWithNet(List<ParseResult> parseResults) {
hardwareInfo.setIpv6Interface(false);
hardwareInfo.setIpv4Interface(false);
for (ParseResult parseResult : parseResults) {
diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/commons/CommandExecutorTest.java b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/commons/CommandExecutorTest.java
index ce4156b5960..5f5aca825c0 100644
--- a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/commons/CommandExecutorTest.java
+++ b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/commons/CommandExecutorTest.java
@@ -5,15 +5,16 @@ import org.junit.Before;
import org.junit.Test;
import java.io.IOException;
-import java.util.ArrayList;
import java.util.List;
import static java.util.Arrays.asList;
import static org.junit.Assert.assertEquals;
/**
- * Created by sgrostad on 12/07/2017.
+ * @author sgrostad
+ * @author olaaun
*/
+
public class CommandExecutorTest {
private CommandExecutor commandExecutor;
@@ -26,7 +27,7 @@ public class CommandExecutorTest {
@Test
public void test_if_executeAString_reads_testReadFile_correct() throws IOException {
String command = "cat src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/resources/testReadFile.txt";
- ArrayList<String> commandOutput = commandExecutor.executeCommand(command);
+ List<String> commandOutput = commandExecutor.executeCommand(command);
List<String> expectedOutput = asList("This test file tests apache commons exec", "Second line");
assertEquals(expectedOutput, commandOutput);
}
diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/commons/HostURLGeneratorTest.java b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/commons/HostURLGeneratorTest.java
index 1519e4e52fa..773172de807 100644
--- a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/commons/HostURLGeneratorTest.java
+++ b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/commons/HostURLGeneratorTest.java
@@ -7,15 +7,17 @@ import org.junit.Test;
import java.io.IOException;
import java.net.URL;
-import java.util.ArrayList;
-import java.util.stream.Collectors;
+import java.util.List;
+
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.fail;
/**
- * Created by olaa on 14/07/2017.
+ * @author sgrostad
+ * @author olaaun
*/
+
public class HostURLGeneratorTest {
private MockCommandExecutor mockCommandExecutor;
@@ -35,7 +37,7 @@ public class HostURLGeneratorTest {
@Test
public void generateNodeInfoUrl_find_config_server_test_if_url_is_formatted_correctly() throws Exception {
mockCommandExecutor.addCommand(CAT_NODE_HOST_NAME_PATH);
- ArrayList<URL> urls = HostURLGenerator.generateNodeInfoUrl(mockCommandExecutor, CONFIG_SERVER_HOSTNAME_1 + "," + CONFIG_SERVER_HOSTNAME_2);
+ List<URL> urls = HostURLGenerator.generateNodeInfoUrl(mockCommandExecutor, CONFIG_SERVER_HOSTNAME_1 + "," + CONFIG_SERVER_HOSTNAME_2);
String expectedUrl1 = CONFIG_SERVER_HOSTNAME_1 + NODE_HOSTNAME_PREFIX + EXPECTED_HOSTNAME;
String expectedUrl2 = CONFIG_SERVER_HOSTNAME_2 + NODE_HOSTNAME_PREFIX + EXPECTED_HOSTNAME;
assertEquals(expectedUrl1, urls.get(0).toString());
@@ -58,7 +60,7 @@ public class HostURLGeneratorTest {
public void generateNodeInfoUrl_retrieve_config_server_as_parameter_test_if_url_is_formatted_correctly() throws Exception {
mockCommandExecutor.addCommand(CAT_NODE_HOST_NAME_PATH);
String configServerHostname = "cfg1.prod.region1";
- ArrayList<URL> actualUrls = HostURLGenerator.generateNodeInfoUrl(mockCommandExecutor, configServerHostname);
+ List<URL> actualUrls = HostURLGenerator.generateNodeInfoUrl(mockCommandExecutor, configServerHostname);
String expectedUrl = CONFIG_SERVER_HOSTNAME_1 + NODE_HOSTNAME_PREFIX + EXPECTED_HOSTNAME;
String actualUrl = actualUrls.get(0).toString();
assertEquals(expectedUrl, actualUrl);
diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/commons/noderepo/IPAddressVerifierTest.java b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/commons/noderepo/IPAddressVerifierTest.java
index d0b378af4b8..197b4836bc8 100644
--- a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/commons/noderepo/IPAddressVerifierTest.java
+++ b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/commons/noderepo/IPAddressVerifierTest.java
@@ -14,6 +14,11 @@ import static org.junit.Assert.assertEquals;
import static org.mockito.Mockito.doReturn;
import static org.mockito.Mockito.spy;
+/**
+ * @author sgrostad
+ * @author olaaun
+ */
+
public class IPAddressVerifierTest {
private IPAddressVerifier ipAddressVerifier = spy(new IPAddressVerifier());
diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/commons/noderepo/NodeJsonConverterTest.java b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/commons/noderepo/NodeJsonConverterTest.java
index c23d29c4076..0940efd08d3 100644
--- a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/commons/noderepo/NodeJsonConverterTest.java
+++ b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/commons/noderepo/NodeJsonConverterTest.java
@@ -8,21 +8,24 @@ import java.io.File;
import java.net.URL;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.List;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
/**
- * Created by olaa on 07/07/2017.
+ * @author sgrostad
+ * @author olaaun
*/
+
public class NodeJsonConverterTest {
private static final double DELTA = 0.1;
@Test
public void convertJsonModel_should_return_correct_HardwareInfo() throws Exception {
- ArrayList<URL> urls = new ArrayList<>(Arrays.asList(new File("src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/nodeInfoTest.json").toURI().toURL()));
+ List<URL> urls = new ArrayList<>(Arrays.asList(new File("src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/nodeInfoTest.json").toURI().toURL()));
NodeRepoJsonModel nodeRepoJsonModel = NodeRepoInfoRetriever.retrieve(urls);
HardwareInfo hardwareInfo = NodeJsonConverter.convertJsonModelToHardwareInfo(nodeRepoJsonModel);
double expectedMinDiskAvailable = 500.0;
diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/commons/noderepo/NodeRepoInfoRetrieverTest.java b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/commons/noderepo/NodeRepoInfoRetrieverTest.java
index 49792963a05..97bf3fdeb11 100644
--- a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/commons/noderepo/NodeRepoInfoRetrieverTest.java
+++ b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/commons/noderepo/NodeRepoInfoRetrieverTest.java
@@ -9,14 +9,20 @@ import java.net.MalformedURLException;
import java.net.URL;
import java.nio.file.Paths;
import java.util.ArrayList;
+import java.util.List;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.fail;
+/**
+ * @author sgrostad
+ * @author olaaun
+ */
+
public class NodeRepoInfoRetrieverTest {
private NodeRepoInfoRetriever nodeRepoInfoRetriever;
- private ArrayList<URL> urls;
+ private List<URL> urls;
private static final double DELTA = 0.1;
private static final String ABSOLUTE_PATH = Paths.get(".").toAbsolutePath().normalize().toString();
private static final String RESOURCE_PATH = "src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources";
diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/commons/parser/OutputParserTest.java b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/commons/parser/OutputParserTest.java
index dc43fcd2e47..09286f4e61a 100644
--- a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/commons/parser/OutputParserTest.java
+++ b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/commons/parser/OutputParserTest.java
@@ -1,20 +1,20 @@
// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.hosted.node.verification.commons.parser;
-import com.yahoo.vespa.hosted.node.verification.commons.parser.OutputParser;
-import com.yahoo.vespa.hosted.node.verification.commons.parser.ParseInstructions;
-import com.yahoo.vespa.hosted.node.verification.commons.parser.ParseResult;
import org.junit.Before;
import org.junit.Test;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.List;
import static org.junit.Assert.assertEquals;
/**
- * Created by sgrostad on 21/07/2017.
+ * @author sgrostad
+ * @author olaaun
*/
+
public class OutputParserTest {
private static final String RETURN_VALUE = "#returnValue#";
@@ -24,8 +24,8 @@ public class OutputParserTest {
private static final String SEARCH_WORD_1 = "Parsing";
private static final String SEARCH_WORD_2 = "this";
private static final String REGEX_SEARCH_WORD = ".*S.*";
- private ArrayList<String> commandOutput;
- private ArrayList<String> searchWords;
+ private List<String> commandOutput;
+ private List<String> searchWords;
@Before
public void setup() {
@@ -36,7 +36,7 @@ public class OutputParserTest {
public void parseOutput_searching_for_two_normal_words() {
searchWords = new ArrayList<>(Arrays.asList(SEARCH_WORD_1, SEARCH_WORD_2));
ParseInstructions parseInstructions = new ParseInstructions(6, 8, " ", searchWords);
- ArrayList<ParseResult> parseResults = OutputParser.parseOutput(parseInstructions, commandOutput);
+ List<ParseResult> parseResults = OutputParser.parseOutput(parseInstructions, commandOutput);
ParseResult expectedParseResult1 = new ParseResult(SEARCH_WORD_1, RETURN_VALUE);
ParseResult expectedParseResult2 = new ParseResult(SEARCH_WORD_2, RETURN_VALUE);
assertEquals(expectedParseResult1, parseResults.get(0));
@@ -47,7 +47,7 @@ public class OutputParserTest {
public void parseOutput_searching_for_two_normal_words_with_semicolon_as_line_split() {
searchWords = new ArrayList<>(Arrays.asList(SEARCH_WORD_1, SEARCH_WORD_2));
ParseInstructions parseInstructions = new ParseInstructions(4, 5, ";", searchWords);
- ArrayList<ParseResult> parseResults = OutputParser.parseOutput(parseInstructions, commandOutput);
+ List<ParseResult> parseResults = OutputParser.parseOutput(parseInstructions, commandOutput);
ParseResult expectedParseResult1 = new ParseResult(SEARCH_WORD_1, RETURN_VALUE);
ParseResult expectedParseResult2 = new ParseResult(SEARCH_WORD_2, RETURN_VALUE);
assertEquals(expectedParseResult1, parseResults.get(0));
@@ -58,7 +58,7 @@ public class OutputParserTest {
public void parseOutput_searching_for_word_containing_capital_s() {
searchWords = new ArrayList<>(Arrays.asList(REGEX_SEARCH_WORD));
ParseInstructions parseInstructions = new ParseInstructions(1, 8, " ", searchWords);
- ArrayList<ParseResult> parseResults = OutputParser.parseOutput(parseInstructions, commandOutput);
+ List<ParseResult> parseResults = OutputParser.parseOutput(parseInstructions, commandOutput);
ParseResult expectedParseResult1 = new ParseResult("Should", RETURN_VALUE);
ParseResult expectedParseResult2 = new ParseResult("thiS", RETURN_VALUE);
assertEquals(expectedParseResult1, parseResults.get(0));
diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/commons/report/BenchmarkReportTest.java b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/commons/report/BenchmarkReportTest.java
index 2963b57110c..931b42bb2a0 100644
--- a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/commons/report/BenchmarkReportTest.java
+++ b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/commons/report/BenchmarkReportTest.java
@@ -6,6 +6,11 @@ import org.junit.Test;
import static org.junit.Assert.assertEquals;
+/**
+ * @author sgrostad
+ * @author olaaun
+ */
+
public class BenchmarkReportTest {
private BenchmarkReport benchmarkReport = new BenchmarkReport();
diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/commons/report/ReportSenderTest.java b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/commons/report/ReportSenderTest.java
deleted file mode 100644
index bc6d1d6d089..00000000000
--- a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/commons/report/ReportSenderTest.java
+++ /dev/null
@@ -1,70 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.hosted.node.verification.commons.report;
-
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-import java.io.PrintStream;
-import java.net.URL;
-import java.nio.file.Paths;
-import java.util.ArrayList;
-import java.util.Arrays;
-
-import static org.junit.Assert.assertEquals;
-
-public class ReportSenderTest {
-
- private final ByteArrayOutputStream println = new ByteArrayOutputStream();
- private static final String ABSOLUTE_PATH = Paths.get(".").toAbsolutePath().normalize().toString();
- private static final String RESOURCE_PATH = "src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/nodeInfoTest.json";
- private static final String URL_RESOURCE_PATH = "file://" + ABSOLUTE_PATH + "/" + RESOURCE_PATH;
- private static ArrayList<URL> nodeInfoUrls;
-
- @Before
- public void setup() throws IOException {
- System.setOut(new PrintStream(println));
- URL nodeInfoUrlWithAlreadyExistingHardwareDivergence = new URL(URL_RESOURCE_PATH);
- nodeInfoUrls = new ArrayList<>(Arrays.asList(nodeInfoUrlWithAlreadyExistingHardwareDivergence));
- }
-
- @After
- public void cleanUpStream() {
- System.setOut(System.out);
- }
-
- @Test
- public void reportBenchmarkResults_should_update_already_existing_hardwareDivergence_changing_existing_values() throws Exception {
- BenchmarkReport benchmarkReport = new BenchmarkReport();
- double cpuCyclesPerSec = 0.3;
- double memoryReadSpeedGBs = 0.1;
- benchmarkReport.setCpuCyclesPerSec(cpuCyclesPerSec);
- benchmarkReport.setMemoryReadSpeedGBs(memoryReadSpeedGBs);
- String expectedReport = "{\"benchmarkReport\":{\"cpuCyclesPerSec\":0.3,\"memoryReadSpeedGBs\":0.1}}";
- ReportSender.reportBenchmarkResults(benchmarkReport,nodeInfoUrls);
- assertEquals(expectedReport, println.toString());
- }
-
- @Test
- public void reportBenchmarkResults_should_should_update_already_existing_hardwareDivergence_prints_null_when_empty_benchmarkReport() throws Exception {
- BenchmarkReport benchmarkReport = new BenchmarkReport();
- String expectedReport = "null";
- ReportSender.reportBenchmarkResults(benchmarkReport, nodeInfoUrls);
- assertEquals(expectedReport, println.toString());
- }
-
- @Test
- public void reportSpecVerificationResults_should_update_already_existing_hardwareDivergence_adding_report_type() throws Exception {
- SpecVerificationReport specVerificationReport = new SpecVerificationReport();
- double actualDiskSpaceAvailable = 150D;
- boolean actualIpv6Connection = false;
- specVerificationReport.setActualDiskSpaceAvailable(actualDiskSpaceAvailable);
- specVerificationReport.setActualIpv6Connection(actualIpv6Connection);
- String expectedReport = "{\"specVerificationReport\":{\"actualDiskSpaceAvailable\":150.0,\"actualIpv6Connection\":false},\"benchmarkReport\":{\"cpuCyclesPerSec\":0.5}}";
- ReportSender.reportSpecVerificationResults(specVerificationReport, nodeInfoUrls);
- assertEquals(expectedReport, println.toString());
- }
-
-}
diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/commons/report/ReporterTest.java b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/commons/report/ReporterTest.java
new file mode 100644
index 00000000000..5f469db7f53
--- /dev/null
+++ b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/commons/report/ReporterTest.java
@@ -0,0 +1,104 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.verification.commons.report;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.PrintStream;
+import java.net.URL;
+import java.nio.file.Paths;
+import java.util.List;
+import java.util.ArrayList;
+import java.util.Arrays;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * @author sgrostad
+ * @author olaaun
+ */
+
+public class ReporterTest {
+
+ private final ByteArrayOutputStream println = new ByteArrayOutputStream();
+ private static final String ABSOLUTE_PATH = Paths.get(".").toAbsolutePath().normalize().toString();
+ private static final String RESOURCE_PATH_TO_VALID_HARDWARE_DIVERGENCE = "src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/nodeInfoTest.json";
+ private static final String RESOURCE_PATH_TO_INVALID_HARDWARE_DIVERGENCE = "src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/nodeInfoNotInterpretableHardwareDivergence.json";
+ private static final String RESOURCE_PATH_TO_EMPTY_HARDWARE_DIVERGENCE = "src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/nodeRepo.json";
+ private static final String URL_VALID_RESOURCE_PATH = "file://" + ABSOLUTE_PATH + "/" + RESOURCE_PATH_TO_VALID_HARDWARE_DIVERGENCE;
+ private static final String URL_INVALID_RESOURCE_PATH = "file://" + ABSOLUTE_PATH + "/" + RESOURCE_PATH_TO_INVALID_HARDWARE_DIVERGENCE;
+ private static final String URL_EMPTY_RESOURCE_PATH = "file://" + ABSOLUTE_PATH + "/" + RESOURCE_PATH_TO_EMPTY_HARDWARE_DIVERGENCE;
+ private static List<URL> nodeInfoUrlsToValidHardwareDivergence;
+ private static List<URL> nodeInfoUrlsToNOTValidHardwareDivergence;
+ private static List<URL> nodeInfoUrlsWithNoHardwareDivergence;
+
+ @Before
+ public void setup() throws IOException {
+ System.setOut(new PrintStream(println));
+ URL nodeInfoUrlWithAlreadyExistingHardwareDivergence = new URL(URL_VALID_RESOURCE_PATH);
+ nodeInfoUrlsToValidHardwareDivergence = new ArrayList<>(Arrays.asList(nodeInfoUrlWithAlreadyExistingHardwareDivergence));
+ URL nodeInfoUrlWithExistingButWrongHardwareDivergence = new URL(URL_INVALID_RESOURCE_PATH);
+ nodeInfoUrlsToNOTValidHardwareDivergence = new ArrayList<>(Arrays.asList(nodeInfoUrlWithExistingButWrongHardwareDivergence));
+ URL nodeInfoUrlWithNoHardwareDivergence = new URL(URL_EMPTY_RESOURCE_PATH);
+ nodeInfoUrlsWithNoHardwareDivergence = new ArrayList<>(Arrays.asList(nodeInfoUrlWithNoHardwareDivergence));
+ }
+
+ @After
+ public void cleanUpStream() {
+ System.setOut(System.out);
+ }
+
+ @Test
+ public void reportBenchmarkResults_should_update_already_existing_hardwareDivergence_changing_existing_values() throws Exception {
+ BenchmarkReport benchmarkReport = new BenchmarkReport();
+ double cpuCyclesPerSec = 0.3;
+ double memoryReadSpeedGBs = 0.1;
+ benchmarkReport.setCpuCyclesPerSec(cpuCyclesPerSec);
+ benchmarkReport.setMemoryReadSpeedGBs(memoryReadSpeedGBs);
+ String expectedReport = "{\"benchmarkReport\":{\"cpuCyclesPerSec\":0.3,\"memoryReadSpeedGBs\":0.1}}";
+ Reporter.reportBenchmarkResults(benchmarkReport, nodeInfoUrlsToValidHardwareDivergence);
+ assertEquals(expectedReport, println.toString());
+ }
+
+ @Test
+ public void reportBenchmarkResults_should_should_update_already_existing_hardwareDivergence_prints_null_when_empty_benchmarkReport() throws Exception {
+ BenchmarkReport benchmarkReport = new BenchmarkReport();
+ String expectedReport = "null";
+ Reporter.reportBenchmarkResults(benchmarkReport, nodeInfoUrlsToValidHardwareDivergence);
+ assertEquals(expectedReport, println.toString());
+ }
+
+ @Test
+ public void reportSpecVerificationResults_should_update_already_existing_hardwareDivergence_adding_report_type() throws Exception {
+ SpecVerificationReport specVerificationReport = new SpecVerificationReport();
+ double actualDiskSpaceAvailable = 150D;
+ boolean actualIpv6Connection = false;
+ specVerificationReport.setActualDiskSpaceAvailable(actualDiskSpaceAvailable);
+ specVerificationReport.setActualIpv6Connection(actualIpv6Connection);
+ String expectedReport = "{\"specVerificationReport\":{\"actualDiskSpaceAvailable\":150.0,\"actualIpv6Connection\":false},\"benchmarkReport\":{\"cpuCyclesPerSec\":0.5}}";
+ Reporter.reportSpecVerificationResults(specVerificationReport, nodeInfoUrlsToValidHardwareDivergence);
+ assertEquals(expectedReport, println.toString());
+ }
+
+ @Test
+ public void reportSpecVerificationResults_make_new_correct_hardwareDivergence_because_old_is_wrong() throws Exception {
+ SpecVerificationReport specVerificationReport = new SpecVerificationReport();
+ double actualDiskSpaceAvailable = 150D;
+ specVerificationReport.setActualDiskSpaceAvailable(actualDiskSpaceAvailable);
+ String expectedReport = "{\"specVerificationReport\":{\"actualDiskSpaceAvailable\":150.0}}";
+ Reporter.reportSpecVerificationResults(specVerificationReport, nodeInfoUrlsToNOTValidHardwareDivergence);
+ assertEquals(expectedReport, println.toString());
+ }
+
+ @Test
+ public void reportSpecVerificationResults_make_new_empty_hardwareDivergence_because_there_is_no_old() throws Exception {
+ SpecVerificationReport specVerificationReport = new SpecVerificationReport();
+ String expectedReport = "null";
+ Reporter.reportSpecVerificationResults(specVerificationReport, nodeInfoUrlsWithNoHardwareDivergence);
+ assertEquals(expectedReport, println.toString());
+ }
+
+} \ No newline at end of file
diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/commons/report/SpecVerificationReportTest.java b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/commons/report/SpecVerificationReportTest.java
index 0b19e1b758c..72629cc565d 100644
--- a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/commons/report/SpecVerificationReportTest.java
+++ b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/commons/report/SpecVerificationReportTest.java
@@ -2,13 +2,17 @@
package com.yahoo.vespa.hosted.node.verification.commons.report;
import com.fasterxml.jackson.databind.ObjectMapper;
-import com.yahoo.vespa.hosted.node.verification.commons.report.SpecVerificationReport;
import com.yahoo.vespa.hosted.node.verification.mock.MockCommandExecutor;
import com.yahoo.vespa.hosted.node.verification.spec.retrievers.HardwareInfo;
import org.junit.Before;
import org.junit.Test;
-import static org.junit.Assert.*;
+import static org.junit.Assert.assertEquals;
+
+/**
+ * @author sgrostad
+ * @author olaaun
+ */
public class SpecVerificationReportTest {
diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/BenchmarkResultInspectorTest.java b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/BenchmarkResultInspectorTest.java
index b1a98acd78a..4c999dba35d 100644
--- a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/BenchmarkResultInspectorTest.java
+++ b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/BenchmarkResultInspectorTest.java
@@ -1,17 +1,18 @@
// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.hosted.node.verification.hardware;
-import com.yahoo.vespa.hosted.node.verification.hardware.benchmarks.BenchmarkResults;
import com.yahoo.vespa.hosted.node.verification.commons.report.BenchmarkReport;
+import com.yahoo.vespa.hosted.node.verification.hardware.benchmarks.BenchmarkResults;
import org.junit.Before;
import org.junit.Test;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
-import static org.junit.Assert.assertTrue;
+/**
+ * @author sgrostad
+ * @author olaaun
+ */
public class BenchmarkResultInspectorTest {
diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/HardwareBenchmarkerTest.java b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/HardwareBenchmarkerTest.java
index 7b4ad9b967b..6848fc045e2 100644
--- a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/HardwareBenchmarkerTest.java
+++ b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/HardwareBenchmarkerTest.java
@@ -12,6 +12,11 @@ import java.util.Arrays;
import static org.junit.Assert.assertTrue;
+/**
+ * @author sgrostad
+ * @author olaaun
+ */
+
public class HardwareBenchmarkerTest {
private MockCommandExecutor mockCommandExecutor;
diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/CPUBenchmarkTest.java b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/CPUBenchmarkTest.java
index b2655aca26e..04e9057b44e 100644
--- a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/CPUBenchmarkTest.java
+++ b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/CPUBenchmarkTest.java
@@ -9,14 +9,17 @@ import org.junit.Test;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.List;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
/**
- * Created by sgrostad on 11/07/2017.
+ * @author sgrostad
+ * @author olaaun
*/
+
public class CPUBenchmarkTest {
private static final String cpuEuropeanDelimiters = "src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/resources/cpuCyclesWithDotsTimeWithCommaTest.txt";
@@ -56,8 +59,8 @@ public class CPUBenchmarkTest {
@Test
public void parseCpuCyclesPerSec_return_correct_ArrayList() throws IOException {
- ArrayList<String> mockCommandOutput = MockCommandExecutor.readFromFile(cpuEuropeanDelimiters);
- ArrayList<ParseResult> parseResults = cpu.parseCpuCyclesPerSec(mockCommandOutput);
+ List<String> mockCommandOutput = MockCommandExecutor.readFromFile(cpuEuropeanDelimiters);
+ List<ParseResult> parseResults = cpu.parseCpuCyclesPerSec(mockCommandOutput);
ParseResult expectedParseCyclesResult = new ParseResult("cycles", "2.066.201.729");
ParseResult expectedParseSecondsResult = new ParseResult("seconds", "0,957617512");
assertEquals(expectedParseCyclesResult, parseResults.get(0));
@@ -66,7 +69,7 @@ public class CPUBenchmarkTest {
@Test
public void test_if_setCpuCyclesPerSec_reads_output_correctly() throws IOException {
- ArrayList<ParseResult> parseResults = new ArrayList<>();
+ List<ParseResult> parseResults = new ArrayList<>();
parseResults.add(new ParseResult("cycles", "2.066.201.729"));
parseResults.add(new ParseResult("seconds", "0,957617512"));
cpu.setCpuCyclesPerSec(parseResults);
diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/DiskBenchmarkTest.java b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/DiskBenchmarkTest.java
index 8bc0266e3b5..8254718f36d 100644
--- a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/DiskBenchmarkTest.java
+++ b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/DiskBenchmarkTest.java
@@ -7,14 +7,17 @@ import org.junit.Before;
import org.junit.Test;
import java.util.ArrayList;
+import java.util.List;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
/**
- * Created by sgrostad on 12/07/2017.
+ * @author sgrostad
+ * @author olaaun
*/
+
public class DiskBenchmarkTest {
private DiskBenchmark diskBenchmark;
@@ -54,7 +57,7 @@ public class DiskBenchmarkTest {
@Test
public void parseDiskSpeed_valid_input() throws Exception {
- ArrayList<String> mockCommandOutput = MockCommandExecutor.readFromFile(VALID_OUTPUT_FILE);
+ List<String> mockCommandOutput = MockCommandExecutor.readFromFile(VALID_OUTPUT_FILE);
ParseResult parseResult = diskBenchmark.parseDiskSpeed(mockCommandOutput);
ParseResult expectedParseResult = new ParseResult("MB/s", "243");
assertEquals(expectedParseResult, parseResult);
@@ -62,7 +65,7 @@ public class DiskBenchmarkTest {
@Test
public void parseDiskSpeed_invalid_input() throws Exception {
- ArrayList<String> mockCommandOutput = MockCommandExecutor.readFromFile(INVALID_OUTPUT_FILE);
+ List<String> mockCommandOutput = MockCommandExecutor.readFromFile(INVALID_OUTPUT_FILE);
ParseResult parseResult = diskBenchmark.parseDiskSpeed(mockCommandOutput);
ParseResult expectedParseResult = new ParseResult("invalid", "invalid");
assertEquals(expectedParseResult, parseResult);
diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/MemoryBenchmarkTest.java b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/MemoryBenchmarkTest.java
index b44fcd8903e..e2c5cdfa555 100644
--- a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/MemoryBenchmarkTest.java
+++ b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/MemoryBenchmarkTest.java
@@ -6,15 +6,17 @@ import com.yahoo.vespa.hosted.node.verification.mock.MockCommandExecutor;
import org.junit.Before;
import org.junit.Test;
-import java.util.ArrayList;
+import java.util.List;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
/**
- * Created by olaa on 14/07/2017.
+ * @author sgrostad
+ * @author olaaun
*/
+
public class MemoryBenchmarkTest {
private MemoryBenchmark memoryBenchmark;
@@ -48,7 +50,7 @@ public class MemoryBenchmarkTest {
public void parseMemorySpeed_valid_output() throws Exception {
Double expectedSpeed = 12.1;
String mockOutput = "This is a test \n the memory speed to be found is " + expectedSpeed + " GB/s";
- ArrayList<String> mockCommandOutput = commandExecutor.outputFromString(mockOutput);
+ List<String> mockCommandOutput = commandExecutor.outputFromString(mockOutput);
ParseResult parseResult = memoryBenchmark.parseMemorySpeed(mockCommandOutput);
ParseResult expectedParseResult = new ParseResult("GB/s", expectedSpeed.toString());
assertEquals(expectedParseResult, parseResult);
@@ -56,7 +58,7 @@ public class MemoryBenchmarkTest {
@Test
public void parseMemorySpeed_invalid_output() throws Exception {
- ArrayList<String> mockCommandOutput = commandExecutor.outputFromString("");
+ List<String> mockCommandOutput = commandExecutor.outputFromString("");
ParseResult parseResult = memoryBenchmark.parseMemorySpeed(mockCommandOutput);
ParseResult expectedParseResult = new ParseResult("invalid", "invalid");
assertEquals(expectedParseResult, parseResult);
diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/mock/MockCommandExecutor.java b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/mock/MockCommandExecutor.java
index 83f542e7142..6e89b91aa94 100644
--- a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/mock/MockCommandExecutor.java
+++ b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/mock/MockCommandExecutor.java
@@ -8,10 +8,13 @@ import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.List;
/**
- * Created by olaa on 17/07/2017.
+ * @author sgrostad
+ * @author olaaun
*/
+
public class MockCommandExecutor extends CommandExecutor {
private ArrayList<String> mockCommands;
@@ -24,7 +27,7 @@ public class MockCommandExecutor extends CommandExecutor {
}
@Override
- public ArrayList<String> executeCommand(String command) throws IOException {
+ public List<String> executeCommand(String command) throws IOException {
String mockCommand = mockCommands.get(counter++);
if (mockCommand.equals(DUMMY_COMMAND)) return null;
return super.executeCommand(mockCommand);
@@ -38,11 +41,11 @@ public class MockCommandExecutor extends CommandExecutor {
mockCommands.add(DUMMY_COMMAND);
}
- public static ArrayList<String> readFromFile(String filepath) throws IOException {
+ public static List<String> readFromFile(String filepath) throws IOException {
return new ArrayList<>(Arrays.asList(new String(Files.readAllBytes(Paths.get(filepath))).split("\n")));
}
- public ArrayList<String> outputFromString(String output) {
+ public List<String> outputFromString(String output) {
return new ArrayList<>(Arrays.asList(output.split("\n")));
}
diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/HardwareNodeComparatorTest.java b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/HardwareNodeComparatorTest.java
index 6113ce315c1..22d1a167a13 100644
--- a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/HardwareNodeComparatorTest.java
+++ b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/HardwareNodeComparatorTest.java
@@ -8,12 +8,12 @@ import org.junit.Before;
import org.junit.Test;
import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
/**
- * Created by olaa on 07/07/2017.
+ * @author sgrostad
+ * @author olaaun
*/
+
public class HardwareNodeComparatorTest {
private HardwareInfo actualHardware;
diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/SpecVerifierTest.java b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/SpecVerifierTest.java
index 61f52537e74..16e72521853 100644
--- a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/SpecVerifierTest.java
+++ b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/SpecVerifierTest.java
@@ -2,11 +2,11 @@
package com.yahoo.vespa.hosted.node.verification.spec;
import com.fasterxml.jackson.databind.ObjectMapper;
-import com.yahoo.vespa.hosted.node.verification.mock.MockCommandExecutor;
import com.yahoo.vespa.hosted.node.verification.commons.noderepo.NodeRepoInfoRetriever;
import com.yahoo.vespa.hosted.node.verification.commons.noderepo.NodeRepoJsonModel;
-import com.yahoo.vespa.hosted.node.verification.spec.retrievers.HardwareInfo;
import com.yahoo.vespa.hosted.node.verification.commons.report.SpecVerificationReport;
+import com.yahoo.vespa.hosted.node.verification.mock.MockCommandExecutor;
+import com.yahoo.vespa.hosted.node.verification.spec.retrievers.HardwareInfo;
import org.junit.Before;
import org.junit.Test;
@@ -14,11 +14,17 @@ import java.io.File;
import java.net.URL;
import java.nio.file.Paths;
import java.util.ArrayList;
+import java.util.List;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
+/**
+ * @author sgrostad
+ * @author olaaun
+ */
+
public class SpecVerifierTest {
private MockCommandExecutor mockCommandExecutor;
@@ -36,7 +42,7 @@ public class SpecVerifierTest {
private static final String PING_RESPONSE = RESOURCE_PATH + "/validpingresponse";
private static final String INVALID_PING_RESPONSE = RESOURCE_PATH + "/pingresponse-all-packets-lost";
private static final double DELTA = 0.1;
- ArrayList<URL> nodeInfoUrls;
+ List<URL> nodeInfoUrls;
@Before
public void setup() {
diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/IPAddressVerifierTest.json b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/IPAddressVerifierTest.json
index 4fcb5efed9e..a3bd4957467 100644
--- a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/IPAddressVerifierTest.json
+++ b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/IPAddressVerifierTest.json
@@ -77,6 +77,5 @@
"2001:db8:0:1234:0:567:8:1"
],
"additionalIpAddresses": [
-
]
} \ No newline at end of file
diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/nodeInfoNotInterpretableHardwareDivergence.json b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/nodeInfoNotInterpretableHardwareDivergence.json
new file mode 100644
index 00000000000..0faf1873446
--- /dev/null
+++ b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/nodeInfoNotInterpretableHardwareDivergence.json
@@ -0,0 +1,82 @@
+{
+ "url": "https://api.vespayahoo.com",
+ "id": "id.prod.yahoo.com",
+ "state": "failed",
+ "type": "host",
+ "hostname": "hostname",
+ "openStackId": "4a8689e1-3a8e-4f50-b99f-d33b245a9661",
+ "flavor": "C-2T/24/500",
+ "canonicalFlavor": "C-2B/24/500",
+ "minDiskAvailableGb": 500.0,
+ "minMainMemoryAvailableGb": 24.0,
+ "description": "BARE_METAL with 24.0 CPUs, 24.0 Gb memory and 500.0 Gb disk",
+ "minCpuCores": 24.0,
+ "cost": 70,
+ "fastDisk": false,
+ "environment": "BARE_METAL",
+ "owner": {
+ "tenant": "hosted-vespa",
+ "application": "routing",
+ "instance": "default"
+ },
+ "membership": {
+ "clustertype": "container",
+ "clusterid": "node-admin",
+ "group": "0",
+ "index": 32,
+ "retired": false
+ },
+ "restartGeneration": 0,
+ "currentRestartGeneration": 0,
+ "wantedDockerImage": "docker-registry.ops.yahoo.com:4443/vespa/ci:6.138.1",
+ "wantedVespaVersion": "6.138.1",
+ "rebootGeneration": 49,
+ "currentRebootGeneration": 49,
+ "vespaVersion": "6.138.18",
+ "currentDockerImage": "docker-registry.ops.yahoo.com:4443/vespa/ci:6.138.18",
+ "hostedVersion": "6.138.18",
+ "convergedStateVersion": "6.138.18",
+ "failCount": 44,
+ "hardwareFailure": true,
+ "hardwareFailureType": "disk_smart",
+ "hardwareFailureDescription": "disk_smart",
+ "wantToRetire": false,
+ "wantToDeprovision": false,
+ "history": [
+ {
+ "event": "readied",
+ "at": 1501070910372,
+ "agent": "system"
+ },
+ {
+ "event": "reserved",
+ "at": 1501070952692,
+ "agent": "application"
+ },
+ {
+ "event": "activated",
+ "at": 1501070958698,
+ "agent": "application"
+ },
+ {
+ "event": "rebooted",
+ "at": 1501656469859,
+ "agent": "system"
+ },
+ {
+ "event": "down",
+ "at": 1502809344596,
+ "agent": "system"
+ },
+ {
+ "event": "failed",
+ "at": 1502812976621,
+ "agent": "system"
+ }
+ ],
+ "ipAddresses": [
+ ],
+ "additionalIpAddresses": [
+ ],
+ "hardwareDivergence": "{\"hardwareDivergence\": null - it will not manage to parse this}"
+} \ No newline at end of file
diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/nodeInfoTest.json b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/nodeInfoTest.json
index 216991846b0..c557afb3256 100644
--- a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/nodeInfoTest.json
+++ b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/nodeInfoTest.json
@@ -45,6 +45,5 @@
"10.200.66.16"
],
"additionalIpAddresses": [],
-
"hardwareDivergence": "{\"benchmarkReport\": {\"cpuCyclesPerSec\": 0.5}}"
} \ No newline at end of file
diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/CPURetrieverTest.java b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/CPURetrieverTest.java
index cda8066734d..d878d6614f7 100644
--- a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/CPURetrieverTest.java
+++ b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/CPURetrieverTest.java
@@ -8,12 +8,15 @@ import org.junit.Test;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.List;
import static org.junit.Assert.assertEquals;
/**
- * Created by olaa on 03/07/2017.
+ * @author sgrostad
+ * @author olaaun
*/
+
public class CPURetrieverTest {
private static final String FILENAME = "src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/cpuinfoTest";
@@ -39,8 +42,8 @@ public class CPURetrieverTest {
@Test
public void parseCPUInfoFile_should_return_valid_ArrayList() throws IOException {
- ArrayList<String> commandOutput = MockCommandExecutor.readFromFile(FILENAME);
- ArrayList<ParseResult> ParseResults = cpu.parseCPUInfoFile(commandOutput);
+ List<String> commandOutput = MockCommandExecutor.readFromFile(FILENAME);
+ List<ParseResult> ParseResults = cpu.parseCPUInfoFile(commandOutput);
String expectedSearchWord = "cpu MHz";
String expectedValue = "2493.821";
@@ -59,7 +62,7 @@ public class CPURetrieverTest {
@Test
public void setCpuCores_counts_cores_correctly() {
- ArrayList<ParseResult> parseResults = new ArrayList<>();
+ List<ParseResult> parseResults = new ArrayList<>();
parseResults.add(new ParseResult("cpu MHz", "2000"));
parseResults.add(new ParseResult("cpu MHz", "2000"));
parseResults.add(new ParseResult("cpu MHz", "2000"));
diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/DiskRetrieverTest.java b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/DiskRetrieverTest.java
index 6cd29ab8f64..de4bf946d59 100644
--- a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/DiskRetrieverTest.java
+++ b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/DiskRetrieverTest.java
@@ -8,14 +8,16 @@ import org.junit.Before;
import org.junit.Test;
import java.io.IOException;
-import java.util.ArrayList;
+import java.util.List;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.fail;
/**
- * Created by olaa on 06/07/2017.
+ * @author sgrostad
+ * @author olaaun
*/
+
public class DiskRetrieverTest {
private MockCommandExecutor commandExecutor;
@@ -59,7 +61,7 @@ public class DiskRetrieverTest {
@Test
public void parseDiskType_should_find_fast_disk() throws Exception {
diskRetriever = new DiskRetriever(hardwareInfo, commandExecutor);
- ArrayList<String> mockOutput = commandExecutor.outputFromString("Name Rota \nsda 0");
+ List<String> mockOutput = commandExecutor.outputFromString("Name Rota \nsda 0");
ParseResult parseResult = diskRetriever.parseDiskType(mockOutput);
ParseResult expectedParseResult = new ParseResult("sda", "0");
assertEquals(expectedParseResult, parseResult);
@@ -67,7 +69,7 @@ public class DiskRetrieverTest {
@Test
public void parseDiskType_should_not_find_fast_disk() throws Exception {
- ArrayList<String> mockOutput = commandExecutor.outputFromString("Name Rota \nsda 1");
+ List<String> mockOutput = commandExecutor.outputFromString("Name Rota \nsda 1");
ParseResult parseResult = diskRetriever.parseDiskType(mockOutput);
ParseResult expectedParseResult = new ParseResult("sda", "1");
assertEquals(expectedParseResult, parseResult);
@@ -75,7 +77,7 @@ public class DiskRetrieverTest {
@Test
public void parseDiskType_with_invalid_outputstream_does_not_contain_searchword_should_throw_exception() throws Exception {
- ArrayList<String> mockOutput = commandExecutor.outputFromString("Name Rota");
+ List<String> mockOutput = commandExecutor.outputFromString("Name Rota");
try {
ParseResult parseResult = diskRetriever.parseDiskType(mockOutput);
fail("Should have thrown IOException when outputstream doesn't contain search word");
@@ -89,8 +91,8 @@ public class DiskRetrieverTest {
@Test
public void parseDiskSize_should_find_size_from_file_and_insert_into_parseResult() throws Exception {
String filepath = "src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/filesize";
- ArrayList<String> mockOutput = MockCommandExecutor.readFromFile(filepath);
- ArrayList<ParseResult> parseResults = diskRetriever.parseDiskSize(mockOutput);
+ List<String> mockOutput = MockCommandExecutor.readFromFile(filepath);
+ List<ParseResult> parseResults = diskRetriever.parseDiskSize(mockOutput);
ParseResult expectedParseResult1 = new ParseResult("Size", "799.65");
assertEquals(expectedParseResult1, parseResults.get(0));
ParseResult expectedParseResult2 = new ParseResult("Size", "960.19");
diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/HardwareInfoRetrieverTest.java b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/HardwareInfoRetrieverTest.java
index ae131e0c4e8..552d57d2b59 100644
--- a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/HardwareInfoRetrieverTest.java
+++ b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/HardwareInfoRetrieverTest.java
@@ -6,10 +6,15 @@ import com.yahoo.vespa.hosted.node.verification.mock.MockCommandExecutor;
import com.yahoo.vespa.hosted.node.verification.spec.VerifierSettings;
import org.junit.Before;
import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
import static org.mockito.Mockito.doReturn;
import static org.mockito.Mockito.spy;
-import static org.junit.Assert.assertEquals;
+/**
+ * @author sgrostad
+ * @author olaaun
+ */
public class HardwareInfoRetrieverTest {
diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/MemoryRetrieverTest.java b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/MemoryRetrieverTest.java
index 9d20acbc762..6b0b5f6d24a 100644
--- a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/MemoryRetrieverTest.java
+++ b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/MemoryRetrieverTest.java
@@ -7,13 +7,15 @@ import org.junit.Before;
import org.junit.Test;
import java.io.IOException;
-import java.util.ArrayList;
+import java.util.List;
import static org.junit.Assert.assertEquals;
/**
- * Created by sgrostad on 06/07/2017.
+ * @author sgrostad
+ * @author olaaun
*/
+
public class MemoryRetrieverTest {
private static final String FILENAME = "src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/meminfoTest";
@@ -39,7 +41,7 @@ public class MemoryRetrieverTest {
@Test
public void parseMemInfoFile_should_return_valid_parseResult() throws IOException {
- ArrayList<String> commandOutput = MockCommandExecutor.readFromFile(FILENAME);
+ List<String> commandOutput = MockCommandExecutor.readFromFile(FILENAME);
ParseResult parseResult = memory.parseMemInfoFile(commandOutput);
ParseResult expectedParseResult = new ParseResult("MemTotal", "4042128 kB");
assertEquals(expectedParseResult, parseResult);
diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/NetRetrieverTest.java b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/NetRetrieverTest.java
index 893f23d6133..565712608f5 100644
--- a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/NetRetrieverTest.java
+++ b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/NetRetrieverTest.java
@@ -10,6 +10,7 @@ import org.junit.Test;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.List;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
@@ -19,8 +20,10 @@ import static org.mockito.Mockito.doReturn;
import static org.mockito.Mockito.spy;
/**
- * Created by sgrostad on 07/07/2017.
+ * @author sgrostad
+ * @author olaaun
*/
+
public class NetRetrieverTest {
private static final String RESOURCE_PATH = "src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/";
@@ -32,7 +35,7 @@ public class NetRetrieverTest {
private HardwareInfo hardwareInfo;
private MockCommandExecutor commandExecutor;
private NetRetriever net;
- private ArrayList<ParseResult> parseResults;
+ private List<ParseResult> parseResults;
private VerifierSettings verifierSettings = spy(new VerifierSettings());
private static final double DELTA = 0.1;
@@ -70,7 +73,7 @@ public class NetRetrieverTest {
@Test
public void parseNetInterface_get_ipv_from_ifconfig_testFile() throws IOException {
- ArrayList<String> mockOutput = MockCommandExecutor.readFromFile(NET_FIND_INTERFACE);
+ List<String> mockOutput = MockCommandExecutor.readFromFile(NET_FIND_INTERFACE);
parseResults = net.parseNetInterface(mockOutput);
net.updateHardwareInfoWithNet(parseResults);
assertTrue(hardwareInfo.getIpv4Interface());
@@ -79,7 +82,7 @@ public class NetRetrieverTest {
@Test
public void parseNetInterface_get_ipv_from_ifconfigNotIpv6_testFile() throws IOException {
- ArrayList<String> mockOutput = MockCommandExecutor.readFromFile(NET_FIND_INTERFACE + "NoIpv6");
+ List<String> mockOutput = MockCommandExecutor.readFromFile(NET_FIND_INTERFACE + "NoIpv6");
parseResults = net.parseNetInterface(mockOutput);
ArrayList<ParseResult> expextedParseResults = new ArrayList<>(Arrays.asList(
new ParseResult("inet", "inet")));
@@ -88,7 +91,7 @@ public class NetRetrieverTest {
@Test
public void parseInterfaceSpeed_get_interfaceSpeed_from_eth0_testFile() throws IOException {
- ArrayList<String> mockOutput = MockCommandExecutor.readFromFile("src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/eth0");
+ List<String> mockOutput = MockCommandExecutor.readFromFile("src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/eth0");
ParseResult parseResult = net.parseInterfaceSpeed(mockOutput);
ParseResult expectedParseResult = new ParseResult("Speed", "1000Mb/s");
assertEquals(expectedParseResult, parseResult);
@@ -117,7 +120,7 @@ public class NetRetrieverTest {
@Test
public void parsePingResponse_valid_ping_response_should_return_ipv6_connectivity() throws IOException {
- ArrayList<String> mockCommandOutput = MockCommandExecutor.readFromFile(VALID_PING_RESPONSE);
+ List<String> mockCommandOutput = MockCommandExecutor.readFromFile(VALID_PING_RESPONSE);
ParseResult parseResult = net.parsePingResponse(mockCommandOutput);
String expectedPing = "0";
assertEquals(expectedPing, parseResult.getValue());
@@ -125,7 +128,7 @@ public class NetRetrieverTest {
@Test
public void parsePingResponse_invalid_ping_response_should_throw_IOException() throws IOException {
- ArrayList<String> mockCommandOutput = MockCommandExecutor.readFromFile(INVALID_PING_RESPONSE);
+ List<String> mockCommandOutput = MockCommandExecutor.readFromFile(INVALID_PING_RESPONSE);
try {
ParseResult parseResult = net.parsePingResponse(mockCommandOutput);
fail("Expected an IOException to be thrown");
diff --git a/pom.xml b/pom.xml
index 60d997b7801..798738c02af 100644
--- a/pom.xml
+++ b/pom.xml
@@ -589,12 +589,7 @@
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
- <version>4.11</version>
- </dependency>
- <dependency>
- <groupId>junit</groupId>
- <artifactId>junit-dep</artifactId>
- <version>4.11</version>
+ <version>4.12</version>
</dependency>
<dependency>
<groupId>org.antlr</groupId>
diff --git a/searchlib/CMakeLists.txt b/searchlib/CMakeLists.txt
index c389cbdc8c6..0bd70592c3d 100644
--- a/searchlib/CMakeLists.txt
+++ b/searchlib/CMakeLists.txt
@@ -131,6 +131,7 @@ vespa_define_module(
src/tests/features/element_similarity_feature
src/tests/features/euclidean_distance
src/tests/features/imported_dot_product
+ src/tests/features/internal_max_reduce_prod_join_feature
src/tests/features/item_raw_score
src/tests/features/native_dot_product
src/tests/features/ranking_expression
@@ -198,6 +199,7 @@ vespa_define_module(
src/tests/queryeval/weak_and_scorers
src/tests/queryeval/weighted_set_term
src/tests/rankingexpression/feature_name_extractor
+ src/tests/rankingexpression/intrinsic_blueprint_adapter
src/tests/ranksetup
src/tests/ranksetup/verify_feature
src/tests/sort
diff --git a/searchlib/src/tests/features/internal_max_reduce_prod_join_feature/.gitignore b/searchlib/src/tests/features/internal_max_reduce_prod_join_feature/.gitignore
new file mode 100644
index 00000000000..752721ef65a
--- /dev/null
+++ b/searchlib/src/tests/features/internal_max_reduce_prod_join_feature/.gitignore
@@ -0,0 +1 @@
+searchlib_internal_max_reduce_prod_join_feature_test_app
diff --git a/searchlib/src/tests/features/internal_max_reduce_prod_join_feature/CMakeLists.txt b/searchlib/src/tests/features/internal_max_reduce_prod_join_feature/CMakeLists.txt
new file mode 100644
index 00000000000..65b8056cb54
--- /dev/null
+++ b/searchlib/src/tests/features/internal_max_reduce_prod_join_feature/CMakeLists.txt
@@ -0,0 +1,8 @@
+# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_executable(searchlib_internal_max_reduce_prod_join_feature_test_app TEST
+ SOURCES
+ internal_max_reduce_prod_join_feature_test.cpp
+ DEPENDS
+ searchlib
+)
+vespa_add_test(NAME searchlib_internal_max_reduce_prod_join_feature_test_app COMMAND searchlib_internal_max_reduce_prod_join_feature_test_app)
diff --git a/searchlib/src/tests/features/internal_max_reduce_prod_join_feature/internal_max_reduce_prod_join_feature_test.cpp b/searchlib/src/tests/features/internal_max_reduce_prod_join_feature/internal_max_reduce_prod_join_feature_test.cpp
new file mode 100644
index 00000000000..72e2240a12a
--- /dev/null
+++ b/searchlib/src/tests/features/internal_max_reduce_prod_join_feature/internal_max_reduce_prod_join_feature_test.cpp
@@ -0,0 +1,173 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/vespalib/testkit/test_kit.h>
+
+#include <vespa/searchlib/attribute/attributefactory.h>
+#include <vespa/searchlib/fef/test/ftlib.h>
+#include <vespa/searchlib/features/internal_max_reduce_prod_join_feature.h>
+#include <vespa/searchlib/attribute/attribute.h>
+
+using search::feature_t;
+using namespace search::fef;
+using namespace search::fef::test;
+using namespace search::features;
+using search::AttributeFactory;
+using search::IntegerAttribute;
+using search::FloatingPointAttribute;
+using CollectionType = FieldInfo::CollectionType;
+using DataType = FieldInfo::DataType;
+
+typedef search::attribute::Config AVC;
+typedef search::attribute::BasicType AVBT;
+typedef search::attribute::CollectionType AVCT;
+typedef search::AttributeVector::SP AttributePtr;
+typedef FtTestApp FTA;
+
+struct SetupFixture
+{
+ InternalMaxReduceProdJoinBlueprint blueprint;
+ IndexEnvironment indexEnv;
+ SetupFixture()
+ : blueprint(),
+ indexEnv()
+ {
+ addAttribute("long", CollectionType::SINGLE, DataType::INT64);
+ addAttribute("longarray", CollectionType::ARRAY, DataType::INT64);
+ addAttribute("intarray", CollectionType::ARRAY, DataType::INT32);
+ addAttribute("doublearray", CollectionType::ARRAY, DataType::DOUBLE);
+ }
+
+ void addAttribute(const vespalib::string &name, const CollectionType &collType, const DataType &dataType) {
+ FieldInfo attrInfo(FieldType::ATTRIBUTE, collType, name, 0);
+ attrInfo.set_data_type(dataType);
+ indexEnv.getFields().push_back(attrInfo);
+ }
+};
+
+TEST_F("require that blueprint can be created", SetupFixture())
+{
+ EXPECT_TRUE(FTA::assertCreateInstance(f.blueprint, "internalMaxReduceProdJoin"));
+}
+
+TEST_F("require that setup fails if attribute does not exist", SetupFixture())
+{
+ FTA::FT_SETUP_FAIL(f.blueprint, f.indexEnv, StringList().add("foo").add("bar"));
+}
+
+TEST_F("require that setup fails if attribute is of wrong type", SetupFixture())
+{
+ FTA::FT_SETUP_FAIL(f.blueprint, f.indexEnv, StringList().add("long").add("bar"));
+}
+
+TEST_F("require that setup fails if attribute is of wrong array type", SetupFixture())
+{
+ FTA::FT_SETUP_FAIL(f.blueprint, f.indexEnv, StringList().add("doublearray").add("bar"));
+}
+
+TEST_F("require that setup succeeds with long array attribute", SetupFixture())
+{
+ FTA::FT_SETUP_OK(f.blueprint, f.indexEnv,
+ StringList().add("longarray").add("query"),
+ StringList(),
+ StringList().add("scalar"));
+}
+
+TEST_F("require that setup succeeds with int array attribute", SetupFixture())
+{
+ FTA::FT_SETUP_OK(f.blueprint, f.indexEnv,
+ StringList().add("intarray").add("query"),
+ StringList(),
+ StringList().add("scalar"));
+}
+
+struct ExecFixture
+{
+ BlueprintFactory factory;
+ FtFeatureTest test;
+ ExecFixture(const vespalib::string &feature)
+ : factory(),
+ test(factory, feature)
+ {
+ factory.addPrototype(std::make_shared<InternalMaxReduceProdJoinBlueprint>());
+ setupAttributeVectors();
+ setupQueryEnvironment();
+ ASSERT_TRUE(test.setup());
+ }
+
+ void setupAttributeVectors() {
+ vespalib::string attrIntArray = "intarray";
+ vespalib::string attrLongArray = "longarray";
+
+ test.getIndexEnv().getBuilder().addField(FieldType::ATTRIBUTE, CollectionType::ARRAY, DataType::INT64, attrLongArray);
+ test.getIndexEnv().getBuilder().addField(FieldType::ATTRIBUTE, CollectionType::ARRAY, DataType::INT32, attrIntArray);
+
+ std::vector<AttributePtr> attrs;
+ attrs.push_back(AttributeFactory::createAttribute(attrLongArray, AVC(AVBT::INT64, AVCT::ARRAY)));
+ attrs.push_back(AttributeFactory::createAttribute(attrIntArray, AVC(AVBT::INT32, AVCT::ARRAY)));
+ for (const auto &attr : attrs) {
+ attr->addReservedDoc();
+ attr->addDocs(1);
+ test.getIndexEnv().getAttributeMap().add(attr);
+ }
+
+ IntegerAttribute *longArray = static_cast<IntegerAttribute *>(attrs[0].get());
+ longArray->append(1, 1111, 0);
+ longArray->append(1, 2222, 0);
+ longArray->append(1, 78, 0);
+
+ IntegerAttribute *intArray = static_cast<IntegerAttribute *>(attrs[1].get());
+ intArray->append(1, 78, 0);
+ intArray->append(1, 1111, 0);
+
+ for (const auto &attr : attrs) {
+ attr->commit();
+ }
+ }
+
+ void setupQueryEnvironment() {
+ test.getQueryEnv().getProperties().add("wset", "{1111:1234, 2222:2245}");
+ test.getQueryEnv().getProperties().add("wsetnomatch", "{1:1000, 2:2000}");
+ test.getQueryEnv().getProperties().add("array", "[1111,2222]");
+ test.getQueryEnv().getProperties().add("negativewset", "{1111:-1000, 78:-42}");
+ }
+
+ bool evaluatesTo(feature_t expectedValue) {
+ return test.execute(expectedValue);
+ }
+
+};
+
+TEST_F("require that executor returns correct result for long array",
+ ExecFixture("internalMaxReduceProdJoin(longarray,wset)"))
+{
+ EXPECT_FALSE(f.evaluatesTo(1234));
+ EXPECT_TRUE(f.evaluatesTo(2245));
+}
+
+TEST_F("require that executor returns correct result for int array",
+ ExecFixture("internalMaxReduceProdJoin(intarray,wset)"))
+{
+ EXPECT_TRUE(f.evaluatesTo(1234));
+ EXPECT_FALSE(f.evaluatesTo(2245));
+}
+
+TEST_F("require that executor returns 0 if no items match",
+ ExecFixture("internalMaxReduceProdJoin(longarray,wsetnomatch)"))
+{
+ EXPECT_TRUE(f.evaluatesTo(0.0));
+}
+
+TEST_F("require that executor return 0 if query is not a weighted set",
+ ExecFixture("internalMaxReduceProdJoin(longarray,array)"))
+{
+ EXPECT_TRUE(f.evaluatesTo(0.0));
+}
+
+TEST_F("require that executor supports negative numbers",
+ ExecFixture("internalMaxReduceProdJoin(intarray,negativewset)"))
+{
+ EXPECT_FALSE(f.evaluatesTo(-1000));
+ EXPECT_TRUE(f.evaluatesTo(-42));
+}
+
+TEST_MAIN() { TEST_RUN_ALL(); }
diff --git a/searchlib/src/tests/rankingexpression/intrinsic_blueprint_adapter/CMakeLists.txt b/searchlib/src/tests/rankingexpression/intrinsic_blueprint_adapter/CMakeLists.txt
new file mode 100644
index 00000000000..d8d6ebe0504
--- /dev/null
+++ b/searchlib/src/tests/rankingexpression/intrinsic_blueprint_adapter/CMakeLists.txt
@@ -0,0 +1,8 @@
+# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_executable(searchlib_intrinsic_blueprint_adapter_test_app TEST
+ SOURCES
+ intrinsic_blueprint_adapter_test.cpp
+ DEPENDS
+ searchlib
+)
+vespa_add_test(NAME searchlib_intrinsic_blueprint_adapter_test_app COMMAND searchlib_intrinsic_blueprint_adapter_test_app)
diff --git a/searchlib/src/tests/rankingexpression/intrinsic_blueprint_adapter/intrinsic_blueprint_adapter_test.cpp b/searchlib/src/tests/rankingexpression/intrinsic_blueprint_adapter/intrinsic_blueprint_adapter_test.cpp
new file mode 100644
index 00000000000..b10da86dd8c
--- /dev/null
+++ b/searchlib/src/tests/rankingexpression/intrinsic_blueprint_adapter/intrinsic_blueprint_adapter_test.cpp
@@ -0,0 +1,116 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/vespalib/testkit/test_kit.h>
+
+#include <vespa/searchlib/fef/blueprint.h>
+#include <vespa/searchlib/features/rankingexpression/intrinsic_blueprint_adapter.h>
+#include <vespa/searchlib/fef/test/indexenvironment.h>
+#include <vespa/searchlib/fef/test/queryenvironment.h>
+#include <set>
+
+using namespace search::features::rankingexpression;
+using namespace search::fef::test;
+using namespace search::fef;
+using vespalib::Stash;
+using vespalib::eval::ValueType;
+
+vespalib::string fail_setup = "fail_setup";
+vespalib::string extra_input = "extra_input";
+vespalib::string extra_output = "extra_output";
+vespalib::string no_output = "no_output";
+vespalib::string object_result = "object_result";
+vespalib::string error_result = "error_result";
+
+struct MyExecutor : FeatureExecutor {
+ void execute(uint32_t) override {}
+};
+
+struct MyBlueprint : Blueprint {
+ std::set<vespalib::string> flags;
+ MyBlueprint() : Blueprint("my_bp"), flags() {}
+ MyBlueprint(const std::set<vespalib::string> &flags_in) : Blueprint("my_bp"), flags(flags_in) {}
+ void set(const vespalib::string &flag) { flags.insert(flag); }
+ bool is_set(const vespalib::string &flag) const { return (flags.count(flag) > 0); }
+ void visitDumpFeatures(const IIndexEnvironment &, IDumpFeatureVisitor &) const override {}
+ Blueprint::UP createInstance() const override { return std::make_unique<MyBlueprint>(flags); }
+ bool setup(const IIndexEnvironment &, const std::vector<vespalib::string> &params) override {
+ EXPECT_EQUAL(getName(), "my_bp(foo,bar)");
+ ASSERT_TRUE(params.size() == 2);
+ EXPECT_EQUAL(params[0], "foo");
+ EXPECT_EQUAL(params[1], "bar");
+ if (is_set(extra_input)) {
+ defineInput("my_input", AcceptInput::ANY);
+ }
+ if (!is_set(no_output)) {
+ if (is_set(error_result)) {
+ describeOutput("out", "my output", FeatureType::object(ValueType::error_type()));
+ } else {
+ if (is_set(object_result)) {
+ describeOutput("out", "my output", FeatureType::object(ValueType::double_type()));
+ } else {
+ describeOutput("out", "my output", FeatureType::number());
+ }
+ }
+ if (is_set(extra_output)) {
+ describeOutput("extra", "my extra output", FeatureType::number());
+ }
+ }
+ return !is_set(fail_setup);
+ }
+ FeatureExecutor &createExecutor(const IQueryEnvironment &, vespalib::Stash &stash) const override {
+ return stash.create<MyExecutor>();
+ }
+};
+
+struct Fixture {
+ Stash stash;
+ IndexEnvironment idx_env;
+ QueryEnvironment query_env;
+ MyBlueprint blueprint;
+ Fixture() : stash(), idx_env(), query_env(&idx_env), blueprint() {}
+ IntrinsicExpression::UP create() const {
+ return IntrinsicBlueprintAdapter::try_create(blueprint, idx_env, {"foo", "bar"});
+ }
+};
+
+TEST_F("require that blueprints can be used for intrinsic expressions", Fixture()) {
+ auto expression = f1.create();
+ ASSERT_TRUE(bool(expression));
+ EXPECT_TRUE(!expression->result_type().is_object());
+ auto &executor = expression->create_executor(f1.query_env, f1.stash);
+ EXPECT_TRUE(dynamic_cast<MyExecutor*>(&executor) != nullptr);
+}
+
+TEST_F("require that result type is propagated for intrinsic blueprints", Fixture()) {
+ f1.blueprint.set(object_result);
+ auto expression = f1.create();
+ ASSERT_TRUE(bool(expression));
+ EXPECT_TRUE(expression->result_type().is_object());
+ EXPECT_TRUE(expression->result_type().type().is_double());
+}
+
+TEST_F("require that intrinsic blueprint adaption fails if blueprint setup fails", Fixture()) {
+ f1.blueprint.set(fail_setup);
+ EXPECT_TRUE(f1.create().get() == nullptr);
+}
+
+TEST_F("require that intrinsic blueprint adaption fails if blueprint has inputs", Fixture()) {
+ f1.blueprint.set(extra_input);
+ EXPECT_TRUE(f1.create().get() == nullptr);
+}
+
+TEST_F("require that intrinsic blueprint adaption fails if blueprint has more than one output", Fixture()) {
+ f1.blueprint.set(extra_output);
+ EXPECT_TRUE(f1.create().get() == nullptr);
+}
+
+TEST_F("require that intrinsic blueprint adaption fails if blueprint has no result", Fixture()) {
+ f1.blueprint.set(no_output);
+ EXPECT_TRUE(f1.create().get() == nullptr);
+}
+
+TEST_F("require that intrinsic blueprint adaption fails if blueprint has error typed output", Fixture()) {
+ f1.blueprint.set(error_result);
+ EXPECT_TRUE(f1.create().get() == nullptr);
+}
+
+TEST_MAIN() { TEST_RUN_ALL(); }
diff --git a/searchlib/src/vespa/searchlib/features/CMakeLists.txt b/searchlib/src/vespa/searchlib/features/CMakeLists.txt
index 6b212bdee0f..9227b845966 100644
--- a/searchlib/src/vespa/searchlib/features/CMakeLists.txt
+++ b/searchlib/src/vespa/searchlib/features/CMakeLists.txt
@@ -25,6 +25,7 @@ vespa_add_library(searchlib_features OBJECT
flow_completeness_feature.cpp
foreachfeature.cpp
freshnessfeature.cpp
+ internal_max_reduce_prod_join_feature.cpp
item_raw_score_feature.cpp
jarowinklerdistancefeature.cpp
matchcountfeature.cpp
diff --git a/searchlib/src/vespa/searchlib/features/internal_max_reduce_prod_join_feature.cpp b/searchlib/src/vespa/searchlib/features/internal_max_reduce_prod_join_feature.cpp
new file mode 100644
index 00000000000..d07654c0a21
--- /dev/null
+++ b/searchlib/src/vespa/searchlib/features/internal_max_reduce_prod_join_feature.cpp
@@ -0,0 +1,246 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include "internal_max_reduce_prod_join_feature.h"
+#include "valuefeature.h"
+#include "weighted_set_parser.h"
+
+#include <vespa/log/log.h>
+#include <vespa/searchlib/attribute/attribute.h>
+#include <vespa/searchlib/attribute/imported_attribute_vector.h>
+#include <vespa/searchlib/attribute/multinumericattribute.h>
+#include <vespa/searchlib/features/dotproductfeature.h>
+#include <vespa/searchlib/fef/properties.h>
+#include <vespa/searchlib/fef/featureexecutor.h>
+#include <vespa/searchcommon/common/datatype.h>
+
+LOG_SETUP(".features.internalmaxreduceprodjoin");
+
+using namespace search::attribute;
+using namespace search::fef;
+
+using search::features::dotproduct::wset::IntegerVector;
+
+namespace search {
+namespace features {
+
+/**
+ * Executor used when array can be accessed directly
+ */
+template <typename BaseType>
+class RawExecutor : public FeatureExecutor {
+protected:
+ const IAttributeVector *_attribute;
+ IntegerVector _queryVector;
+
+public:
+ RawExecutor(const IAttributeVector *attribute, const IntegerVector &queryVector);
+ void execute(uint32_t docId) override;
+};
+
+template <typename BaseType>
+RawExecutor<BaseType>::RawExecutor(const IAttributeVector *attribute,
+ const IntegerVector &queryVector) :
+ FeatureExecutor(),
+ _attribute(attribute),
+ _queryVector(queryVector)
+{
+ _queryVector.syncMap();
+}
+
+template <typename A, typename V>
+feature_t maxProduct(const A &array, size_t count, const V &query)
+{
+ feature_t val = -std::numeric_limits<double>::max();
+ for (size_t i = 0; i < count; ++i) {
+ auto itr = query.getDimMap().find(array[i].value());
+ if (itr != query.getDimMap().end()) {
+ feature_t v = itr->second; // weight from attribute is assumed to be 1.0
+ if (v > val) {
+ val = v;
+ }
+ }
+ }
+ return val == -std::numeric_limits<double>::max() ? 0.0 : val;
+}
+
+template <typename BaseType>
+void
+RawExecutor<BaseType>::execute(uint32_t docId)
+{
+ using A = IntegerAttributeTemplate<BaseType>;
+ const multivalue::Value<BaseType> *values(nullptr);
+ const A *iattr = dynamic_cast<const A *>(_attribute);
+ size_t count = iattr->getRawValues(docId, values);
+ outputs().set_number(0, maxProduct(values, count, _queryVector));
+}
+
+/**
+ * Executor when array can't be accessed directly
+ */
+template <typename BaseType>
+class BufferedExecutor : public RawExecutor<BaseType> {
+private:
+ WeightedIntegerContent _buffer;
+
+public:
+ BufferedExecutor(const IAttributeVector *attribute, const IntegerVector &queryVector);
+ void execute(uint32_t docId) override;
+};
+
+template <typename BaseType>
+BufferedExecutor<BaseType>::BufferedExecutor(const IAttributeVector *attribute, const IntegerVector &queryVector) :
+ RawExecutor<BaseType>(attribute, queryVector),
+ _buffer()
+{
+}
+
+
+template <typename BaseType>
+void
+BufferedExecutor<BaseType>::execute(uint32_t docId)
+{
+ _buffer.fill(*(this->_attribute), docId);
+ this->outputs().set_number(0, maxProduct(_buffer, _buffer.size(), this->_queryVector));
+}
+
+/**
+ * Blueprint
+ */
+InternalMaxReduceProdJoinBlueprint::InternalMaxReduceProdJoinBlueprint() :
+ Blueprint("internalMaxReduceProdJoin")
+{
+}
+
+InternalMaxReduceProdJoinBlueprint::~InternalMaxReduceProdJoinBlueprint()
+{
+}
+
+void
+InternalMaxReduceProdJoinBlueprint::visitDumpFeatures(const IIndexEnvironment &,
+ IDumpFeatureVisitor &) const
+{
+}
+
+Blueprint::UP
+InternalMaxReduceProdJoinBlueprint::createInstance() const
+{
+ return Blueprint::UP(new InternalMaxReduceProdJoinBlueprint());
+}
+
+ParameterDescriptions
+InternalMaxReduceProdJoinBlueprint::getDescriptions() const
+{
+ return ParameterDescriptions().desc().attribute(ParameterCollection::ARRAY).string();
+}
+
+bool supportedAttributeType(Parameter param) {
+ const FieldInfo *attributeInfo = param.asField();
+ if (attributeInfo == nullptr) {
+ return false;
+ }
+ if (attributeInfo->collection() != FieldInfo::CollectionType::ARRAY) {
+ return false;
+ }
+ if (attributeInfo->get_data_type() == FieldInfo::DataType::INT64) {
+ return true;
+ }
+ if (attributeInfo->get_data_type() == FieldInfo::DataType::INT32) {
+ return true;
+ }
+ return false;
+}
+
+bool
+InternalMaxReduceProdJoinBlueprint::setup(const IIndexEnvironment &env, const ParameterList &params)
+{
+ if (!supportedAttributeType(params[0])) {
+ return false;
+ }
+ _attribute = params[0].getValue();
+ _query = params[1].getValue();
+ describeOutput("scalar", "Internal executor for optimized execution of reduce(join(A,Q,f(x,y)(x*y)),max)");
+ env.hintAttributeAccess(_attribute);
+ return true;
+}
+
+bool isImportedAttribute(const IAttributeVector &attribute) noexcept {
+ return dynamic_cast<const ImportedAttributeVector*>(&attribute) != nullptr;
+}
+
+template<typename A>
+bool supportsGetRawValues(const A &attr) noexcept {
+ try {
+ const multivalue::Value<typename A::BaseType> *tmp = nullptr;
+ attr.getRawValues(0, tmp); // Throws if unsupported
+ return true;
+ } catch (const std::runtime_error &e) {
+ (void) e;
+ return false;
+ }
+}
+
+template <typename BaseType>
+FeatureExecutor &
+selectTypedExecutor(const IAttributeVector *attribute, const IntegerVector &vector, vespalib::Stash &stash)
+{
+ if (!isImportedAttribute(*attribute)) {
+ using A = IntegerAttributeTemplate<BaseType>;
+ using VT = multivalue::Value<BaseType>;
+ using ExactA = MultiValueNumericAttribute<A, VT>;
+
+ const A *iattr = dynamic_cast<const A *>(attribute);
+ if (supportsGetRawValues(*iattr)) {
+ const ExactA *exactA = dynamic_cast<const ExactA *>(iattr);
+ if (exactA != nullptr) {
+ return stash.create<RawExecutor<BaseType>>(attribute, vector);
+ }
+ }
+ }
+ return stash.create<BufferedExecutor<BaseType>>(attribute, vector);
+}
+
+FeatureExecutor &
+selectExecutor(const IAttributeVector *attribute, const IntegerVector &vector, vespalib::Stash &stash)
+{
+ if (attribute->getCollectionType() == CollectionType::ARRAY) {
+ switch (attribute->getBasicType()) {
+ case BasicType::INT32:
+ return selectTypedExecutor<int32_t>(attribute, vector, stash);
+ case BasicType::INT64:
+ return selectTypedExecutor<int64_t>(attribute, vector, stash);
+ default:
+ break;
+ }
+ }
+ LOG(warning, "The attribute vector '%s' is not of type "
+ "array<int/long>, returning executor with default value.", attribute->getName().c_str());
+ return stash.create<SingleZeroValueExecutor>();
+}
+
+
+FeatureExecutor &
+InternalMaxReduceProdJoinBlueprint::createExecutor(const IQueryEnvironment &env, vespalib::Stash &stash) const
+{
+ const IAttributeVector *attribute = env.getAttributeContext().getAttribute(_attribute);
+ if (attribute == nullptr) {
+ LOG(warning, "The attribute vector '%s' was not found in the attribute manager, "
+ "returning executor with default value.",
+ _attribute.c_str());
+ return stash.create<SingleZeroValueExecutor>();
+ }
+ Property prop = env.getProperties().lookup(_query);
+ if (prop.found() && !prop.get().empty()) {
+ IntegerVector vector;
+ WeightedSetParser::parse(prop.get(), vector);
+ if (!vector.getVector().empty()) {
+ return selectExecutor(attribute, vector, stash);
+ }
+ }
+ return stash.create<SingleZeroValueExecutor>();
+}
+
+
+}
+}
+
+
diff --git a/searchlib/src/vespa/searchlib/features/internal_max_reduce_prod_join_feature.h b/searchlib/src/vespa/searchlib/features/internal_max_reduce_prod_join_feature.h
new file mode 100644
index 00000000000..2c576b58345
--- /dev/null
+++ b/searchlib/src/vespa/searchlib/features/internal_max_reduce_prod_join_feature.h
@@ -0,0 +1,44 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/searchlib/fef/blueprint.h>
+
+namespace search {
+namespace features {
+
+/**
+ * Feature for the specific replacement of the expression:
+ *
+ * reduce(
+ * join(
+ * tensorFromLabels(attribute(A)),
+ * tensorFromWeightedset(query(Q)),
+ * f(x,y)(x*y)
+ * ),
+ * max
+ * )
+ *
+ * where A is an array attribute of int or long type and Q is a query that parses as
+ * a weighted set. This expression is replaced with this feature to avoid incurring
+ * the cost of creating temporary tensors.
+ */
+class InternalMaxReduceProdJoinBlueprint : public fef::Blueprint {
+private:
+ vespalib::string _attribute;
+ vespalib::string _query;
+
+public:
+ InternalMaxReduceProdJoinBlueprint();
+ ~InternalMaxReduceProdJoinBlueprint();
+
+ fef::ParameterDescriptions getDescriptions() const override;
+ fef::Blueprint::UP createInstance() const override;
+ bool setup(const fef::IIndexEnvironment &env, const fef::ParameterList &params) override;
+ fef::FeatureExecutor &createExecutor(const fef::IQueryEnvironment &env, vespalib::Stash &stash) const override;
+ void visitDumpFeatures(const fef::IIndexEnvironment &env, fef::IDumpFeatureVisitor &visitor) const override;
+
+};
+
+}
+}
diff --git a/searchlib/src/vespa/searchlib/features/rankingexpression/CMakeLists.txt b/searchlib/src/vespa/searchlib/features/rankingexpression/CMakeLists.txt
index 3ca8c36398c..68b4c4bb043 100644
--- a/searchlib/src/vespa/searchlib/features/rankingexpression/CMakeLists.txt
+++ b/searchlib/src/vespa/searchlib/features/rankingexpression/CMakeLists.txt
@@ -3,6 +3,7 @@ vespa_add_library(searchlib_features_rankingexpression OBJECT
SOURCES
expression_replacer.cpp
feature_name_extractor.cpp
+ intrinsic_blueprint_adapter.cpp
intrinsic_expression.cpp
DEPENDS
)
diff --git a/searchlib/src/vespa/searchlib/features/rankingexpression/intrinsic_blueprint_adapter.cpp b/searchlib/src/vespa/searchlib/features/rankingexpression/intrinsic_blueprint_adapter.cpp
new file mode 100644
index 00000000000..92dbfb75eda
--- /dev/null
+++ b/searchlib/src/vespa/searchlib/features/rankingexpression/intrinsic_blueprint_adapter.cpp
@@ -0,0 +1,75 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include "intrinsic_blueprint_adapter.h"
+#include <vespa/searchlib/fef/fef.h>
+
+using search::fef::Blueprint;
+using search::fef::FeatureExecutor;
+using search::fef::FeatureNameBuilder;
+using search::fef::FeatureType;
+
+namespace search::features::rankingexpression {
+
+namespace {
+
+bool is_valid(const FeatureType *type) {
+ if (type == nullptr) {
+ return false;
+ }
+ if (!type->is_object()) {
+ return true;
+ }
+ return !type->type().is_error();
+}
+
+struct IntrinsicBlueprint : IntrinsicExpression {
+ Blueprint::UP blueprint;
+ FeatureType type;
+ IntrinsicBlueprint(Blueprint::UP blueprint_in, const FeatureType &type_in)
+ : blueprint(std::move(blueprint_in)), type(type_in) {}
+ FeatureType result_type() const override { return type; }
+ FeatureExecutor &create_executor(const QueryEnv &queryEnv, vespalib::Stash &stash) const override {
+ return blueprint->createExecutor(queryEnv, stash);
+ }
+};
+
+struct ResultTypeExtractor : Blueprint::DependencyHandler {
+ std::unique_ptr<FeatureType> result_type;
+ bool too_much;
+ ResultTypeExtractor() : result_type(), too_much(false) {}
+ const FeatureType &resolve_input(const vespalib::string &, Blueprint::AcceptInput) override {
+ too_much = true;
+ return FeatureType::number();
+ }
+ void define_output(const vespalib::string &, const FeatureType &type) override {
+ too_much = (too_much || bool(result_type));
+ result_type = std::make_unique<FeatureType>(type);
+ }
+ bool valid() const { return (is_valid(result_type.get()) && !too_much); }
+ const FeatureType &get() const { return *result_type; }
+};
+
+} // namespace search::features::rankingexpression::<unnamed>
+
+IntrinsicExpression::UP
+IntrinsicBlueprintAdapter::try_create(const search::fef::Blueprint &proto,
+ const search::fef::IIndexEnvironment &env,
+ const std::vector<vespalib::string> &params)
+{
+ FeatureNameBuilder name_builder;
+ ResultTypeExtractor result_type;
+ Blueprint::UP blueprint = proto.createInstance();
+ name_builder.baseName(blueprint->getBaseName());
+ for (const auto &param: params) {
+ name_builder.parameter(param);
+ }
+ blueprint->setName(name_builder.buildName());
+ blueprint->attach_dependency_handler(result_type);
+ if (!blueprint->setup(env, params) || !result_type.valid()) {
+ return IntrinsicExpression::UP(nullptr);
+ }
+ blueprint->detach_dependency_handler();
+ return std::make_unique<IntrinsicBlueprint>(std::move(blueprint), result_type.get());
+}
+
+} // namespace search::features::rankingexpression
diff --git a/searchlib/src/vespa/searchlib/features/rankingexpression/intrinsic_blueprint_adapter.h b/searchlib/src/vespa/searchlib/features/rankingexpression/intrinsic_blueprint_adapter.h
new file mode 100644
index 00000000000..01c6ced2391
--- /dev/null
+++ b/searchlib/src/vespa/searchlib/features/rankingexpression/intrinsic_blueprint_adapter.h
@@ -0,0 +1,25 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include "intrinsic_expression.h"
+#include <vespa/vespalib/stllike/string.h>
+#include <vector>
+
+namespace search::fef { class Blueprint; }
+namespace search::fef { class IIndexEnvironment; }
+
+namespace search::features::rankingexpression {
+
+/**
+ * Adapt a Blueprint with no inputs and a single output to the
+ * IntrinsicExpression interface.
+ **/
+struct IntrinsicBlueprintAdapter
+{
+ static IntrinsicExpression::UP try_create(const search::fef::Blueprint &proto,
+ const search::fef::IIndexEnvironment &env,
+ const std::vector<vespalib::string> &params);
+};
+
+} // namespace search::features::rankingexpression
diff --git a/searchlib/src/vespa/searchlib/fef/test/indexenvironmentbuilder.cpp b/searchlib/src/vespa/searchlib/fef/test/indexenvironmentbuilder.cpp
index da27cb143cc..be78e8c45f8 100644
--- a/searchlib/src/vespa/searchlib/fef/test/indexenvironmentbuilder.cpp
+++ b/searchlib/src/vespa/searchlib/fef/test/indexenvironmentbuilder.cpp
@@ -1,5 +1,6 @@
// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/searchcommon/common/datatype.h>
#include "indexenvironmentbuilder.h"
namespace search {
@@ -17,8 +18,18 @@ IndexEnvironmentBuilder::addField(const FieldType &type,
const FieldInfo::CollectionType &coll,
const vespalib::string &name)
{
+ return addField(type, coll, FieldInfo::DataType::DOUBLE, name);
+}
+
+IndexEnvironmentBuilder &
+IndexEnvironmentBuilder::addField(const FieldType &type,
+ const FieldInfo::CollectionType &coll,
+ const FieldInfo::DataType &dataType,
+ const vespalib::string &name)
+{
uint32_t idx = _env.getFields().size();
FieldInfo field(type, coll, name, idx);
+ field.set_data_type(dataType);
_env.getFields().push_back(field);
return *this;
}
diff --git a/searchlib/src/vespa/searchlib/fef/test/indexenvironmentbuilder.h b/searchlib/src/vespa/searchlib/fef/test/indexenvironmentbuilder.h
index fc7af0dc6aa..e6a81a46008 100644
--- a/searchlib/src/vespa/searchlib/fef/test/indexenvironmentbuilder.h
+++ b/searchlib/src/vespa/searchlib/fef/test/indexenvironmentbuilder.h
@@ -30,6 +30,19 @@ public:
const FieldInfo::CollectionType &coll,
const vespalib::string &name);
+ /**
+ * Add a field to the index environment with specified data type.
+ *
+ * @param type The type of field to add.
+ * @param coll Collection type
+ * @param dataType Collection base data type
+ * @param name The name of the field.
+ */
+ IndexEnvironmentBuilder &addField(const FieldType &type,
+ const FieldInfo::CollectionType &coll,
+ const FieldInfo::DataType &dataType,
+ const vespalib::string &name);
+
/** Returns a reference to the index environment of this. */
IndexEnvironment &getIndexEnv() { return _env; }
diff --git a/vagrant/README.md b/vagrant/README.md
new file mode 100644
index 00000000000..7c69f8f22b0
--- /dev/null
+++ b/vagrant/README.md
@@ -0,0 +1,28 @@
+
+# Create C++ dev environment on CentOS using VirtualBox and Vagrant
+
+## Prerequisites
+* [Install VirtualBox](https://www.virtualbox.org/wiki/Downloads)
+* [Install Vagrant](https://www.vagrantup.com/downloads.html)
+
+## Create dev environment
+
+### Change working directory to <vespa-source>/vagrant
+ cd <vespa-source>/vagrant
+
+### Start and provision the environment
+ vagrant up
+
+### Connect to machine via SSH
+SSH agent forwarding is enabled to ensure easy interaction with GitHub inside the machine.
+
+ vagrant ssh
+
+### Checkout vespa source inside machine
+This is needed in order to compile and run tests fast on the local file system inside the machine.
+
+ git clone git@github.com:vespa-engine/vespa.git
+
+
+## Build C++ modules
+Please follow the instructions described [here](../README.md).
diff --git a/vagrant/Vagrantfile b/vagrant/Vagrantfile
new file mode 100644
index 00000000000..a4cea6aa10f
--- /dev/null
+++ b/vagrant/Vagrantfile
@@ -0,0 +1,38 @@
+# -*- mode: ruby -*-
+# vi: set ft=ruby :
+
+# For a complete reference, please see the online documentation at https://docs.vagrantup.com.
+Vagrant.configure("2") do |config|
+
+ config.vm.box = "boxcutter/centos73-desktop"
+
+ config.ssh.forward_agent = true
+
+ config.vm.synced_folder "../dist", "/vagrant/dist"
+
+ config.vm.provider "virtualbox" do |vb|
+ # Display the VirtualBox GUI when booting the machine
+ vb.gui = true
+
+ vb.memory = "8192"
+ vb.cpus = 8
+ end
+
+ # Install required and nice-to-have packages
+ config.vm.provision "shell", inline: <<-SHELL
+ yum-config-manager --add-repo https://copr.fedorainfracloud.org/coprs/g/vespa/vespa/repo/epel-7/group_vespa-vespa-epel-7.repo
+ yum -y install epel-release
+ yum -y install centos-release-scl
+ yum -y install yum-utils
+ yum -y install git \
+ ccache \
+ rpm-build \
+ valgrind \
+ sudo \
+ firefox \
+ vim
+ yum-builddep -y /vagrant/dist/vespa.spec
+ echo -e "* soft nproc 409600\n* hard nproc 409600" > /etc/security/limits.d/99-nproc.conf
+ echo -e "* soft nofile 262144\n* hard nofile 262144" > /etc/security/limits.d/99-nofile.conf
+ SHELL
+end
diff --git a/vespa-hadoop/pom.xml b/vespa-hadoop/pom.xml
index 4aa7d12e006..ea461e9783f 100644
--- a/vespa-hadoop/pom.xml
+++ b/vespa-hadoop/pom.xml
@@ -78,7 +78,6 @@
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
- <version>4.12</version>
<scope>test</scope>
</dependency>