diff options
-rw-r--r-- | docker/ci/build-and-test.sh | 70 | ||||
-rwxr-xr-x | docker/ci/vespa-ci-internal.sh | 57 |
2 files changed, 73 insertions, 54 deletions
diff --git a/docker/ci/build-and-test.sh b/docker/ci/build-and-test.sh new file mode 100644 index 00000000000..030d662bf0f --- /dev/null +++ b/docker/ci/build-and-test.sh @@ -0,0 +1,70 @@ +#!/bin/bash +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +set -e +set -m + +if [ $# -ne 4 ]; then + echo "Usage: $0 <source dir> <build dir> <log dir> <num threads>" + echo "This script should not be called manually." + exit 1 +fi + +SOURCE_DIR=$1 +BUILD_DIR=$2 +LOG_DIR=$3 +NUM_THREADS=$4 + +function bootstrap { + source /opt/rh/devtoolset-6/enable || true + export MAVEN_OPTS="-Xms128m -Xmx512m" + sh ./bootstrap.sh full + cmake3 -DCMAKE_INSTALL_PREFIX=/opt/vespa \ + -DJAVA_HOME=/usr/lib/jvm/java-openjdk \ + -DEXTRA_LINK_DIRECTORY="/opt/vespa-boost/lib;/opt/vespa-libtorrent/lib;/opt/vespa-zookeeper-c-client/lib;/opt/vespa-cppunit/lib;/usr/lib64/llvm3.9/lib" \ + -DEXTRA_INCLUDE_DIRECTORY="/opt/vespa-boost/include;/opt/vespa-libtorrent/include;/opt/vespa-zookeeper-c-client/include;/opt/vespa-cppunit/include;/usr/include/llvm3.9" \ + -DCMAKE_INSTALL_RPATH="/opt/vespa/lib64;/opt/vespa-boost/lib;/opt/vespa-libtorrent/lib;/opt/vespa-zookeeper-c-client/lib;/opt/vespa-cppunit/lib;/usr/lib/jvm/java-1.8.0/jre/lib/amd64/server;/usr/include/llvm3.9" \ + -DCMAKE_BUILD_RPATH=/opt/vespa/lib64 \ + -DVALGRIND_UNIT_TESTS=no \ + "${SOURCE_DIR}" +} + +function build_java { + cd "${SOURCE_DIR}" + mvn install -nsu -B -T ${NUM_THREADS} -V # Should ideally split out test phase, but some unit tests fails on 'mvn test' +} + +function build_cpp { + cd "${BUILD_DIR}" + make -j ${NUM_THREADS} + ctest3 -j ${NUM_THREADS} +} + +bootstrap + +pids=() +set -o pipefail + +# Java build and test +# Should be waited for first, because it takes much shorter time than C++ build. +build_java 2>&1 | tee ${LOG_DIR}/java.log & +pids+=($!) + +# C++ build and test +build_cpp 2>&1 | tee ${LOG_DIR}/cpp.log & +pids+=($!) + +EXIT_CODE=0 +for pid in "${pids[@]}"; do + wait $pid || EXIT_CODE=$? + echo "$pid finished with status ${EXIT_CODE}" + if [[ "${EXIT_CODE}" != 0 ]]; then + echo "Exiting and killing remaining jobs." + break; + fi +done +set +o pipefail + +# Kill any remaining jobs, ignoring error when no jobs are running +kill $(jobs -p) 2>/dev/null || true + +exit ${EXIT_CODE} diff --git a/docker/ci/vespa-ci-internal.sh b/docker/ci/vespa-ci-internal.sh index 2398b3fec24..e15dcec8555 100755 --- a/docker/ci/vespa-ci-internal.sh +++ b/docker/ci/vespa-ci-internal.sh @@ -1,8 +1,6 @@ #!/bin/bash # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. set -e -set -m -set -x if [ $# -ne 1 ]; then echo "Usage: $0 <git commit>" @@ -11,31 +9,11 @@ if [ $# -ne 1 ]; then fi GIT_COMMIT=$1 -MAPPED_DIR=/vespa SOURCE_DIR=~/vespa BUILD_DIR=~/build +MAPPED_DIR=/vespa LOG_DIR=~/log -function build_java { - cd "${SOURCE_DIR}" - mvn install -nsu -B -T 2.0C -V # Should ideally split out test phase, but some unit tests fails on 'mvn test' -} - -function build_cpp { - cd "${BUILD_DIR}" - cmake3 -DCMAKE_INSTALL_PREFIX=/opt/vespa \ - -DJAVA_HOME=/usr/lib/jvm/java-openjdk \ - -DEXTRA_LINK_DIRECTORY="/opt/vespa-boost/lib;/opt/vespa-libtorrent/lib;/opt/vespa-zookeeper-c-client/lib;/opt/vespa-cppunit/lib;/usr/lib64/llvm3.9/lib" \ - -DEXTRA_INCLUDE_DIRECTORY="/opt/vespa-boost/include;/opt/vespa-libtorrent/include;/opt/vespa-zookeeper-c-client/include;/opt/vespa-cppunit/include;/usr/include/llvm3.9" \ - -DCMAKE_INSTALL_RPATH="/opt/vespa/lib64;/opt/vespa-boost/lib;/opt/vespa-libtorrent/lib;/opt/vespa-zookeeper-c-client/lib;/opt/vespa-cppunit/lib;/usr/lib/jvm/java-1.8.0/jre/lib/amd64/server;/usr/include/llvm3.9" \ - -DCMAKE_BUILD_RPATH=/opt/vespa/lib64 \ - -DVALGRIND_UNIT_TESTS=no \ - "${SOURCE_DIR}" - NUM_THREADS=$(($(nproc --all) * 2)) - make -j ${NUM_THREADS} - ctest3 -j ${NUM_THREADS} -} - mkdir "${SOURCE_DIR}" mkdir "${BUILD_DIR}" mkdir "${LOG_DIR}" @@ -43,39 +21,10 @@ mkdir "${LOG_DIR}" git clone --no-checkout --no-hardlinks file://${MAPPED_DIR} "${SOURCE_DIR}" cd "${SOURCE_DIR}" git -c advice.detachedHead=false checkout ${GIT_COMMIT} -source /opt/rh/devtoolset-6/enable || true -export MAVEN_OPTS="-Xms128m -Xmx512m" -sh ./bootstrap.sh full +NUM_THREADS=$(($(nproc --all) * 2)) +bash ${MAPPED_DIR}/docker/ci/build-and-test.sh ${SOURCE_DIR} ${BUILD_DIR} ${LOG_DIR} ${NUM_THREADS} -pids=() -set -o pipefail TIMESTAMP=$(date +%Y-%m-%dT%H:%M:%S) - -# Java build and test -# Should be waited for first, because it takes much shorter time than C++ build. -build_java 2>&1 | tee ${LOG_DIR}/java.log & -pids+=($!) - -# C++ build and test -build_cpp 2>&1 | tee ${LOG_DIR}/cpp.log & -pids+=($!) - -EXIT_CODE=0 -for pid in "${pids[@]}"; do - wait $pid || EXIT_CODE=$? - echo "$pid finished with status ${EXIT_CODE}" - if [[ "${EXIT_CODE}" != 0 ]]; then - echo "Exiting and killing remaining jobs." - break; - fi -done -set +o pipefail - -# Kill any remaining jobs, ignoring error when no jobs are running -kill $(jobs -p) 2>/dev/null || true - sudo cp ${LOG_DIR}/java.log ${MAPPED_DIR}/docker/logs/vespa-ci-java-${TIMESTAMP}.log sudo cp ${LOG_DIR}/cpp.log ${MAPPED_DIR}/docker/logs/vespa-ci-cpp-${TIMESTAMP}.log - -exit ${EXIT_CODE} |