Merge pull request #3020 from yahoo/bjorncs/docker-dev

Bjorncs/docker dev

3 files changed, 83 insertions, 65 deletions

diff --git a/docker/ci/build-and-test.sh b/docker/ci/build-and-test.sh
new file mode 100644
index 00000000000..030d662bf0f
--- /dev/null
+++ b/docker/ci/build-and-test.sh
@@ -0,0 +1,70 @@
+#!/bin/bash
+# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+set -e
+set -m
+
+if [ $# -ne 4 ]; then
+  echo "Usage: $0 <source dir> <build dir> <log dir> <num threads>"
+  echo "This script should not be called manually."
+  exit 1
+fi
+
+SOURCE_DIR=$1
+BUILD_DIR=$2
+LOG_DIR=$3
+NUM_THREADS=$4
+
+function bootstrap {
+    source /opt/rh/devtoolset-6/enable || true
+    export MAVEN_OPTS="-Xms128m -Xmx512m"
+    sh ./bootstrap.sh full
+    cmake3 -DCMAKE_INSTALL_PREFIX=/opt/vespa \
+        -DJAVA_HOME=/usr/lib/jvm/java-openjdk \
+        -DEXTRA_LINK_DIRECTORY="/opt/vespa-boost/lib;/opt/vespa-libtorrent/lib;/opt/vespa-zookeeper-c-client/lib;/opt/vespa-cppunit/lib;/usr/lib64/llvm3.9/lib" \
+        -DEXTRA_INCLUDE_DIRECTORY="/opt/vespa-boost/include;/opt/vespa-libtorrent/include;/opt/vespa-zookeeper-c-client/include;/opt/vespa-cppunit/include;/usr/include/llvm3.9" \
+        -DCMAKE_INSTALL_RPATH="/opt/vespa/lib64;/opt/vespa-boost/lib;/opt/vespa-libtorrent/lib;/opt/vespa-zookeeper-c-client/lib;/opt/vespa-cppunit/lib;/usr/lib/jvm/java-1.8.0/jre/lib/amd64/server;/usr/include/llvm3.9" \
+        -DCMAKE_BUILD_RPATH=/opt/vespa/lib64 \
+        -DVALGRIND_UNIT_TESTS=no \
+        "${SOURCE_DIR}"
+}
+
+function build_java {
+    cd "${SOURCE_DIR}"
+    mvn install -nsu -B -T ${NUM_THREADS} -V # Should ideally split out test phase, but some unit tests fails on 'mvn test'
+}
+
+function build_cpp {
+    cd "${BUILD_DIR}"
+    make -j ${NUM_THREADS}
+    ctest3 -j ${NUM_THREADS}
+}
+
+bootstrap
+
+pids=()
+set -o pipefail
+
+# Java build and test
+# Should be waited for first, because it takes much shorter time than C++ build.
+build_java 2>&1 | tee ${LOG_DIR}/java.log &
+pids+=($!)
+
+# C++ build and test
+build_cpp 2>&1 | tee ${LOG_DIR}/cpp.log &
+pids+=($!)
+
+EXIT_CODE=0
+for pid in "${pids[@]}"; do
+    wait $pid || EXIT_CODE=$?
+    echo "$pid finished with status ${EXIT_CODE}"
+    if [[ "${EXIT_CODE}" != 0 ]]; then
+        echo "Exiting and killing remaining jobs."
+        break;
+    fi
+done
+set +o pipefail
+
+# Kill any remaining jobs, ignoring error when no jobs are running
+kill $(jobs -p) 2>/dev/null || true
+
+exit ${EXIT_CODE}
diff --git a/docker/ci/vespa-ci-internal.sh b/docker/ci/vespa-ci-internal.sh
index bad2b37ec85..8ba77a56a69 100755
--- a/docker/ci/vespa-ci-internal.sh
+++ b/docker/ci/vespa-ci-internal.sh
@@ -1,82 +1,30 @@
 #!/bin/bash
 # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 set -e
-set -m
-set -x
 
-if [ $# -ne 1 ]; then
-  echo "Usage: $0 <git commit>"
+if [ $# -ne 2 ]; then
+  echo "Usage: $0 <git commit> <log prefix>"
   echo "This script should not be called manually."
   exit 1
 fi
 
 GIT_COMMIT=$1
-MAPPED_DIR=/vespa
+LOG_PREFIX=$2
 SOURCE_DIR=~/vespa
 BUILD_DIR=~/build
+MAPPED_DIR=/vespa
 LOG_DIR=~/log
 
-function build_java {
-    cd "${SOURCE_DIR}"
-    mvn install -nsu -B -T 2.0C -V -DskipTests=true -Dmaven.javadoc.skip=true
-    mvn install -nsu -B -T 2.0C -V
-}
-
-function build_cpp {
-    cd "${BUILD_DIR}"
-    cmake3 -DCMAKE_INSTALL_PREFIX=/opt/vespa \
-        -DJAVA_HOME=/usr/lib/jvm/java-openjdk \
-        -DEXTRA_LINK_DIRECTORY="/opt/vespa-boost/lib;/opt/vespa-libtorrent/lib;/opt/vespa-zookeeper-c-client/lib;/opt/vespa-cppunit/lib;/usr/lib64/llvm3.9/lib" \
-        -DEXTRA_INCLUDE_DIRECTORY="/opt/vespa-boost/include;/opt/vespa-libtorrent/include;/opt/vespa-zookeeper-c-client/include;/opt/vespa-cppunit/include;/usr/include/llvm3.9" \
-        -DCMAKE_INSTALL_RPATH="/opt/vespa/lib64;/opt/vespa-boost/lib;/opt/vespa-libtorrent/lib;/opt/vespa-zookeeper-c-client/lib;/opt/vespa-cppunit/lib;/usr/lib/jvm/java-1.8.0/jre/lib/amd64/server;/usr/include/llvm3.9" \
-        -DCMAKE_BUILD_RPATH=/opt/vespa/lib64 \
-        -DVALGRIND_UNIT_TESTS=no \
-        "${SOURCE_DIR}"
-    NUM_THREADS=$(($(nproc --all) * 2))
-    make -j ${NUM_THREADS}
-    ctest3 -j ${NUM_THREADS}
-}
-
 mkdir "${SOURCE_DIR}"
 mkdir "${BUILD_DIR}"
 mkdir "${LOG_DIR}"
 
-git clone --no-checkout --no-hardlinks file://${MAPPED_DIR} "${SOURCE_DIR}"
+git clone --no-checkout --no-hardlinks "file://${MAPPED_DIR}" "${SOURCE_DIR}"
 cd "${SOURCE_DIR}"
 git -c advice.detachedHead=false checkout ${GIT_COMMIT}
-source /opt/rh/devtoolset-6/enable || true
-
-export MAVEN_OPTS="-Xms128m -Xmx512m"
-sh ./bootstrap.sh full
-
-pids=()
-set -o pipefail
-TIMESTAMP=$(date +%Y-%m-%dT%H:%M:%S)
-
-# Java build and test
-# Should be waited for first, because it takes much shorter time than C++ build.
-build_java 2>&1 | tee ${LOG_DIR}/java.log &
-pids+=($!)
-
-# C++ build and test
-build_cpp 2>&1 | tee ${LOG_DIR}/cpp.log &
-pids+=($!)
-
-EXIT_CODE=0
-for pid in "${pids[@]}"; do
-    wait $pid || EXIT_CODE=$?
-    echo "$pid finished with status ${EXIT_CODE}"
-    if [[ "${EXIT_CODE}" != 0 ]]; then
-        echo "Exiting and killing remaining jobs."
-        break;
-    fi
-done
-set +o pipefail
-
-# Kill any remaining jobs, ignoring error when no jobs are running
-kill $(jobs -p) 2>/dev/null || true
 
-sudo cp ${LOG_DIR}/java.log ${MAPPED_DIR}/docker/logs/vespa-ci-java-${TIMESTAMP}.log
-sudo cp ${LOG_DIR}/cpp.log ${MAPPED_DIR}/docker/logs/vespa-ci-cpp-${TIMESTAMP}.log
+NUM_THREADS=$(($(nproc --all) * 2))
+bash "${MAPPED_DIR}/docker/ci/build-and-test.sh" "${SOURCE_DIR}" "${BUILD_DIR}" "${LOG_DIR}" ${NUM_THREADS}
 
-exit ${EXIT_CODE}
+sudo cp "${LOG_DIR}/java.log" "${MAPPED_DIR}/docker/logs/${LOG_PREFIX}-java.log"
+sudo cp "${LOG_DIR}/cpp.log" "${MAPPED_DIR}/docker/logs/${LOG_PREFIX}-cpp.log"
diff --git a/docker/vespa-ci.sh b/docker/vespa-ci.sh
index c9fefe96a21..f4c46f8919a 100755
--- a/docker/vespa-ci.sh
+++ b/docker/vespa-ci.sh
@@ -1,7 +1,6 @@
 #!/bin/bash
 # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 set -e
-set -x
 
 if [ $# -ne 1 ]; then
   echo "Usage: $0 <git commit>"
@@ -10,16 +9,17 @@ fi
 
 RELATIVE_DIR=$(dirname "${BASH_SOURCE[0]}")
 DIR=$(cd "${RELATIVE_DIR}" && pwd)
-cd $DIR
+cd "${DIR}"
 
 GIT_COMMIT=$1
 DOCKER_IMAGE="vespaengine/vespa-dev:latest"
 INTERNAL_DIR=/vespa
+LOG_PREFIX=vespa-ci-$(date +%Y-%m-%dT%H:%M:%S)
 
 mkdir -p logs
 
-docker run --rm -v ${DIR}/..:${INTERNAL_DIR} --entrypoint ${INTERNAL_DIR}/docker/ci/vespa-ci-internal.sh "$DOCKER_IMAGE" "$GIT_COMMIT" \
-   2>&1 | tee logs/vespa-ci-$(date +%Y-%m-%dT%H:%M:%S).log
+docker run --rm -v ${DIR}/..:${INTERNAL_DIR} --entrypoint "${INTERNAL_DIR}/docker/ci/vespa-ci-internal.sh" "$DOCKER_IMAGE" \
+   "$GIT_COMMIT" "$LOG_PREFIX" 2>&1 | tee logs/${LOG_PREFIX}.log
 
 # Needed because of piping docker run to tee above
 exit ${PIPESTATUS[0]}