diff options
| -rw-r--r-- | screwdriver.yaml | 77 | ||||
| -rwxr-xr-x | travis/travis-build.sh | 23 |
2 files changed, 90 insertions, 10 deletions
diff --git a/screwdriver.yaml b/screwdriver.yaml new file mode 100644 index 00000000000..6b3795e6da6 --- /dev/null +++ b/screwdriver.yaml @@ -0,0 +1,77 @@ +# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +--- +cache: + job: + main: [/main_job_cache] + +jobs: + main: + requires: [~pr, ~commit] + image: vespaengine/vespa-build-centos7 + annotations: + screwdriver.cd/cpu: 8 + screwdriver.cd/ram: 16 + screwdriver.cd/disk: HIGH + screwdriver.cd/timeout: 600 + + environment: + LOCAL_MVN_REPO: "/tmp/vespa/mvnrepo" + VESPA_MAVEN_EXTRA_OPTS: "-Dmaven.repo.local=/tmp/vespa/mvnrepo -Dmaven.javadoc.skip=true -Dmaven.source.skip=true" + CCACHE_TMP_DIR: "/tmp/ccache_tmp" + CCACHE_DATA_DIR: "/tmp/vespa/ccache" + MAIN_CACHE_FILE: "/main_job_cache/vespa.tar" + + steps: + - inspect: | + set -x + env | grep -v TOKEN + cat /proc/cpuinfo + cat /proc/meminfo + df -h + uname -a + - restore-cache: | + (cd /tmp && if [[ -f $MAIN_CACHE_FILE ]]; then tar xf $MAIN_CACHE_FILE; fi) + + mkdir -p $CCACHE_DATA_DIR + mkdir -p $CCACHE_TMP_DIR + + export CCACHE_DIR=$CCACHE_DATA_DIR + + rm -f $CCACHE_DIR/ccache.conf + ccache -M 20G + ccache -o log_file=$SD_ARTIFACTS_DIR/ccache_log.txt + ccache -o temporary_dir=$CCACHE_TMP_DIR + ccache -p + ccache -z + + - compile: | + export TRAVIS_REPO_SLUG="vespa-engine/vespa" + if [[ -z $SD_PULL_REQUEST ]]; then + export TRAVIS_PULL_REQUEST=false + else + export TRAVIS_PULL_REQUEST=$SD_PULL_REQUEST + fi + + travis/travis-build.sh + + - save-cache: | + if [[ -z "$SD_PULL_REQUEST" ]]; then + # Remove what we have produced + rm -rf $LOCAL_MVN_REPO/com/yahoo + rm -rf $LOCAL_MVN_REPO/ai/vespa + + # Tar toghether the /tmp/vespa folder containing ccache and cleaned mvn repo + mkdir -p $(dirname $MAIN_CACHE_FILE) + (cd /tmp && tar cf $MAIN_CACHE_FILE vespa) + + # Wipe the cache if we exceed 2GB to avoid pulling and pusing too large files + if (( $(stat --format='%s' $MAIN_CACHE_FILE) > $(( 2*1000*1000*1000 )) )); then + tar cf $MAIN_CACHE_FILE --files-from=/dev/null; + echo "Cleaning cache file. $MAIN_CACHE_FILE is now $(stat --format='%s' $MAIN_CACHE_FILE) bytes." + fi + fi + + - inspect-after: | + du -sh /tmp/vespa/* + ls -la /main_job_cache || true + df -h diff --git a/travis/travis-build.sh b/travis/travis-build.sh index a638771e950..47d1496b0cf 100755 --- a/travis/travis-build.sh +++ b/travis/travis-build.sh @@ -3,18 +3,21 @@ set -e -export SOURCE_DIR=/source -export NUM_THREADS=6 -export MALLOC_ARENA_MAX=1 -export MAVEN_OPTS="-Xss1m -Xms128m -Xmx2g" +readonly SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd )" +readonly NUM_THREADS=$(( $(nproc) + 2 )) + source /etc/profile.d/enable-devtoolset-9.sh source /etc/profile.d/enable-rh-maven35.sh +export MALLOC_ARENA_MAX=1 +export MAVEN_OPTS="-Xss1m -Xms128m -Xmx2g" +export VESPA_MAVEN_EXTRA_OPTS="${VESPA_MAVEN_EXTRA_OPTS:+${VESPA_MAVEN_EXTRA_OPTS} }--no-snapshot-updates --batch-mode --threads ${NUM_THREADS}" + ccache --max-size=1600M ccache --set-config=compression=true ccache -p -if ! source /source/travis/detect-what-to-build.sh; then +if ! source $SOURCE_DIR/travis/detect-what-to-build.sh; then echo "Could not detect what to build." SHOULD_BUILD=all fi @@ -25,19 +28,19 @@ cd ${SOURCE_DIR} case $SHOULD_BUILD in cpp) - env VESPA_MAVEN_EXTRA_OPTS="--no-snapshot-updates --batch-mode --threads ${NUM_THREADS}" sh ./bootstrap.sh full + ./bootstrap.sh full cmake3 -DVESPA_UNPRIVILEGED=no . make -j ${NUM_THREADS} ctest3 --output-on-failure -j ${NUM_THREADS} ccache --show-stats ;; java) - env VESPA_MAVEN_EXTRA_OPTS="--no-snapshot-updates --batch-mode --threads ${NUM_THREADS}" sh ./bootstrap.sh java - mvn -V install --no-snapshot-updates --batch-mode --threads ${NUM_THREADS} + ./bootstrap.sh java + mvn -V $VESPA_MAVEN_EXTRA_OPTS install ;; *) - env VESPA_MAVEN_EXTRA_OPTS="--no-snapshot-updates --batch-mode --threads ${NUM_THREADS}" sh ./bootstrap.sh java - mvn -V install --no-snapshot-updates --batch-mode --threads ${NUM_THREADS} + ./bootstrap.sh java + mvn -V $VESPA_MAVEN_EXTRA_OPTS install cmake3 -DVESPA_UNPRIVILEGED=no . make -j ${NUM_THREADS} ctest3 --output-on-failure -j ${NUM_THREADS} |