# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
---
cache:
job:
build-vespa: [/main_job_cache]
component-build: [/main_job_cache]
shared:
image: vespaengine/vespa-build-centos-stream8:latest
environment:
USER_SHELL_BIN: bash
annotations:
restore-cache: &restore-cache
restore-cache: |
(cd /tmp && if [[ -f $MAIN_CACHE_FILE ]]; then tar xf $MAIN_CACHE_FILE; fi)
mkdir -p $CCACHE_DATA_DIR
mkdir -p $CCACHE_TMP_DIR
export CCACHE_DIR=$CCACHE_DATA_DIR
rm -f $CCACHE_DIR/ccache.conf
ccache -M 20G
ccache -o log_file=$SD_ARTIFACTS_DIR/ccache_log.txt
ccache -o temporary_dir=$CCACHE_TMP_DIR
ccache -p
ccache -z
du -sh /tmp/*
save-cache: &save-cache
save-cache: |
ls -lart /tmp/vespa
du -sh /tmp/vespa/*
if [[ -z "$SD_PULL_REQUEST" ]]; then
# Remove what we have produced
rm -rf $LOCAL_MVN_REPO/com/yahoo
rm -rf $LOCAL_MVN_REPO/ai/vespa
# Tar toghether the /tmp/vespa folder containing ccache and cleaned mvn repo
mkdir -p $(dirname $MAIN_CACHE_FILE)
(cd /tmp && tar cf $MAIN_CACHE_FILE vespa)
# Wipe the cache if we exceed 2GB to avoid pulling and pusing too large files
if (( $(stat --format='%s' $MAIN_CACHE_FILE) > $(( 2200*1000*1000 )) )); then
tar cf $MAIN_CACHE_FILE --files-from=/dev/null;
echo "Cleaning cache file. $MAIN_CACHE_FILE is now $(stat --format='%s' $MAIN_CACHE_FILE) bytes."
fi
fi
jobs:
build-vespa:
requires: [~pr]
annotations:
screwdriver.cd/cpu: 7
screwdriver.cd/ram: 16
screwdriver.cd/disk: HIGH
screwdriver.cd/timeout: 60
screwdriver.cd/buildPeriodically: H 4 * * *
environment:
LOCAL_MVN_REPO: "/tmp/vespa/mvnrepo"
VESPA_MAVEN_EXTRA_OPTS: "-Dmaven.repo.local=/tmp/vespa/mvnrepo -Dmaven.source.skip=true"
CCACHE_TMP_DIR: "/tmp/ccache_tmp"
CCACHE_DATA_DIR: "/tmp/vespa/ccache"
MAIN_CACHE_FILE: "/main_job_cache/vespa.tar"
steps:
- inspect: |
set -x
env | grep -v TOKEN
cat /proc/cpuinfo
cat /proc/meminfo
df -h
uname -a
rpm -qa | grep "vespa"
- *restore-cache
- compile: |
screwdriver/build-vespa.sh
- *save-cache
- inspect-after: |
du -sh /tmp/vespa/*
ls -la /main_job_cache || true
df -h
component-build:
annotations:
screwdriver.cd/cpu: 7
screwdriver.cd/ram: 16
screwdriver.cd/disk: HIGH
screwdriver.cd/timeout: 90
screwdriver.cd/dockerEnabled: true
screwdriver.cd/dockerCpu: TURBO
screwdriver.cd/dockerRam: HIGH
environment:
LOCAL_MVN_REPO: "/tmp/vespa/mvnrepo"
VESPA_MAVEN_EXTRA_OPTS: "--show-version --batch-mode --no-snapshot-updates -Dmaven.repo.local=/tmp/vespa/mvnrepo"
CCACHE_TMP_DIR: "/tmp/ccache_tmp"
CCACHE_DATA_DIR: "/tmp/vespa/ccache"
MAIN_CACHE_FILE: "/main_job_cache/vespa.tar"
CCACHE_COMPRESS: "1"
secrets:
- DOCKER_HUB_DEPLOY_KEY
steps:
- get-vespa-version: |
VESPA_VERSION=$(meta get vespa.version)
VESPA_REF=$(meta get vespa.ref)
SYSTEM_TEST_REF=$(meta get system-test.ref)
if [[ $VESPA_VERSION == null ]] || [[ $VESPA_REF == null ]] || [[ $SYSTEM_TEST_REF == null ]]; then
echo "Must have valid Vespa version and reference to continue \
(got VESPA_VERSION=$VESPA_VERSION, VESPA_REF=$VESPA_REF, SYSTEM_TEST_REF=$SYSTEM_TEST_REF)."
exit 1
fi
- install-dependencies: |
dnf config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo
dnf -y install docker-ce docker-ce-cli containerd.io
docker system info
- checkout: |
mkdir -p workdir
cd workdir
WORKDIR=$(pwd)
git clone -q https://github.com/vespa-engine/vespa
(cd vespa && git checkout $VESPA_REF && git tag v$VESPA_VERSION)
git clone -q https://github.com/vespa-engine/system-test
(cd system-test && git checkout $SYSTEM_TEST_REF)
git clone -q https://github.com/vespa-engine/sample-apps
# Set correct version in pom.xml files
(cd vespa && screwdriver/replace-vespa-version-in-poms.sh $VESPA_VERSION $(pwd) )
(cd sample-apps && find . -name "pom.xml" -exec sed -i -e "s,.*,$VESPA_VERSION," {} \;)
(cd sample-apps && find . -name "pom.xml" -exec sed -i -e "s:[[]8,9[)]:$VESPA_VERSION:" {} \;)
- make-srpm: |
make -C $WORKDIR/vespa -f .copr/Makefile srpm outdir=$WORKDIR
- *restore-cache
- build: |
cd $WORKDIR/vespa
export FACTORY_VESPA_VERSION=$VESPA_VERSION
NUM_THREADS=$(( $(nproc) + 2 ))
VESPA_CMAKE_SANITIZERS_OPTION=""
time make -C client/go BIN=$WORKDIR/vespa-install/opt/vespa/bin SHARE=$WORKDIR/vespa-install/usr/share install-all
time ./bootstrap.sh java
time mvn -T $NUM_THREADS $VESPA_MAVEN_EXTRA_OPTS install
cmake3 -DVESPA_UNPRIVILEGED=no $VESPA_CMAKE_SANITIZERS_OPTION .
time make -j ${NUM_THREADS}
time ctest3 --output-on-failure -j ${NUM_THREADS}
ccache --show-stats
time make -j ${NUM_THREADS} install DESTDIR=$WORKDIR/vespa-install
- build-sample-apps: |
(cd $WORKDIR/sample-apps && time mvn -T $NUM_THREADS $VESPA_MAVEN_EXTRA_OPTS package)
- verify-rpm-build: |
cd $WORKDIR
ulimit -c 0
time rpmbuild --rebuild --define="_topdir $WORKDIR/vespa-rpmbuild" \
--define "_debugsource_template %{nil}" \
--define "installdir $WORKDIR/vespa-install" *.src.rpm
mkdir -p $WORKDIR/system-test/docker/rpms
mv $WORKDIR/vespa-rpmbuild/RPMS/x86_64/*.rpm $WORKDIR/system-test/docker/rpms
if test -z "$VESPA_CMAKE_SANITIZERS_OPTION"
then
rm -f $WORKDIR/system-test/docker/rpms/*debuginfo*rpm
fi
ls -la $WORKDIR/system-test/docker/rpms
- build-test-image: |
cd $WORKDIR/system-test
mkdir -p docker/vespa-systemtests
git archive HEAD --format tar | tar x -C docker/vespa-systemtests
cp -a $LOCAL_MVN_REPO docker/repository
cd docker
docker build --file Dockerfile.systemtest \
--build-arg VESPA_BASE_IMAGE=vespaengine/vespa-systemtest-base-centos-stream8:latest \
--build-arg SYSTEMTEST_BASE_IMAGE=vespa --build-arg SKIP_M2_POPULATE=false \
--target systemtest \
--tag vespaengine/vespa-systemtest-centos-stream8:$VESPA_VERSION \
--tag vespaengine/vespa-systemtest-centos-stream8:latest .
- verify-test-image: |
docker run --rm -ti --entrypoint bash vespaengine/vespa-systemtest-centos-stream8:$VESPA_VERSION -lc \
"/opt/vespa-systemtests/lib/node_server.rb & sleep 3 && ruby /opt/vespa-systemtests/tests/search/basicsearch/basic_search.rb --run test_basicsearch__ELASTIC"
- publish-test-image: |
if [[ -z "$SD_PULL_REQUEST" ]]; then
OPT_STATE="$(set +o)"
set +x
docker login --username aressem --password "$DOCKER_HUB_DEPLOY_KEY"
eval "$OPT_STATE"
docker push docker.io/vespaengine/vespa-systemtest-centos-stream8:$VESPA_VERSION
docker push docker.io/vespaengine/vespa-systemtest-centos-stream8:latest
fi
- *save-cache
publish-release:
image: docker.io/vespaengine/vespa-build-centos-stream8:latest
annotations:
screwdriver.cd/cpu: 7
screwdriver.cd/ram: 16
screwdriver.cd/disk: HIGH
screwdriver.cd/timeout: 300
screwdriver.cd/dockerEnabled: true
screwdriver.cd/dockerCpu: TURBO
screwdriver.cd/dockerRam: HIGH
secrets:
- COPR_WEBHOOK
- OSSRH_USER
- OSSRH_TOKEN
- GPG_KEYNAME
- GPG_PASSPHRASE
- GPG_ENCPHRASE
- SAMPLE_APPS_DEPLOY_KEY
- VESPA_DEPLOY_KEY
- DOCKER_IMAGE_DEPLOY_KEY
- DOCKER_HUB_DEPLOY_KEY
- GHCR_DEPLOY_KEY
environment:
GIT_SSH_COMMAND: "ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no"
steps:
- get-vespa-version: |
VESPA_VERSION=$(meta get vespa.version)
VESPA_REF=$(meta get vespa.ref)
if [[ $VESPA_VERSION == null ]] || [[ $VESPA_REF == null ]]; then
echo "Must have valid Vespa version and reference to continue (got VESPA_VERSION=$VESPA_VERSION, VESPA_REF=$VESPA_REF)."
return 1
fi
meta set vespa.version $VESPA_VERSION
- install-dependencies: |
dnf config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo
dnf install -y docker-ce docker-ce-cli containerd.io
docker system info
- release-java-artifacts: |
screwdriver/release-java-artifacts.sh $VESPA_VERSION $VESPA_REF
- release-rpms: |
screwdriver/release-rpms.sh $VESPA_VERSION $VESPA_REF
- release-container-image: |
screwdriver/release-container-image.sh $VESPA_VERSION
- update-sample-apps: |
screwdriver/update-vespa-version-in-sample-apps.sh $VESPA_VERSION
publish-legacy-release:
image: docker.io/vespaengine/vespa-build-centos-stream8:latest
annotations:
screwdriver.cd/cpu: 7
screwdriver.cd/ram: 16
screwdriver.cd/disk: HIGH
screwdriver.cd/timeout: 300
screwdriver.cd/buildPeriodically: H 6 1 * *
environment:
IMAGE_NAME: "vespaengine/vespa-generic-intel-x86_64"
secrets:
- DOCKER_HUB_DEPLOY_KEY
steps:
- get-vespa-version: |
set -x
VESPA_VERSION=$(meta get vespa.version --external publish-release)
if [[ $VESPA_VERSION == null ]] || [[ $VESPA_REF == null ]]; then
echo "Must have valid Vespa version to continue (got VESPA_VERSION=$VESPA_VERSION)."
return 1
fi
- install-dependencies: |
dnf install -y podman podman-docker buildah skopeo
touch /etc/containers/nodocker
- checkout: |
mkdir -p workdir
cd workdir
export WORKDIR=$(pwd)
git clone -q https://github.com/vespa-engine/vespa
(cd vespa && git checkout v$VESPA_VERSION)
git clone -q https://github.com/vespa-engine/system-test
# Set correct version in pom.xml files
(cd vespa && screwdriver/replace-vespa-version-in-poms.sh $VESPA_VERSION $(pwd) )
- build-rpms: |
cd $WORKDIR
make -C $WORKDIR/vespa -f .copr/Makefile srpm outdir=$WORKDIR
rpmbuild --rebuild \
--define="_topdir $WORKDIR/vespa-rpmbuild" \
--define "debug_package %{nil}" \
--define "_debugsource_template %{nil}" \
--define '_cmake_extra_opts "-DDEFAULT_VESPA_CPU_ARCH_FLAGS=-msse3 -mcx16 -mtune=intel"' \
*.src.rpm
rm -f *.src.rpm
mv $WORKDIR/vespa-rpmbuild/RPMS/x86_64/*.rpm .
- build-container-image: |
cat < Dockerfile
ARG VESPA_VERSION
FROM docker.io/vespaengine/vespa:\$VESPA_VERSION
USER root
RUN --mount=type=bind,target=/rpms/,source=. dnf reinstall -y /rpms/vespa*rpm && dnf clean all
USER vespa
EOF
buildah bud --security-opt label=disable --network host --squash --build-arg VESPA_VERSION=$VESPA_VERSION --tag docker.io/$IMAGE_NAME:$VESPA_VERSION \
--tag docker.io/$IMAGE_NAME:latest --file Dockerfile .
- verify-container-image: |
# Trick to be able to use the documentation testing to verify the image built locally
buildah tag docker.io/$IMAGE_NAME:$VESPA_VERSION vespaengine/vespa:latest
# Run quick start guide
$SD_SOURCE_DIR/screwdriver/test-quick-start-guide.sh
- publish-test-image: |
if [[ -z $SD_PULL_REQUEST ]]; then
if curl -fsSL https://index.docker.io/v1/repositories/$IMAGE_NAME/tags/$VESPA_VERSION &> /dev/null; then
echo "Container image docker.io/$IMAGE_NAME:$VESPA_VERSION aldready exists."
else
OPT_STATE="$(set +o)"
set +x
buildah login --username aressem --password "$DOCKER_HUB_DEPLOY_KEY" docker.io
eval "$OPT_STATE"
buildah push --format v2s2 docker.io/$IMAGE_NAME:$VESPA_VERSION docker://docker.io/$IMAGE_NAME:$VESPA_VERSION
buildah push --format v2s2 docker.io/$IMAGE_NAME:$VESPA_VERSION docker://docker.io/$IMAGE_NAME:latest
fi
fi
publish-cli-release:
image: homebrew/brew:latest
annotations:
# Run once an hour, in the hours 7-15 UTC, Monday-Thursday
screwdriver.cd/buildPeriodically: H 7-15 * * 1-4
secrets:
- HOMEBREW_GITHUB_API_TOKEN
- GH_TOKEN
steps:
- install-dependencies: |
export HOMEBREW_NO_INSTALL_CLEANUP=1 HOMEBREW_NO_ANALYTICS=1
brew install --quiet gh zip go
- publish-github:
make -C client/go clean dist-github
- publish-homebrew:
make -C client/go clean dist-homebrew
- verify-brew-install:
make -C client/go install-brew
verify-opensource-release-7days:
annotations:
screwdriver.cd/buildPeriodically: H 0 * * *
steps:
- now-epoch: |
now_epoch=`date "+%s"`
echo "Now epoch: " $now_epoch
- calculate-current-release-age: |
current_release_date=`curl -s 'https://repo1.maven.org/maven2/com/yahoo/vespa/parent/' | \
grep '^