From 704b27404c6cf3170bc43d094d93401d7c18cc8d Mon Sep 17 00:00:00 2001 From: Arnstein Ressem Date: Wed, 13 Oct 2021 12:08:04 +0200 Subject: Add public component build. --- screwdriver.yaml | 169 ++++++++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 135 insertions(+), 34 deletions(-) diff --git a/screwdriver.yaml b/screwdriver.yaml index bbffbd2ecde..2e5df434eb8 100644 --- a/screwdriver.yaml +++ b/screwdriver.yaml @@ -3,15 +3,51 @@ cache: job: build-vespa: [/main_job_cache] + component-build: [/main_job_cache] shared: image: vespaengine/vespa-build-centos7:latest environment: USER_SHELL_BIN: bash + annotations: + restore-cache: &restore-cache + restore-cache: | + (cd /tmp && if [[ -f $MAIN_CACHE_FILE ]]; then tar xf $MAIN_CACHE_FILE; fi) + + mkdir -p $CCACHE_DATA_DIR + mkdir -p $CCACHE_TMP_DIR + + export CCACHE_DIR=$CCACHE_DATA_DIR + + rm -f $CCACHE_DIR/ccache.conf + ccache -M 20G + ccache -o log_file=$SD_ARTIFACTS_DIR/ccache_log.txt + ccache -o temporary_dir=$CCACHE_TMP_DIR + ccache -p + ccache -z + du -sh /tmp/* + + save-cache: &save-cache + save-cache: | + if [[ -z "$SD_PULL_REQUEST" ]]; then + # Remove what we have produced + rm -rf $LOCAL_MVN_REPO/com/yahoo + rm -rf $LOCAL_MVN_REPO/ai/vespa + + # Tar toghether the /tmp/vespa folder containing ccache and cleaned mvn repo + mkdir -p $(dirname $MAIN_CACHE_FILE) + (cd /tmp && tar cf $MAIN_CACHE_FILE vespa) + + # Wipe the cache if we exceed 2GB to avoid pulling and pusing too large files + if (( $(stat --format='%s' $MAIN_CACHE_FILE) > $(( 2*1000*1000*1000 )) )); then + tar cf $MAIN_CACHE_FILE --files-from=/dev/null; + echo "Cleaning cache file. $MAIN_CACHE_FILE is now $(stat --format='%s' $MAIN_CACHE_FILE) bytes." + fi + fi jobs: build-vespa: - requires: [~pr, ~commit] + requires: [~pr] annotations: screwdriver.cd/cpu: 7 screwdriver.cd/ram: 16 @@ -34,46 +70,111 @@ jobs: df -h uname -a rpm -qa | grep "vespa" - - restore-cache: | - (cd /tmp && if [[ -f $MAIN_CACHE_FILE ]]; then tar xf $MAIN_CACHE_FILE; fi) - - mkdir -p $CCACHE_DATA_DIR - mkdir -p $CCACHE_TMP_DIR - - export CCACHE_DIR=$CCACHE_DATA_DIR - - rm -f $CCACHE_DIR/ccache.conf - ccache -M 20G - ccache -o log_file=$SD_ARTIFACTS_DIR/ccache_log.txt - ccache -o temporary_dir=$CCACHE_TMP_DIR - ccache -p - ccache -z - + - *restore-cache - compile: | screwdriver/build-vespa.sh - - - save-cache: | - if [[ -z "$SD_PULL_REQUEST" ]]; then - # Remove what we have produced - rm -rf $LOCAL_MVN_REPO/com/yahoo - rm -rf $LOCAL_MVN_REPO/ai/vespa - - # Tar toghether the /tmp/vespa folder containing ccache and cleaned mvn repo - mkdir -p $(dirname $MAIN_CACHE_FILE) - (cd /tmp && tar cf $MAIN_CACHE_FILE vespa) - - # Wipe the cache if we exceed 2GB to avoid pulling and pusing too large files - if (( $(stat --format='%s' $MAIN_CACHE_FILE) > $(( 2*1000*1000*1000 )) )); then - tar cf $MAIN_CACHE_FILE --files-from=/dev/null; - echo "Cleaning cache file. $MAIN_CACHE_FILE is now $(stat --format='%s' $MAIN_CACHE_FILE) bytes." - fi - fi - + - *save-cache - inspect-after: | du -sh /tmp/vespa/* ls -la /main_job_cache || true df -h + component-build: + annotations: + screwdriver.cd/cpu: 7 + screwdriver.cd/ram: 16 + screwdriver.cd/disk: HIGH + screwdriver.cd/timeout: 90 + screwdriver.cd/dockerEnabled: true + screwdriver.cd/dockerCpu: TURBO + screwdriver.cd/dockerRam: TURBO + environment: + LOCAL_MVN_REPO: "/tmp/vespa/mvnrepo" + VESPA_MAVEN_EXTRA_OPTS: "--show-version --batch-mode --no-snapshot-updates -Dmaven.repo.local=/tmp/vespa/mvnrepo" + CCACHE_TMP_DIR: "/tmp/ccache_tmp" + CCACHE_DATA_DIR: "/tmp/vespa/ccache" + MAIN_CACHE_FILE: "/main_job_cache/vespa.tar" + secrets: + - DOCKER_HUB_DEPLOY_KEY + steps: + - get-vespa-version: | + VESPA_VERSION=$(meta get vespa.version) + VESPA_REF=$(meta get vespa.ref) + SYSTEM_TEST_REF=$(meta get system-test.ref) + + if [[ $VESPA_VERSION == null ]] || [[ $VESPA_REF == null ]] || [[ $SYSTEM_TEST_REF == null ]]; then + echo "Must have valid Vespa version and reference to continue \ + (got VESPA_VERSION=$VESPA_VERSION, VESPA_REF=$VESPA_REF, SYSTEM_TEST_REF=$SYSTEM_TEST_REF)." + exit 1 + fi + - install-dependencies: | + yum-config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo + yum install -y --setopt skip_missing_names_on_install=False docker-ce docker-ce-cli containerd.io + docker system info + ln -sf $(which yum) /usr/local/bin/dnf + - checkout: | + mkdir -p workdir + cd workdir + WORKDIR=$(pwd) + git clone -q https://github.com/vespa-engine/vespa + (cd vespa && git checkout $VESPA_REF && git tag v$VESPA_VERSION) + git clone -q https://github.com/vespa-engine/system-test + (cd system-test && git checkout $SYSTEM_TEST_REF) + # Set correct version in pom.xml files + (cd vespa && \ + find . -name "pom.xml" -exec sed -i'' \ + -e "s,.*SNAPSHOT.*,$VESPA_VERSION," \ + -e "s,.*project.version.*,$VESPA_VERSION," \ + -e "s,.*project.version.*,$VESPA_VERSION," \ + {} \;) + - make-srpm: | + make -C $WORKDIR/vespa -f .copr/Makefile srpm outdir=$WORKDIR + - *restore-cache + - build: | + cd $WORKDIR/vespa + export FACTORY_VESPA_VERSION=$VESPA_VERSION + NUM_THREADS=$(( $(nproc) + 2 )) + time ./bootstrap.sh java + time mvn -T $NUM_THREADS $VESPA_MAVEN_EXTRA_OPTS install + cmake3 -DVESPA_UNPRIVILEGED=no . + time make -j ${NUM_THREADS} + time ctest3 --output-on-failure -j ${NUM_THREADS} + ccache --show-stats + time make -j ${NUM_THREADS} install DESTDIR=$WORKDIR/vespa-install + - verify-rpm-build: | + cd $WORKDIR + time rpmbuild --rebuild --define="_topdir $WORKDIR/vespa-rpmbuild" \ + --define "debug_package %{nil}" --define "_debugsource_template %{nil}" \ + --define "installdir $WORKDIR/vespa-install" *.src.rpm + mkdir -p $WORKDIR/system-test/docker/rpms + mv $WORKDIR/vespa-rpmbuild/RPMS/x86_64/*.rpm $WORKDIR/system-test/docker/rpms + ls -la $WORKDIR/system-test/docker/rpms + - build-test-image: | + cd $WORKDIR/system-test + mkdir -p docker/vespa-systemtests + git archive HEAD --format tar | tar x -C docker/vespa-systemtests + cp -a $LOCAL_MVN_REPO docker/repository + cd docker + docker build --file Dockerfile.multistage \ + --build-arg BASE_IMAGE=centos:7 --build-arg VESPA_BASE_IMAGE=base \ + --build-arg SYSTEMTEST_BASE_IMAGE=vespa --build-arg SKIP_M2_POPULATE=true \ + --target systemtest \ + --tag vespaengine/vespa-systemtest-centos7:$VESPA_VERSION \ + --tag vespaengine/vespa-systemtest-centos7:latest . + - verify-test-image: | + docker run --rm -ti --entrypoint bash vespaengine/vespa-systemtest-centos7:$VESPA_VERSION -lc \ + "source /opt/rh/rh-ruby27/enable; /opt/vespa-systemtests/lib/node_server.rb & sleep 3 && ruby /opt/vespa-systemtests/tests/search/basicsearch/basic_search.rb --run test_basicsearch__ELASTIC" + - publish-test-image: | + if [[ -z "$SD_PULL_REQUEST" ]]; then + OPT_STATE="$(set +o)" + set +x + docker login --username aressem --password "$DOCKER_HUB_DEPLOY_KEY" + eval "$OPT_STATE" + docker push docker.io/vespaengine/vespa-systemtest-centos7:$VESPA_VERSION + docker push docker.io/vespaengine/vespa-systemtest-centos7:latest + fi + - *save-cache + publish-release: annotations: screwdriver.cd/cpu: 7 -- cgit v1.2.3 From a46dde058e895c8c7afd71f7e898292cfe8b23ec Mon Sep 17 00:00:00 2001 From: Arnstein Ressem Date: Fri, 15 Oct 2021 22:50:11 +0200 Subject: Fix indent. --- screwdriver.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/screwdriver.yaml b/screwdriver.yaml index 2e5df434eb8..71e21db0488 100644 --- a/screwdriver.yaml +++ b/screwdriver.yaml @@ -80,7 +80,7 @@ jobs: df -h component-build: - annotations: + annotations: screwdriver.cd/cpu: 7 screwdriver.cd/ram: 16 screwdriver.cd/disk: HIGH -- cgit v1.2.3 From fefb6e03d23b41fc46a04f1443fdea2bb266ddea Mon Sep 17 00:00:00 2001 From: Arnstein Ressem Date: Fri, 15 Oct 2021 22:59:02 +0200 Subject: Create the debuginfo package, but do not install. --- screwdriver.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/screwdriver.yaml b/screwdriver.yaml index 71e21db0488..f16cb987c27 100644 --- a/screwdriver.yaml +++ b/screwdriver.yaml @@ -144,10 +144,11 @@ jobs: - verify-rpm-build: | cd $WORKDIR time rpmbuild --rebuild --define="_topdir $WORKDIR/vespa-rpmbuild" \ - --define "debug_package %{nil}" --define "_debugsource_template %{nil}" \ + --define "_debugsource_template %{nil}" \ --define "installdir $WORKDIR/vespa-install" *.src.rpm mkdir -p $WORKDIR/system-test/docker/rpms mv $WORKDIR/vespa-rpmbuild/RPMS/x86_64/*.rpm $WORKDIR/system-test/docker/rpms + rm -f $WORKDIR/system-test/docker/rpms/*debuginfo*rpm ls -la $WORKDIR/system-test/docker/rpms - build-test-image: | cd $WORKDIR/system-test -- cgit v1.2.3