path: root/screwdriver.yaml

# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
---
cache:
 job:
   build-vespa: [/main_job_cache]
   component-build: [/main_job_cache]

shared:
  image: vespaengine/vespa-build-centos-stream8:latest
  environment:
    USER_SHELL_BIN: bash
  annotations:
    restore-cache: &restore-cache
      restore-cache: |
        (cd /tmp && if [[ -f $MAIN_CACHE_FILE ]]; then tar xf $MAIN_CACHE_FILE; fi)

        mkdir -p $CCACHE_DATA_DIR
        mkdir -p $CCACHE_TMP_DIR

        export CCACHE_DIR=$CCACHE_DATA_DIR

        rm -f $CCACHE_DIR/ccache.conf
        ccache -M 20G
        ccache -o log_file=$SD_ARTIFACTS_DIR/ccache_log.txt
        ccache -o temporary_dir=$CCACHE_TMP_DIR
        ccache -p
        ccache -z
        du -sh /tmp/*

    save-cache: &save-cache
      save-cache: |
        ls -lart /tmp/vespa
        du -sh /tmp/vespa/*

        if [[ -z "$SD_PULL_REQUEST" ]]; then
          # Remove what we have produced
          rm -rf $LOCAL_MVN_REPO/com/yahoo
          rm -rf $LOCAL_MVN_REPO/ai/vespa

          # Tar toghether the /tmp/vespa folder containing ccache and cleaned mvn repo
          mkdir -p $(dirname $MAIN_CACHE_FILE)
          (cd /tmp && tar cf $MAIN_CACHE_FILE vespa)

          # Wipe the cache if we exceed 2GB to avoid pulling and pusing too large files
          if (( $(stat --format='%s' $MAIN_CACHE_FILE) > $(( 2200*1000*1000 )) )); then
            tar cf $MAIN_CACHE_FILE --files-from=/dev/null;
            echo "Cleaning cache file. $MAIN_CACHE_FILE is now $(stat --format='%s' $MAIN_CACHE_FILE) bytes."
          fi
        fi

jobs:
  build-vespa:
    requires: [~pr]
    annotations:
      screwdriver.cd/cpu: 7
      screwdriver.cd/ram: 16
      screwdriver.cd/disk: HIGH
      screwdriver.cd/timeout: 60
      screwdriver.cd/buildPeriodically: H 4 * * *

    environment:
      LOCAL_MVN_REPO: "/tmp/vespa/mvnrepo"
      VESPA_MAVEN_EXTRA_OPTS: "-Dmaven.repo.local=/tmp/vespa/mvnrepo -Dmaven.source.skip=true"
      CCACHE_TMP_DIR: "/tmp/ccache_tmp"
      CCACHE_DATA_DIR: "/tmp/vespa/ccache"
      MAIN_CACHE_FILE: "/main_job_cache/vespa.tar"

    steps:
      - inspect: |
          set -x
          env | grep -v TOKEN
          cat /proc/cpuinfo
          cat /proc/meminfo
          df -h
          uname -a
          rpm -qa | grep "vespa"
      - *restore-cache
      - compile: |
          screwdriver/build-vespa.sh
      - *save-cache
      - inspect-after: |
          du -sh /tmp/vespa/*
          ls -la /main_job_cache || true
          df -h

  component-build:
    annotations:
      screwdriver.cd/cpu: 7
      screwdriver.cd/ram: 16
      screwdriver.cd/disk: HIGH
      screwdriver.cd/timeout: 150
      screwdriver.cd/dockerEnabled: true
      screwdriver.cd/dockerCpu: TURBO
      screwdriver.cd/dockerRam: HIGH
    environment:
      LOCAL_MVN_REPO: "/tmp/vespa/mvnrepo"
      VESPA_MAVEN_EXTRA_OPTS: "--show-version --batch-mode --no-snapshot-updates -Dmaven.repo.local=/tmp/vespa/mvnrepo"
      CCACHE_TMP_DIR: "/tmp/ccache_tmp"
      CCACHE_DATA_DIR: "/tmp/vespa/ccache"
      MAIN_CACHE_FILE: "/main_job_cache/vespa.tar"
      CCACHE_COMPRESS: "1"
    secrets:
      - DOCKER_HUB_DEPLOY_KEY
    steps:
      - get-vespa-version: |
          VESPA_VERSION=$(meta get vespa.version)
          VESPA_REF=$(meta get vespa.ref)
          SYSTEM_TEST_REF=$(meta get system-test.ref)

          if [[ $VESPA_VERSION == null ]] || [[ $VESPA_REF == null ]] || [[ $SYSTEM_TEST_REF == null ]]; then
            echo "Must have valid Vespa version and reference to continue \
                  (got VESPA_VERSION=$VESPA_VERSION, VESPA_REF=$VESPA_REF, SYSTEM_TEST_REF=$SYSTEM_TEST_REF)."
            exit 1
          fi
      - install-dependencies: |
          dnf config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo
          dnf -y install docker-ce docker-ce-cli containerd.io
          docker system info
      - checkout: |
          mkdir -p workdir
          cd workdir
          WORKDIR=$(pwd)
          git clone -q https://github.com/vespa-engine/vespa
          (cd vespa && git checkout $VESPA_REF && git tag v$VESPA_VERSION)
          git clone -q https://github.com/vespa-engine/system-test
          (cd system-test && git checkout $SYSTEM_TEST_REF)
          git clone -q https://github.com/vespa-engine/sample-apps
          # Set correct version in pom.xml files
          (cd vespa && screwdriver/replace-vespa-version-in-poms.sh $VESPA_VERSION $(pwd) )
          (cd sample-apps && find . -name "pom.xml" -exec sed -i -e "s,<vespa_version>.*</vespa_version>,<vespa_version>$VESPA_VERSION</vespa_version>," {} \;)
      - make-srpm: |
          make -C $WORKDIR/vespa -f .copr/Makefile srpm outdir=$WORKDIR
      - *restore-cache
      - build: |
          cd $WORKDIR/vespa
          export FACTORY_VESPA_VERSION=$VESPA_VERSION
          NUM_THREADS=$(( $(nproc) + 2 ))
          VESPA_CMAKE_SANITIZERS_OPTION="-DVESPA_USE_SANITIZER=thread"
          time make -C client/go BIN=$WORKDIR/vespa-install/opt/vespa/bin SHARE=$WORKDIR/vespa-install/usr/share install-all
          time ./bootstrap.sh java
          time mvn -T $NUM_THREADS $VESPA_MAVEN_EXTRA_OPTS install
          cmake3 -DVESPA_UNPRIVILEGED=no $VESPA_CMAKE_SANITIZERS_OPTION .
          time make -j ${NUM_THREADS}
          time ctest3 --output-on-failure -j ${NUM_THREADS}
          ccache --show-stats
          time make -j ${NUM_THREADS} install DESTDIR=$WORKDIR/vespa-install
      - build-sample-apps: |
          (cd $WORKDIR/sample-apps && time mvn -T $NUM_THREADS $VESPA_MAVEN_EXTRA_OPTS package)
      - verify-rpm-build: |
          cd $WORKDIR
          ulimit -c 0
          time rpmbuild --rebuild --define="_topdir $WORKDIR/vespa-rpmbuild" \
                        --define "_debugsource_template %{nil}" \
                        --define "installdir $WORKDIR/vespa-install" *.src.rpm
          mkdir -p $WORKDIR/system-test/docker/rpms
          mv $WORKDIR/vespa-rpmbuild/RPMS/x86_64/*.rpm $WORKDIR/system-test/docker/rpms
          if test -z "$VESPA_CMAKE_SANITIZERS_OPTION"
          then
              rm -f $WORKDIR/system-test/docker/rpms/*debuginfo*rpm
          fi
          ls -la $WORKDIR/system-test/docker/rpms
      - build-test-image: |
          cd $WORKDIR/system-test
          mkdir -p docker/vespa-systemtests
          git archive HEAD --format tar | tar x -C docker/vespa-systemtests
          cp -a $LOCAL_MVN_REPO docker/repository
          cd docker
          docker build --file Dockerfile.systemtest \
                       --build-arg VESPA_BASE_IMAGE=vespaengine/vespa-systemtest-base-centos-stream8:latest \
                       --build-arg SYSTEMTEST_BASE_IMAGE=vespa --build-arg SKIP_M2_POPULATE=false \
                       --target systemtest \
                       --tag vespaengine/vespa-systemtest-centos-stream8:$VESPA_VERSION \
                       --tag vespaengine/vespa-systemtest-centos-stream8:latest .
      - verify-test-image: |
          docker run --rm -ti --entrypoint bash vespaengine/vespa-systemtest-centos-stream8:$VESPA_VERSION -lc \
          "/opt/vespa-systemtests/lib/node_server.rb & sleep 3 && ruby /opt/vespa-systemtests/tests/search/basicsearch/basic_search.rb --run test_basicsearch__ELASTIC"
      - publish-test-image: |
          if [[ -z "$SD_PULL_REQUEST" ]]; then
            OPT_STATE="$(set +o)"
            set +x
            docker login --username aressem --password "$DOCKER_HUB_DEPLOY_KEY"
            eval "$OPT_STATE"
            docker push docker.io/vespaengine/vespa-systemtest-centos-stream8:$VESPA_VERSION
            docker push docker.io/vespaengine/vespa-systemtest-centos-stream8:latest
          fi
      - *save-cache

  publish-release:
    image: docker.io/vespaengine/vespa-build-centos-stream8:latest

    annotations:
      screwdriver.cd/cpu: 7
      screwdriver.cd/ram: 16
      screwdriver.cd/disk: HIGH
      screwdriver.cd/timeout: 300
      screwdriver.cd/dockerEnabled: true
      screwdriver.cd/dockerCpu: TURBO
      screwdriver.cd/dockerRam: HIGH

    secrets:
      - COPR_WEBHOOK
      - OSSRH_USER
      - OSSRH_TOKEN
      - GPG_KEYNAME
      - GPG_PASSPHRASE
      - GPG_ENCPHRASE
      - SAMPLE_APPS_DEPLOY_KEY
      - VESPA_DEPLOY_KEY
      - DOCKER_IMAGE_DEPLOY_KEY
      - DOCKER_HUB_DEPLOY_KEY
      - GHCR_DEPLOY_KEY

    environment:
       GIT_SSH_COMMAND: "ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no"

    steps:
      - get-vespa-version: |
          VESPA_VERSION=$(meta get vespa.version)
          VESPA_REF=$(meta get vespa.ref)
          if [[ $VESPA_VERSION == null ]] || [[ $VESPA_REF == null ]]; then
            echo "Must have valid Vespa version and reference to continue (got VESPA_VERSION=$VESPA_VERSION, VESPA_REF=$VESPA_REF)."
            exit 1
          fi
      - install-dependencies: |
          dnf config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo
          dnf install -y docker-ce docker-ce-cli containerd.io
          docker system info
      - release-java-artifacts: |
          screwdriver/release-java-artifacts.sh $VESPA_VERSION $VESPA_REF
      - release-rpms: |
          screwdriver/release-rpms.sh $VESPA_VERSION $VESPA_REF
      - release-container-image: |
          screwdriver/release-container-image.sh $VESPA_VERSION
      - update-sample-apps: |
          screwdriver/update-vespa-version-in-sample-apps.sh $VESPA_VERSION

  publish-cli-release:
    image: homebrew/brew:latest
    annotations:
      # Run once an hour, in the hours 7-15 UTC, Monday-Thursday
      screwdriver.cd/buildPeriodically: H 7-15 * * 1-4
    secrets:
      - HOMEBREW_GITHUB_API_TOKEN
      - GH_TOKEN
    steps:
      - install-dependencies: |
          export HOMEBREW_NO_INSTALL_CLEANUP=1 HOMEBREW_NO_ANALYTICS=1
          brew install --quiet gh zip go
      - publish-github:
          make -C client/go clean dist-github
      - publish-homebrew:
          make -C client/go clean dist-homebrew
      - verify-brew-install:
          make -C client/go install-brew

  verify-opensource-release-7days:
    annotations:
      screwdriver.cd/buildPeriodically: H 0 * * *
    steps:
      - now-epoch: |
          now_epoch=`date "+%s"`
          echo "Now epoch: " $now_epoch
      - calculate-current-release-age: |
          current_release_date=`curl -s 'https://repo1.maven.org/maven2/com/yahoo/vespa/parent/' | \
            grep '^<a href="[0-9]' | awk '{print $4}' | sort | tail -1`
          echo "Current release date: " $current_release_date
          current_release_epoch=`date -d "$current_release_date" "+%s"`
          echo "Current release epoch: " $current_release_epoch
          release_age_days=$((($now_epoch-$current_release_epoch)/86400))
          echo "Release age days: " $release_age_days
      - error-if-current-release-too-old: |
          if [ "$release_age_days" -gt 7 ]; then
            echo "Current open source release is older than 7 days"
            exit 1
          fi
      - calculate-docker-image-age: |
          image_date=`curl https://hub.docker.com/v2/repositories/vespaengine/vespa/ | \
            python -m json.tool| grep last_updated | awk '{print $2}' | tr -d '",'`
          echo "Docker image last_updated: " $image_date
          image_epoch=`date -d "$image_date" "+%s"`
          echo "Docker image epoch: " $image_epoch
          docker_image_age_days=$((($now_epoch-$image_epoch)/86400))
          echo "Docker image age days: " $docker_image_age_days
      - error-if-docker-image-too-old: |
          if [ "$docker_image_age_days" -gt 7 ]; then
            echo "Current Docker image is older than 7 days"
            exit 1
          fi

  verify-opensource-rpm-installable:
    image: quay.io/centos/centos:stream8
    annotations:
      screwdriver.cd/buildPeriodically: H 0 * * *
    steps:
      - install: |
          dnf config-manager --add-repo https://raw.githubusercontent.com/vespa-engine/vespa/master/dist/vespa-engine.repo
          dnf config-manager --enable powertools
          dnf install -y epel-release
          dnf install -y vespa

  mirror-copr-rpms-to-artifactory:
    annotations:
      screwdriver.cd/cpu: LOW
      screwdriver.cd/ram: LOW
      screwdriver.cd/disk: HIGH
      screwdriver.cd/timeout: 60
      screwdriver.cd/buildPeriodically: H 6 * * *
    secrets:
      - JFROG_API_TOKEN
    steps:
      - mirror: |
          screwdriver/publish-unpublished-rpms-to-jfrog-cloud.sh

  delete-old-versions-on-artifactory:
    annotations:
      screwdriver.cd/cpu: LOW
      screwdriver.cd/ram: LOW
      screwdriver.cd/timeout: 10
      screwdriver.cd/buildPeriodically: H 6 * * 1
    secrets:
      - JFROG_API_TOKEN
    steps:
      - cleanup: |
          screwdriver/delete-old-artifactory-artifacts.sh