path: root/screwdriver.yaml

# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
---
cache:
 job:
   build-vespa: [/main_job_cache]
   component-build: [/main_job_cache]

shared:
  image: vespaengine/vespa-build-centos-stream8:latest
  environment:
    USER_SHELL_BIN: bash
  annotations:
    restore-cache: &restore-cache
      restore-cache: |
        (cd /tmp && if [[ -f $MAIN_CACHE_FILE ]]; then tar xf $MAIN_CACHE_FILE; fi)

        mkdir -p $CCACHE_DATA_DIR
        mkdir -p $CCACHE_TMP_DIR

        export CCACHE_DIR=$CCACHE_DATA_DIR

        rm -f $CCACHE_DIR/ccache.conf
        ccache -M 20G
        ccache -o log_file=$SD_ARTIFACTS_DIR/ccache_log.txt
        ccache -o temporary_dir=$CCACHE_TMP_DIR
        ccache -p
        ccache -z
        du -sh /tmp/*

    save-cache: &save-cache
      save-cache: |
        ls -lart /tmp/vespa
        du -sh /tmp/vespa/*

        if [[ -z "$SD_PULL_REQUEST" ]]; then
          # Remove what we have produced
          rm -rf $LOCAL_MVN_REPO/com/yahoo
          rm -rf $LOCAL_MVN_REPO/ai/vespa

          # Tar toghether the /tmp/vespa folder containing ccache and cleaned mvn repo
          mkdir -p $(dirname $MAIN_CACHE_FILE)
          (cd /tmp && tar cf $MAIN_CACHE_FILE vespa)

          # Wipe the cache if we exceed 2GB to avoid pulling and pusing too large files
          if (( $(stat --format='%s' $MAIN_CACHE_FILE) > $(( 2200*1000*1000 )) )); then
            tar cf $MAIN_CACHE_FILE --files-from=/dev/null;
            echo "Cleaning cache file. $MAIN_CACHE_FILE is now $(stat --format='%s' $MAIN_CACHE_FILE) bytes."
          fi
        fi

    install-podman-multi-arch: &install-podman-multi-arch
      install-podman-multi-arch: |
          dnf install -y podman podman-docker buildah skopeo
          sed -i 's,.*netns.*=.*private.*,netns = "host",' /usr/share/containers/containers.conf
          touch /etc/containers/nodocker
          podman run --rm --quiet --cap-add SYS_ADMIN docker.io/multiarch/qemu-user-static --reset -p yes

jobs:
  build-vespa:
    requires: [~pr]
    annotations:
      screwdriver.cd/cpu: 7
      screwdriver.cd/ram: 16
      screwdriver.cd/disk: HIGH
      screwdriver.cd/timeout: 60
      screwdriver.cd/buildPeriodically: H 4 * * *

    environment:
      LOCAL_MVN_REPO: "/tmp/vespa/mvnrepo"
      VESPA_MAVEN_EXTRA_OPTS: "-Dmaven.repo.local=/tmp/vespa/mvnrepo -Dmaven.source.skip=true"
      CCACHE_TMP_DIR: "/tmp/ccache_tmp"
      CCACHE_DATA_DIR: "/tmp/vespa/ccache"
      MAIN_CACHE_FILE: "/main_job_cache/vespa.tar"

    steps:
      - inspect: |
          set -x
          env | grep -v TOKEN
          cat /proc/cpuinfo
          cat /proc/meminfo
          df -h
          uname -a
          rpm -qa | grep "vespa"
      - *restore-cache
      - compile: |
          screwdriver/build-vespa.sh
      - *save-cache
      - inspect-after: |
          du -sh /tmp/vespa/*
          ls -la /main_job_cache || true
          df -h

  component-build:
    annotations:
      screwdriver.cd/cpu: 7
      screwdriver.cd/ram: 16
      screwdriver.cd/disk: HIGH
      screwdriver.cd/timeout: 90
    environment:
      LOCAL_MVN_REPO: "/tmp/vespa/mvnrepo"
      VESPA_MAVEN_EXTRA_OPTS: "--show-version --batch-mode --no-snapshot-updates -Dmaven.repo.local=/tmp/vespa/mvnrepo"
      CCACHE_TMP_DIR: "/tmp/ccache_tmp"
      CCACHE_DATA_DIR: "/tmp/vespa/ccache"
      MAIN_CACHE_FILE: "/main_job_cache/vespa.tar"
      CCACHE_COMPRESS: "1"
    secrets:
      - DOCKER_HUB_DEPLOY_KEY
      - SVC_OKTA_VESPA_FACTORY_TOKEN
    steps:
      - get-vespa-version: |
          VERSION_INFO=$(screwdriver/factory-command.sh create-build)
          VESPA_VERSION=$(jq -re '.version' <<< "$VERSION_INFO")
          VESPA_REF=$(jq -re '.commits|.[]|select(.repo=="vespa")|.ref' <<< "$VERSION_INFO")
          SYSTEM_TEST_REF=$(jq -re '.commits|.[]|select(.repo=="system-test")|.ref' <<< "$VERSION_INFO")
          if [[ $VESPA_VERSION == null ]] || [[ $VESPA_REF == null ]] || [[ $SYSTEM_TEST_REF == null ]]; then
            echo "Must have valid Vespa version and reference to continue \
                  (got VESPA_VERSION=$VESPA_VERSION, VESPA_REF=$VESPA_REF, SYSTEM_TEST_REF=$SYSTEM_TEST_REF)."
            exit 1
          fi
      - *install-podman-multi-arch
      - checkout: |
          mkdir -p workdir
          cd workdir
          WORKDIR=$(pwd)
          git clone -q https://github.com/vespa-engine/vespa
          (cd vespa && git checkout $VESPA_REF && git tag v$VESPA_VERSION)
          git clone -q https://github.com/vespa-engine/system-test
          (cd system-test && git checkout $SYSTEM_TEST_REF)
          git clone -q https://github.com/vespa-engine/sample-apps
          # Set correct version in pom.xml files
          (cd vespa && screwdriver/replace-vespa-version-in-poms.sh $VESPA_VERSION $(pwd) )
          (cd sample-apps && find . -name "pom.xml" -exec sed -i -e "s,<vespa_version>.*</vespa_version>,<vespa_version>$VESPA_VERSION</vespa_version>," {} \;)
          (cd sample-apps && find . -name "pom.xml" -exec sed -i -e "s:<version>[[]8,9[)]</version>:<version>$VESPA_VERSION</version>:" {} \;)
      - make-srpm: |
          $SD_SOURCE_DIR/screwdriver/factory-command.sh update-build-status running "Building"

          make -C $WORKDIR/vespa -f .copr/Makefile srpm outdir=$WORKDIR
      - *restore-cache
      - build: |
          cd $WORKDIR/vespa
          export FACTORY_VESPA_VERSION=$VESPA_VERSION
          NUM_THREADS=$(( $(nproc) + 2 ))
          VESPA_CMAKE_SANITIZERS_OPTION=""
          time make -C client/go BIN=$WORKDIR/vespa-install/opt/vespa/bin SHARE=$WORKDIR/vespa-install/usr/share install-all
          time ./bootstrap.sh java
          time mvn -T $NUM_THREADS $VESPA_MAVEN_EXTRA_OPTS install
          cmake3 -DVESPA_UNPRIVILEGED=no $VESPA_CMAKE_SANITIZERS_OPTION .
          time make -j ${NUM_THREADS}
          time ctest3 --output-on-failure -j ${NUM_THREADS}
          ccache --show-stats
          time make -j ${NUM_THREADS} install DESTDIR=$WORKDIR/vespa-install
      - build-sample-apps: |
          (cd $WORKDIR/sample-apps && time mvn -T $NUM_THREADS $VESPA_MAVEN_EXTRA_OPTS package)
      - verify-rpm-build: |
          cd $WORKDIR
          ulimit -c 0
          time rpmbuild --rebuild --define="_topdir $WORKDIR/vespa-rpmbuild" \
                        --define "_debugsource_template %{nil}" \
                        --define "installdir $WORKDIR/vespa-install" *.src.rpm
          mkdir -p $WORKDIR/system-test/docker/rpms
          mv $WORKDIR/vespa-rpmbuild/RPMS/x86_64/*.rpm $WORKDIR/system-test/docker/rpms
          if test -z "$VESPA_CMAKE_SANITIZERS_OPTION"
          then
              rm -f $WORKDIR/system-test/docker/rpms/*debuginfo*rpm
          fi
          ls -la $WORKDIR/system-test/docker/rpms
      - build-test-image: |
          cd $WORKDIR/system-test
          mkdir -p docker/vespa-systemtests
          git archive HEAD --format tar | tar x -C docker/vespa-systemtests
          cp -a $LOCAL_MVN_REPO docker/repository
          cd docker
          buildah bud --file Dockerfile.systemtest \
                      --build-arg VESPA_BASE_IMAGE=docker.io/vespaengine/vespa-systemtest-base-centos-stream8:latest \
                      --build-arg SYSTEMTEST_BASE_IMAGE=vespa --build-arg SKIP_M2_POPULATE=false \
                      --target systemtest \
                      --tag docker.io/vespaengine/vespa-systemtest-centos-stream8:$VESPA_VERSION .
      - verify-test-image: |
          podman run --rm -ti --entrypoint bash docker.io/vespaengine/vespa-systemtest-centos-stream8:$VESPA_VERSION -lc \
          "/opt/vespa-systemtests/lib/node_server.rb & sleep 3 && ruby /opt/vespa-systemtests/tests/search/basicsearch/basic_search.rb --run test_basicsearch__ELASTIC"
      - publish-test-image: |
          if [[ -z "$SD_PULL_REQUEST" ]]; then
            OPT_STATE="$(set +o)"
            set +x
            buildah login --username aressem --password "$DOCKER_HUB_DEPLOY_KEY" docker.io
            eval "$OPT_STATE"
            buildah push --format v2s2 docker.io/vespaengine/vespa-systemtest-centos-stream8:$VESPA_VERSION docker://docker.io/vespaengine/vespa-systemtest-centos-stream8:$VESPA_VERSION
            buildah push --format v2s2 docker.io/vespaengine/vespa-systemtest-centos-stream8:$VESPA_VERSION docker://docker.io/vespaengine/vespa-systemtest-centos-stream8:latest
          fi
      - *save-cache
      - update-build-status: |
          meta set vespa.factory.build.status success
      - teardown-report-build-status: |
          FACTORY_BUILD_STATUS=$(meta get vespa.factory.build.status)
          if [[ $FACTORY_BUILD_STATUS == success ]]; then
            DESCRIPTION="Build success"
          else
            FACTORY_BUILD_STATUS=failure
            DESCRIPTION="Build failed"
          fi
          echo "Reporting build status as $FACTORY_BUILD_STATUS"
          $SD_SOURCE_DIR/screwdriver/factory-command.sh update-build-status $FACTORY_BUILD_STATUS "$DESCRIPTION"

  publish-release:
    image: docker.io/vespaengine/vespa-build-centos-stream8:latest

    annotations:
      screwdriver.cd/cpu: 7
      screwdriver.cd/ram: 16
      screwdriver.cd/disk: HIGH
      screwdriver.cd/timeout: 300
      screwdriver.cd/dockerEnabled: true
      screwdriver.cd/dockerCpu: TURBO
      screwdriver.cd/dockerRam: HIGH

    secrets:
      - COPR_WEBHOOK
      - OSSRH_USER
      - OSSRH_TOKEN
      - GPG_KEYNAME
      - GPG_PASSPHRASE
      - GPG_ENCPHRASE
      - SAMPLE_APPS_DEPLOY_KEY
      - VESPA_DEPLOY_KEY
      - DOCKER_IMAGE_DEPLOY_KEY
      - DOCKER_HUB_DEPLOY_KEY
      - GHCR_DEPLOY_KEY
      - SVC_OKTA_VESPA_FACTORY_TOKEN

    environment:
       GIT_SSH_COMMAND: "ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no"

    steps:
      - get-vespa-version: |
          JSON=$(screwdriver/factory-command.sh create-release)
          VESPA_VERSION=$(jq -re '.version' <<< "$JSON")
          JSON=$(screwdriver/factory-command.sh get-version $VESPA_VERSION)
          VESPA_REF=$(jq -re '.commits|.[]|select(.repo=="vespa")|.ref' <<< "$JSON")
          echo "VESPA_VERSION=$VESPA_VERSION VESPA_REF=$VESPA_REF"
          if [[ $VESPA_VERSION == null ]] || [[ $VESPA_REF == null ]]; then
            echo "Must have valid Vespa version and reference to continue (got VESPA_VERSION=$VESPA_VERSION, VESPA_REF=$VESPA_REF)."
            return 1
          fi
          meta set vespa.version $VESPA_VERSION
      - *install-podman-multi-arch
      - release-java-artifacts: |
          screwdriver/release-java-artifacts.sh $VESPA_VERSION $VESPA_REF
      - release-rpms: |
          screwdriver/release-rpms.sh $VESPA_VERSION $VESPA_REF
      - release-container-image: |
          screwdriver/release-container-image.sh $VESPA_VERSION
      - update-sample-apps: |
          screwdriver/update-vespa-version-in-sample-apps.sh $VESPA_VERSION
      - update-released-time: |
          screwdriver/factory-command.sh update-released-time

  publish-legacy-release:
    image: docker.io/vespaengine/vespa-build-centos-stream8:latest

    annotations:
      screwdriver.cd/cpu: 7
      screwdriver.cd/ram: 16
      screwdriver.cd/disk: HIGH
      screwdriver.cd/timeout: 300
      screwdriver.cd/buildPeriodically: H 6 1 * *

    environment:
      IMAGE_NAME: "vespaengine/vespa-generic-intel-x86_64"

    secrets:
      - DOCKER_HUB_DEPLOY_KEY

    steps:
      - get-vespa-version: |
          set -x
          VESPA_VERSION=$(meta get vespa.version --external publish-release)
          if [[ $VESPA_VERSION == null ]] || [[ $VESPA_REF == null ]]; then
            echo "Must have valid Vespa version to continue (got VESPA_VERSION=$VESPA_VERSION)."
            return 1
          fi
      - *install-podman-multi-arch
      - checkout: |
          mkdir -p workdir
          cd workdir
          export WORKDIR=$(pwd)
          git clone -q https://github.com/vespa-engine/vespa
          (cd vespa && git checkout v$VESPA_VERSION)
          git clone -q https://github.com/vespa-engine/system-test
          # Set correct version in pom.xml files
          (cd vespa && screwdriver/replace-vespa-version-in-poms.sh $VESPA_VERSION $(pwd) )
      - build-rpms: |
          cd $WORKDIR
          make -C $WORKDIR/vespa -f .copr/Makefile srpm outdir=$WORKDIR
          rpmbuild --rebuild \
                   --define="_topdir $WORKDIR/vespa-rpmbuild" \
                   --define "debug_package %{nil}" \
                   --define "_debugsource_template %{nil}" \
                   --define '_cmake_extra_opts "-DDEFAULT_VESPA_CPU_ARCH_FLAGS=-msse3 -mcx16 -mtune=intel"' \
                   *.src.rpm
          rm -f *.src.rpm
          mv $WORKDIR/vespa-rpmbuild/RPMS/x86_64/*.rpm .
      - build-container-image: |
          cat <<EOF > Dockerfile
          ARG VESPA_VERSION
          FROM docker.io/vespaengine/vespa:\$VESPA_VERSION
          USER root
          RUN --mount=type=bind,target=/rpms/,source=. dnf reinstall -y /rpms/vespa*rpm && dnf clean all
          USER vespa
          EOF
          buildah bud --security-opt label=disable --network host --squash --build-arg VESPA_VERSION=$VESPA_VERSION --tag docker.io/$IMAGE_NAME:$VESPA_VERSION \
            --tag docker.io/$IMAGE_NAME:latest --file Dockerfile .
      - verify-container-image: |
          # Trick to be able to use the documentation testing to verify the image built locally
          buildah tag docker.io/$IMAGE_NAME:$VESPA_VERSION vespaengine/vespa:latest
          # Run quick start guide
          $SD_SOURCE_DIR/screwdriver/test-quick-start-guide.sh
      - publish-image: |
          if [[ -z $SD_PULL_REQUEST ]]; then
            if curl -fsSL https://index.docker.io/v1/repositories/$IMAGE_NAME/tags/$VESPA_VERSION &> /dev/null; then
              echo "Container image docker.io/$IMAGE_NAME:$VESPA_VERSION aldready exists."
            else
              OPT_STATE="$(set +o)"
              set +x
              buildah login --username aressem --password "$DOCKER_HUB_DEPLOY_KEY" docker.io
              eval "$OPT_STATE"
              buildah push --format v2s2 docker.io/$IMAGE_NAME:$VESPA_VERSION docker://docker.io/$IMAGE_NAME:$VESPA_VERSION
              buildah push --format v2s2 docker.io/$IMAGE_NAME:$VESPA_VERSION docker://docker.io/$IMAGE_NAME:latest
            fi
          fi

  publish-el9-preview:
    image: docker.io/vespaengine/vespa-build-centos-stream8:latest
    annotations:
      screwdriver.cd/cpu: 7
      screwdriver.cd/ram: 16
      screwdriver.cd/disk: HIGH
      screwdriver.cd/timeout: 300
      screwdriver.cd/buildPeriodically: H 6 1 * *

    environment:
      IMAGE_NAME: "vespaengine/vespa-el9-preview"

    secrets:
      - DOCKER_HUB_DEPLOY_KEY

    steps:
      - get-vespa-version: |
          set -x
          VESPA_VERSION=$(meta get vespa.version --external publish-release)
          if [[ $VESPA_VERSION == null ]] || [[ $VESPA_REF == null ]]; then
            echo "Must have valid Vespa version to continue (got VESPA_VERSION=$VESPA_VERSION)."
            return 1
          fi
      - *install-podman-multi-arch
      - checkout: |
          git clone https://github.com/vespa-engine/docker-image
          cd docker-image
      - build-container-image: |
          buildah bud \
                --build-arg VESPA_BASE_IMAGE=el9 \
                --build-arg VESPA_VERSION=$VESPA_VERSION \
                --file Dockerfile \
                --jobs 2  \
                --layers=false \
                --manifest "$IMAGE_NAME:$VESPA_VERSION" \
                --platform linux/amd64,linux/arm64
      - verify-container-image: |
          # Trick to be able to use the documentation testing to verify the image built locally
          buildah tag $IMAGE_NAME:$VESPA_VERSION vespaengine/vespa:latest
          # Run quick start guide
          $SD_SOURCE_DIR/screwdriver/test-quick-start-guide.sh
      - publish-image: |
          if [[ -z $SD_PULL_REQUEST ]]; then
            if curl -fsSL https://index.docker.io/v1/repositories/$IMAGE_NAME/tags/$VESPA_VERSION &> /dev/null; then
              echo "Container image docker.io/$IMAGE_NAME:$VESPA_VERSION aldready exists."
            else
              OPT_STATE="$(set +o)"
              set +x
              buildah login --username aressem --password "$DOCKER_HUB_DEPLOY_KEY" docker.io
              eval "$OPT_STATE"
              buildah manifest push --all --format v2s2 $IMAGE_NAME:$VESPA_VERSION docker://docker.io/$IMAGE_NAME:$VESPA_VERSION
              buildah manifest push --all --format v2s2 $IMAGE_NAME:$VESPA_VERSION docker://docker.io/$IMAGE_NAME:$VESPA_MAJOR
              buildah manifest push --all --format v2s2 $IMAGE_NAME:$VESPA_VERSION docker://docker.io/$IMAGE_NAME:latest
            fi
          fi

  publish-cli-release:
    image: homebrew/brew:latest
    annotations:
      # Run once an hour, in the hours 7-15 UTC, Monday-Thursday
      screwdriver.cd/buildPeriodically: H 7-15 * * 1-4
    secrets:
      - HOMEBREW_GITHUB_API_TOKEN
      - GH_TOKEN
    steps:
      - install-dependencies: |
          export HOMEBREW_NO_INSTALL_CLEANUP=1 HOMEBREW_NO_ANALYTICS=1
          brew install --quiet gh zip go
      - publish-github:
          make -C client/go clean dist-github
      - publish-homebrew:
          make -C client/go clean dist-homebrew
      - verify-brew-install:
          make -C client/go install-brew

  verify-opensource-release-7days:
    annotations:
      screwdriver.cd/buildPeriodically: H 0 * * *
    steps:
      - now-epoch: |
          now_epoch=`date "+%s"`
          echo "Now epoch: " $now_epoch
      - calculate-current-release-age: |
          current_release_date=`curl -s 'https://repo1.maven.org/maven2/com/yahoo/vespa/parent/' | \
            grep '^<a href="[0-9]' | awk '{print $4}' | sort | tail -1`
          echo "Current release date: " $current_release_date
          current_release_epoch=`date -d "$current_release_date" "+%s"`
          echo "Current release epoch: " $current_release_epoch
          release_age_days=$((($now_epoch-$current_release_epoch)/86400))
          echo "Release age days: " $release_age_days
      - error-if-current-release-too-old: |
          if [ "$release_age_days" -gt 7 ]; then
            echo "Current open source release is older than 7 days"
            exit 1
          fi
      - calculate-docker-image-age: |
          image_date=`curl https://hub.docker.com/v2/repositories/vespaengine/vespa/ | \
            python -m json.tool| grep last_updated | awk '{print $2}' | tr -d '",'`
          echo "Docker image last_updated: " $image_date
          image_epoch=`date -d "$image_date" "+%s"`
          echo "Docker image epoch: " $image_epoch
          docker_image_age_days=$((($now_epoch-$image_epoch)/86400))
          echo "Docker image age days: " $docker_image_age_days
      - error-if-docker-image-too-old: |
          if [ "$docker_image_age_days" -gt 7 ]; then
            echo "Current Docker image is older than 7 days"
            exit 1
          fi

  verify-opensource-rpm-installable:
    image: quay.io/centos/centos:stream8
    annotations:
      screwdriver.cd/buildPeriodically: H 0 * * *
    steps:
      - install: |
          dnf install -y dnf-plugins-core
          dnf config-manager --add-repo https://raw.githubusercontent.com/vespa-engine/vespa/master/dist/vespa-engine.repo
          dnf config-manager --enable powertools
          dnf install -y epel-release
          dnf install -y vespa

  mirror-copr-rpms-to-artifactory:
    annotations:
      screwdriver.cd/cpu: LOW
      screwdriver.cd/ram: LOW
      screwdriver.cd/disk: HIGH
      screwdriver.cd/timeout: 60
      screwdriver.cd/buildPeriodically: H 6 * * *
    secrets:
      - JFROG_API_TOKEN
    steps:
      - mirror: |
          screwdriver/publish-unpublished-rpms-to-jfrog-cloud.sh

  delete-old-versions-on-artifactory:
    annotations:
      screwdriver.cd/cpu: LOW
      screwdriver.cd/ram: LOW
      screwdriver.cd/timeout: 10
      screwdriver.cd/buildPeriodically: H 6 * * 1
    secrets:
      - JFROG_API_TOKEN
    steps:
      - cleanup: |
          screwdriver/delete-old-artifactory-artifacts.sh