Add release job for Vespa artifacts.

9 files changed, 255 insertions, 3 deletions

diff --git a/screwdriver.yaml b/screwdriver.yaml
index 4f62f1ef302..7cb66b9009d 100644
--- a/screwdriver.yaml
+++ b/screwdriver.yaml
@@ -6,6 +6,8 @@ cache:
 
 shared:
   image: vespaengine/vespa-build-centos7:latest
+  environment:
+    USER_SHELL_BIN: bash
 
 jobs:
   build-vespa:
@@ -17,7 +19,6 @@ jobs:
       screwdriver.cd/timeout: 60
 
     environment:
-      USER_SHELL_BIN: bash
       LOCAL_MVN_REPO: "/tmp/vespa/mvnrepo"
       VESPA_MAVEN_EXTRA_OPTS: "-Dmaven.repo.local=/tmp/vespa/mvnrepo -Dmaven.javadoc.skip=true -Dmaven.source.skip=true"
       CCACHE_TMP_DIR: "/tmp/ccache_tmp"
@@ -73,9 +74,49 @@ jobs:
           ls -la /main_job_cache || true
           df -h
 
+  publish-release:
+    annotations:
+      screwdriver.cd/cpu: 7
+      screwdriver.cd/ram: 16
+      screwdriver.cd/disk: HIGH
+      screwdriver.cd/timeout: 300
+      screwdriver.cd/dockerEnabled: true
+      screwdriver.cd/dockerCpu: TURBO
+      screwdriver.cd/dockerRam: TURBO
+
+    secrets:
+      - COPR_WEBHOOK
+      - OSSRH_USER
+      - OSSRH_TOKEN
+      - GPG_KEYNAME
+      - GPG_PASSPHRASE
+      - GPG_ENCPHRASE
+      - SAMPLE_APPS_DEPLOY_KEY
+      - VESPA_DEPLOY_KEY
+      - DOCKER_HUB_DEPLOY_KEY
+
+    steps:
+      - get-vespa-version: |
+          VESPA_VERSION=$(meta get vespa.version)
+          VESPA_REF=$(meta get vespa.ref)
+          if [[ $VESPA_VERSION == null ]] || [[ $VESPA_REF == null ]]; then
+            echo "Must have valid Vespa version and reference to continue (got VESPA_VERSION=$VESPA_VERSION, VESPA_REF=$VESPA_REF)."
+            exit 1
+          fi
+      - install-docker-client: |
+          yum-config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo
+          yum install -y --setopt skip_missing_names_on_install=False docker-ce docker-ce-cli containerd.io
+          docker system info
+      - release-rpms: |
+          screwdriver/release-rpms.sh $VESPA_VERSION $VESPA_REF
+      - release-container-image: |
+          screwdriver/release-container-image.sh $VESPA_VERSION
+      - release-java-artifacts: |
+          screwdriver/release-java-artifacts.sh $VESPA_VERSION $VESPA_REF
+      - update-sample-apps: |
+          screwdriver/update-vespa-version-in-sample-apps.sh $VESPA_VERSION
+
   verify-opensource-release-7days:
-    environment:
-      USER_SHELL_BIN: bash
     annotations:
       screwdriver.cd/buildPeriodically: H 0 * * *
     steps:
diff --git a/screwdriver/delete-old-artifactory-artifacts.sh b/screwdriver/delete-old-artifactory-artifacts.sh
index 431dbd221bf..a08a1479a5a 100755
--- a/screwdriver/delete-old-artifactory-artifacts.sh
+++ b/screwdriver/delete-old-artifactory-artifacts.sh
@@ -1,4 +1,6 @@
 #!/bin/bash
+# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
 
 set -euo pipefail
 
diff --git a/screwdriver/publish-unpublished-rpms-to-jfrog-cloud.sh b/screwdriver/publish-unpublished-rpms-to-jfrog-cloud.sh
index c200d347621..bea4cd46978 100755
--- a/screwdriver/publish-unpublished-rpms-to-jfrog-cloud.sh
+++ b/screwdriver/publish-unpublished-rpms-to-jfrog-cloud.sh
@@ -1,4 +1,6 @@
 #!/bin/bash
+# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
 
 set -euo pipefail
 
diff --git a/screwdriver/release-container-image.sh b/screwdriver/release-container-image.sh
new file mode 100755
index 00000000000..f3d345a3552
--- /dev/null
+++ b/screwdriver/release-container-image.sh
@@ -0,0 +1,34 @@
+#!/bin/bash
+# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+set -euo pipefail
+
+if [[ $# -ne 1 ]]; then
+    echo "Usage: $0 <Vespa version>"
+    exit 1
+fi
+
+readonly VESPA_VERSION=$1
+
+if curl -fsSL https://index.docker.io/v1/repositories/vespaengine/vespa/tags/$VESPA_VERSION &> /dev/null; then
+    echo "Container image docker.io/vespaengine/vespa:$VESPA_VERSION aldready exists."
+    exit 0
+fi
+
+if [[ -z "$DOCKER_HUB_DEPLOY_KEY" ]]; then
+    echo "Environment variable DOCKER_HUB_DEPLOY_KEY must be set, but is empty."
+    exit 1
+fi
+
+BUILD_DIR=$(mktemp -d)
+trap "rm -rf $BUILD_DIR" EXIT
+cd $BUILD_DIR
+
+git clone --depth 1 https://github.com/vespa-engine/docker-image
+cd docker-image
+
+docker login --username aressem --password "$DOCKER_HUB_DEPLOY_KEY"
+docker build --build-arg VESPA_VERSION=$VESPA_VERSION --tag docker.io/vespaengine/vespa:$VESPA_VERSION --tag docker.io/vespaengine/vespa:latest .
+
+docker push docker.io/vespaengine/vespa:$VESPA_VERSION
+docker push docker.io/vespaengine/vespa:latest
diff --git a/screwdriver/release-java-artifacts.sh b/screwdriver/release-java-artifacts.sh
new file mode 100755
index 00000000000..a91585d6ad5
--- /dev/null
+++ b/screwdriver/release-java-artifacts.sh
@@ -0,0 +1,67 @@
+#!/bin/bash
+# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+set -euo pipefail
+
+if [[ $# -ne 2 ]]; then
+    echo "Usage: $0 <Vespa version> <Git reference>"
+    exit 1
+fi
+
+if [[ -z $OSSRH_USER ]] || [[ -z $OSSRH_TOKEN ]]  || [[ -z $GPG_KEYNAME ]] || [[ -z $GPG_PASSPHRASE ]] || [[ -z $GPG_ENCPHRASE ]]; then
+    echo -e "The follwing env variables must be set:\n OSSRH_USER\n OSSRH_TOKEN\n GPG_KEYNAME\n GPG_PASSPHRASE\n GPG_ENCPHRASE"
+    exit 1
+fi
+
+readonly VESPA_RELEASE="$1"
+readonly VESPA_REF="$2"
+
+QUERY_VERSION_HTTP_CODE=$(curl --write-out %{http_code} --silent --location --output /dev/null https://oss.sonatype.org/content/repositories/releases/com/yahoo/vespa/parent/${VESPA_RELEASE}/)
+if [[ "200" == $QUERY_VERSION_HTTP_CODE ]]; then
+  echo "Vespa version $VESPA_RELEASE is already promoted, exiting"
+  exit 0
+fi
+
+export JAVA_HOME=$(dirname $(dirname $(readlink -f /usr/bin/java)))
+
+VESPA_DIR=vespa-clean
+git clone https://github.com/vespa-engine/vespa.git $VESPA_DIR
+
+cd $VESPA_DIR
+git checkout $VESPA_REF
+
+mkdir -p $SD_SOURCE_DIR/screwdriver/deploy
+openssl aes-256-cbc -pass pass:$GPG_ENCPHRASE -in $SD_SOURCE_DIR/screwdriver/pubring.gpg.enc -out $SD_SOURCE_DIR/screwdriver/deploy/pubring.gpg -d
+openssl aes-256-cbc -pass pass:$GPG_ENCPHRASE -in $SD_SOURCE_DIR/screwdriver/secring.gpg.enc -out $SD_SOURCE_DIR/screwdriver/deploy/secring.gpg -d
+
+# Build the Java code with the correct version set
+find . -name "pom.xml" -exec sed -i'' -e "s,<version>.*SNAPSHOT.*</version>,<version>$VESPA_RELEASE</version>," \
+     -e "s,<vespaversion>.*project.version.*</vespaversion>,<vespaversion>$VESPA_RELEASE</vespaversion>," \
+     -e "s,<test-framework.version>.*project.version.*</test-framework.version>,<test-framework.version>$VESPA_RELEASE</test-framework.version>," \
+     {} \;
+
+./bootstrap.sh
+
+COMMON_MAVEN_OPTS="--batch-mode --no-snapshot-updates --settings $(pwd)/screwdriver/settings-publish.xml --activate-profiles ossrh-deploy-vespa -DskipTests"
+TMPFILE=$(mktemp)
+mvn $COMMON_MAVEN_OPTS  -pl :container-dependency-versions -DskipStagingRepositoryClose=true deploy 2>&1 | tee $TMPFILE
+
+# Find the stage repo name
+STG_REPO=$(cat $TMPFILE | grep 'Staging repository at http' | head -1 | awk -F/ '{print $NF}')
+rm -f $TMPFILE
+
+# Deploy plugins
+mvn $COMMON_MAVEN_OPTS --file ./maven-plugins/pom.xml -DskipStagingRepositoryClose=true -DstagingRepositoryId=$STG_REPO deploy
+
+# Deploy the rest of the artifacts
+mvn $COMMON_MAVEN_OPTS --threads 1C -DskipStagingRepositoryClose=true -DstagingRepositoryId=$STG_REPO deploy
+
+# Close with checks
+mvn $COMMON_MAVEN_OPTS -N org.sonatype.plugins:nexus-staging-maven-plugin:rc-close -DnexusUrl=https://oss.sonatype.org/ -DserverId=ossrh -DstagingRepositoryId=$STG_REPO
+
+# Release if ok
+mvn $COMMON_MAVEN_OPTS -N org.sonatype.plugins:nexus-staging-maven-plugin:rc-release -DnexusUrl=https://oss.sonatype.org/ -DserverId=ossrh -DstagingRepositoryId=$STG_REPO
+
+# Delete the GPG rings
+rm -rf $SD_SOURCE_DIR/screwdriver/deploy
+
diff --git a/screwdriver/release-rpms.sh b/screwdriver/release-rpms.sh
new file mode 100755
index 00000000000..3346e0bef35
--- /dev/null
+++ b/screwdriver/release-rpms.sh
@@ -0,0 +1,38 @@
+#!/usr/bin/ssh-agent /bin/bash 
+# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+set -euo pipefail
+set -x
+
+if [ $# -ne 2 ]; then
+    echo "Usage: $0 <Vespa version> <Git reference>"
+    exit 1
+fi
+
+readonly VESPA_RELEASE="$1"
+readonly VESPA_REF="$2"
+
+VESPA_RPM=$(repoquery --repofrompath=vespa,https://copr-be.cloud.fedoraproject.org/results/@vespa/vespa/epel-7-x86_64 --repoid=vespa -q vespa | cut -d: -f2 | cut -d- -f1)
+echo "Latest RPM on Copr: $VESPA_RPM"
+
+if [ "$VESPA_RELEASE" == "$VESPA_RPM" ]; then
+  echo "Vespa rpm for version $VESPA_RELEASE already exists. Exiting."
+  exit 0
+fi
+
+echo "Using vespa repository git reference: $VESPA_REF"
+
+ssh-add -D
+set +x
+ssh-add <(echo $VESPA_DEPLOY_KEY | base64 -d)
+set -x
+git clone git@github.com:vespa-engine/vespa
+
+cd vespa
+dist/release-vespa-rpm.sh $VESPA_RELEASE $VESPA_REF
+
+while [ "$VESPA_RELEASE" != "$VESPA_RPM" ]; do
+  VESPA_RPM=$(repoquery --repofrompath=vespa,https://copr-be.cloud.fedoraproject.org/results/@vespa/vespa/epel-7-x86_64 --repoid=vespa -q vespa | cut -d: -f2 | cut -d- -f1)
+  echo "RPM: $VESPA_RPM"
+  sleep 150
+done
diff --git a/screwdriver/settings-publish.xml b/screwdriver/settings-publish.xml
index 33b6e48ea40..5524bf9d7ac 100644
--- a/screwdriver/settings-publish.xml
+++ b/screwdriver/settings-publish.xml
@@ -1,4 +1,6 @@
 <?xml version="1.0" encoding="UTF-8"?>
+<!-- Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
+
 <settings xmlns="http://maven.apache.org/SETTINGS/1.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/SETTINGS/1.0.0 http://maven.apache.org/xsd/settings-1.0.0.xsd">
     <servers>
         <server>
diff --git a/screwdriver/update-vespa-version-in-sample-apps.sh b/screwdriver/update-vespa-version-in-sample-apps.sh
new file mode 100755
index 00000000000..d20e8846716
--- /dev/null
+++ b/screwdriver/update-vespa-version-in-sample-apps.sh
@@ -0,0 +1,65 @@
+#!/usr/bin/ssh-agent /bin/bash
+# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+set -euo pipefail
+set -x
+
+if [[ $# -ne 1 ]]; then
+    echo "Usage: $0 <Vespa version>"
+    exit 1
+fi
+
+readonly VESPA_RELEASE="$1"
+readonly TRUE="true"
+readonly FALSE="false"
+
+export JAVA_HOME=$(dirname $(dirname $(readlink -f /usr/bin/java)))
+
+function is_published {
+    BUNDLE_PLUGIN_HTTP_CODE=$(curl --write-out %{http_code} --silent --location --output /dev/null https://repo.maven.apache.org/maven2/com/yahoo/vespa/bundle-plugin/${VESPA_RELEASE}/)
+    if [ $BUNDLE_PLUGIN_HTTP_CODE = "200" ] ; then
+        echo "$TRUE"
+    else
+        echo "$FALSE"
+    fi
+}
+
+function wait_until_published {
+    cnt=0
+    until [[ $(is_published) = "$TRUE" ]]; do
+        ((cnt+=1))
+        # Wait max 60 minutes
+        if (( $cnt > 60 )); then
+            echo "ERROR: Artifacts with version ${VESPA_RELEASE} not found on central maven repo."
+            exit 1
+        fi
+        echo "Waiting 60 seconds for version ${VESPA_RELEASE} on central maven repo ($cnt times)."
+        sleep 60
+    done
+}
+
+wait_until_published
+
+ssh-add -D
+set +x
+ssh-add <(echo $SAMPLE_APPS_DEPLOY_KEY | base64 -d)
+set -x
+
+# Supported from git 2.3
+export GIT_SSH_COMMAND="ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no"
+git --version
+
+git clone git@github.com:vespa-engine/sample-apps.git
+cd sample-apps
+
+# Update Vespa version property in pom.xml
+mvn versions:set-property -Dproperty=vespa_version -DnewVersion=${VESPA_RELEASE}
+
+changes=$(git status --porcelain | wc -l)
+
+if (( changes > 0 )); then
+    echo "Updating Vespa version to ${VESPA_RELEASE}."
+    git commit -a -m "Update Vespa version to ${VESPA_RELEASE}."
+    git pull --rebase
+    git push
+fi
diff --git a/screwdriver/upload-rpm-to-artifactory.sh b/screwdriver/upload-rpm-to-artifactory.sh
index 638f70806a8..21a38ffdb2f 100755
--- a/screwdriver/upload-rpm-to-artifactory.sh
+++ b/screwdriver/upload-rpm-to-artifactory.sh
@@ -1,4 +1,5 @@
 #!/bin/bash
+# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 
 set -euo pipefail