Add jobs to mirror RPMs and do repo cleanup.

3 files changed, 138 insertions, 0 deletions

diff --git a/screwdriver/delete-old-artifactory-artifacts.sh b/screwdriver/delete-old-artifactory-artifacts.sh
new file mode 100755
index 00000000000..062e37fdaa1
--- /dev/null
+++ b/screwdriver/delete-old-artifactory-artifacts.sh
@@ -0,0 +1,40 @@
+#!/bin/bash
+
+set -euo pipefail
+
+MAX_NUMBER_OF_RELEASES=200
+ARTIFACTORY_URL="https://artifactory.verizonmedia.com/artifactory"
+
+# JFrog Cloud repo file
+if [[ ! -f /etc/yum.repos.d/vespa.repo ]]; then
+    cat << EOF > /etc/yum.repos.d/vespa.repo
+[vespa-release]
+name=Vespa releases
+baseurl=$ARTIFACTORY_URL/vespa/centos/7/release/\$basearch
+gpgcheck=0
+enabled=1
+EOF
+fi
+
+VERSIONS_TO_DELETE=$(yum list --quiet --showduplicates --disablerepo='*' --enablerepo=vespa-release vespa | awk '/[0-9].*\.[0-9].*\.[0-9].*/{print $2}' | sort -V | head -n -200)
+
+RPMS_TO_DELETE=$(mktemp)
+trap "rm -f $RPMS_TO_DELETE" EXIT
+
+for VERSION in $VERSIONS_TO_DELETE; do
+    curl -sSL -H "content-type:text/plain" -H "Authorization: Bearer $JFROG_API_TOKEN" \
+         --data "items.find({ \"repo\": { \"\$eq\": \"vespa\" }, \"name\": {\"\$match\": \"vespa*$VERSION*\"}   }).include(\"repo\", \"path\", \"name\")" \
+         "$ARTIFACTORY_URL/api/search/aql" \
+        | jq -re ".results[]|\"$ARTIFACTORY_URL/artifactory/\(.repo)/\(.path)/\(.name)\"" >> $RPMS_TO_DELETE
+done
+
+echo "Deleting the following RPMs:"
+cat $RPMS_TO_DELETE
+
+if [[ -n $SCREWDRIVER ]] && [[ -z $SD_PULL_REQUEST ]]; then
+    for RPM in $(cat $RPMS_TO_DELETE); do
+        curl -sSL -H "Authorization: Bearer $JFROG_API_TOKEN" -X DELETE $RPM
+    done
+fi
+
+
diff --git a/screwdriver/publish-unpublished-rpms-to-jfrog-cloud.sh b/screwdriver/publish-unpublished-rpms-to-jfrog-cloud.sh
new file mode 100755
index 00000000000..c200d347621
--- /dev/null
+++ b/screwdriver/publish-unpublished-rpms-to-jfrog-cloud.sh
@@ -0,0 +1,72 @@
+#!/bin/bash
+
+set -euo pipefail
+
+readonly MYDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+yum install -y yum-utils &> /dev/null
+
+# Copr repo file
+if [[ ! -f /etc/yum.repos.d/group_vespa-vespa-epel-7.repo ]]; then
+  cat << 'EOF' > /etc/yum.repos.d/group_vespa-vespa-epel-7.repo
+[copr:copr.fedorainfracloud.org:group_vespa:vespa]
+name=Copr repo for vespa owned by @vespa
+baseurl=https://copr-be.cloud.fedoraproject.org/results/@vespa/vespa/epel-7-$basearch/
+type=rpm-md
+gpgcheck=1
+gpgkey=https://copr-be.cloud.fedoraproject.org/results/@vespa/vespa/pubkey.gpg
+repo_gpgcheck=0
+enabled=1
+EOF
+fi
+
+# JFrog Cloud repo file
+if [[ ! -f /etc/yum.repos.d/vespa.repo ]]; then
+  cat << 'EOF' > /etc/yum.repos.d/vespa.repo
+[vespa-release]
+name=Vespa releases
+baseurl=https://artifactory.verizonmedia.com/artifactory/vespa/centos/7/release/$basearch
+gpgcheck=0
+enabled=1
+EOF
+fi
+
+readonly COPR_PACKAGES=$(mktemp)
+trap "rm -f $COPR_PACKAGES" EXIT
+
+yum list -q --disablerepo='*' --enablerepo=copr:copr.fedorainfracloud.org:group_vespa:vespa --showduplicates 'vespa*' | grep "Available Packages" -A 100000 | tail -n +2 | sed "s/\.x86_64\ */-/"| awk '{print $1}' > $COPR_PACKAGES
+
+echo "Packages on Copr:"
+cat $COPR_PACKAGES
+echo 
+
+for pv in $(cat $COPR_PACKAGES); do
+  if ! yum list -q --disablerepo='*' --enablerepo=vespa-release $pv &> /dev/null; then
+    echo "$pv not found on JFrog Clould. Downloading..."
+    yumdownloader -q $pv
+    echo "$pv downloaded."
+  fi
+done
+echo
+
+if ! ls *.rpm &> /dev/null; then
+  echo "All packages already on JFrog Cloud."
+  exit 0
+fi
+
+echo "RPMs missing on JFrog Cloud:"
+ls -lh  *.rpm
+echo
+
+if [[ -n $SCREWDRIVER ]] && [[ -z $SD_PULL_REQUEST ]]; then
+  for rpm in $(ls *.rpm); do
+    echo "Uploading $rpm ..."
+    if ! $MYDIR/upload-rpm-to-artifactory.sh $rpm ; then
+      echo "Could not upload $rpm"
+    else
+      echo "$rpm uploaded"
+    fi
+  done
+  echo
+fi
+
diff --git a/screwdriver/upload-rpm-to-artifactory.sh b/screwdriver/upload-rpm-to-artifactory.sh
new file mode 100755
index 00000000000..638f70806a8
--- /dev/null
+++ b/screwdriver/upload-rpm-to-artifactory.sh
@@ -0,0 +1,26 @@
+#!/bin/bash
+
+set -euo pipefail
+
+RPM=$1
+OS_DISTRO=centos
+RELEASEVER=7
+MATURITY=release
+BASEARCH=x86_64
+
+main() {
+  if [[ -z $JFROG_API_TOKEN ]] || [[ -z $RPM ]]; then
+      echo "Usage: $0 <RPM package>."
+      echo "Environment variable JFROG_API_TOKEN must be set in environment."
+      exit 1
+  fi
+
+  curl -vL -H "Authorization: Bearer $JFROG_API_TOKEN" \
+           -H "X-Checksum-Sha1: $(sha1sum $RPM | awk '{print $1}')" \
+           -H "X-Checksum-Sha256: $(sha256sum $RPM | awk '{print $1}')" \
+           -H "X-Checksum-MD5: $(md5sum $RPM | awk '{print $1}')" \
+           -X PUT "https://artifactory.verizonmedia.com/artifactory/vespa/$OS_DISTRO/$RELEASEVER/$MATURITY/$BASEARCH/Packages/$RPM" \
+           -T $RPM
+}
+
+main "$@"