diff options
-rw-r--r-- | screwdriver.yaml | 25 | ||||
-rwxr-xr-x | screwdriver/delete-old-artifactory-artifacts.sh | 40 | ||||
-rwxr-xr-x | screwdriver/publish-unpublished-rpms-to-jfrog-cloud.sh | 72 | ||||
-rwxr-xr-x | screwdriver/upload-rpm-to-artifactory.sh | 26 |
4 files changed, 163 insertions, 0 deletions
diff --git a/screwdriver.yaml b/screwdriver.yaml index 3acb5b13867..0619974faef 100644 --- a/screwdriver.yaml +++ b/screwdriver.yaml @@ -107,3 +107,28 @@ jobs: echo "Current Docker image is older than 7 days" exit 1 fi + + mirror-copr-rpms-to-artifactory: + annotations: + screwdriver.cd/cpu: LOW + screwdriver.cd/ram: LOW + screwdriver.cd/disk: HIGH + screwdriver.cd/timeout: 60 + screwdriver.cd/buildPeriodically: H 6 * * * + secrets: + - JFROG_API_TOKEN + steps: + - mirror: | + screwdriver/publish-unpublished-rpms-to-jfrog-cloud.sh + + delete-old-versions-on-artifactory: + annotations: + screwdriver.cd/cpu: LOW + screwdriver.cd/ram: LOW + screwdriver.cd/timeout: 10 + screwdriver.cd/buildPeriodically: H 6 * * 1 + secrets: + - JFROG_API_TOKEN + steps: + - mirror: | + screwdriver/delete-old-artifactory-artifacts.sh diff --git a/screwdriver/delete-old-artifactory-artifacts.sh b/screwdriver/delete-old-artifactory-artifacts.sh new file mode 100755 index 00000000000..062e37fdaa1 --- /dev/null +++ b/screwdriver/delete-old-artifactory-artifacts.sh @@ -0,0 +1,40 @@ +#!/bin/bash + +set -euo pipefail + +MAX_NUMBER_OF_RELEASES=200 +ARTIFACTORY_URL="https://artifactory.verizonmedia.com/artifactory" + +# JFrog Cloud repo file +if [[ ! -f /etc/yum.repos.d/vespa.repo ]]; then + cat << EOF > /etc/yum.repos.d/vespa.repo +[vespa-release] +name=Vespa releases +baseurl=$ARTIFACTORY_URL/vespa/centos/7/release/\$basearch +gpgcheck=0 +enabled=1 +EOF +fi + +VERSIONS_TO_DELETE=$(yum list --quiet --showduplicates --disablerepo='*' --enablerepo=vespa-release vespa | awk '/[0-9].*\.[0-9].*\.[0-9].*/{print $2}' | sort -V | head -n -200) + +RPMS_TO_DELETE=$(mktemp) +trap "rm -f $RPMS_TO_DELETE" EXIT + +for VERSION in $VERSIONS_TO_DELETE; do + curl -sSL -H "content-type:text/plain" -H "Authorization: Bearer $JFROG_API_TOKEN" \ + --data "items.find({ \"repo\": { \"\$eq\": \"vespa\" }, \"name\": {\"\$match\": \"vespa*$VERSION*\"} }).include(\"repo\", \"path\", \"name\")" \ + "$ARTIFACTORY_URL/api/search/aql" \ + | jq -re ".results[]|\"$ARTIFACTORY_URL/artifactory/\(.repo)/\(.path)/\(.name)\"" >> $RPMS_TO_DELETE +done + +echo "Deleting the following RPMs:" +cat $RPMS_TO_DELETE + +if [[ -n $SCREWDRIVER ]] && [[ -z $SD_PULL_REQUEST ]]; then + for RPM in $(cat $RPMS_TO_DELETE); do + curl -sSL -H "Authorization: Bearer $JFROG_API_TOKEN" -X DELETE $RPM + done +fi + + diff --git a/screwdriver/publish-unpublished-rpms-to-jfrog-cloud.sh b/screwdriver/publish-unpublished-rpms-to-jfrog-cloud.sh new file mode 100755 index 00000000000..c200d347621 --- /dev/null +++ b/screwdriver/publish-unpublished-rpms-to-jfrog-cloud.sh @@ -0,0 +1,72 @@ +#!/bin/bash + +set -euo pipefail + +readonly MYDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +yum install -y yum-utils &> /dev/null + +# Copr repo file +if [[ ! -f /etc/yum.repos.d/group_vespa-vespa-epel-7.repo ]]; then + cat << 'EOF' > /etc/yum.repos.d/group_vespa-vespa-epel-7.repo +[copr:copr.fedorainfracloud.org:group_vespa:vespa] +name=Copr repo for vespa owned by @vespa +baseurl=https://copr-be.cloud.fedoraproject.org/results/@vespa/vespa/epel-7-$basearch/ +type=rpm-md +gpgcheck=1 +gpgkey=https://copr-be.cloud.fedoraproject.org/results/@vespa/vespa/pubkey.gpg +repo_gpgcheck=0 +enabled=1 +EOF +fi + +# JFrog Cloud repo file +if [[ ! -f /etc/yum.repos.d/vespa.repo ]]; then + cat << 'EOF' > /etc/yum.repos.d/vespa.repo +[vespa-release] +name=Vespa releases +baseurl=https://artifactory.verizonmedia.com/artifactory/vespa/centos/7/release/$basearch +gpgcheck=0 +enabled=1 +EOF +fi + +readonly COPR_PACKAGES=$(mktemp) +trap "rm -f $COPR_PACKAGES" EXIT + +yum list -q --disablerepo='*' --enablerepo=copr:copr.fedorainfracloud.org:group_vespa:vespa --showduplicates 'vespa*' | grep "Available Packages" -A 100000 | tail -n +2 | sed "s/\.x86_64\ */-/"| awk '{print $1}' > $COPR_PACKAGES + +echo "Packages on Copr:" +cat $COPR_PACKAGES +echo + +for pv in $(cat $COPR_PACKAGES); do + if ! yum list -q --disablerepo='*' --enablerepo=vespa-release $pv &> /dev/null; then + echo "$pv not found on JFrog Clould. Downloading..." + yumdownloader -q $pv + echo "$pv downloaded." + fi +done +echo + +if ! ls *.rpm &> /dev/null; then + echo "All packages already on JFrog Cloud." + exit 0 +fi + +echo "RPMs missing on JFrog Cloud:" +ls -lh *.rpm +echo + +if [[ -n $SCREWDRIVER ]] && [[ -z $SD_PULL_REQUEST ]]; then + for rpm in $(ls *.rpm); do + echo "Uploading $rpm ..." + if ! $MYDIR/upload-rpm-to-artifactory.sh $rpm ; then + echo "Could not upload $rpm" + else + echo "$rpm uploaded" + fi + done + echo +fi + diff --git a/screwdriver/upload-rpm-to-artifactory.sh b/screwdriver/upload-rpm-to-artifactory.sh new file mode 100755 index 00000000000..638f70806a8 --- /dev/null +++ b/screwdriver/upload-rpm-to-artifactory.sh @@ -0,0 +1,26 @@ +#!/bin/bash + +set -euo pipefail + +RPM=$1 +OS_DISTRO=centos +RELEASEVER=7 +MATURITY=release +BASEARCH=x86_64 + +main() { + if [[ -z $JFROG_API_TOKEN ]] || [[ -z $RPM ]]; then + echo "Usage: $0 <RPM package>." + echo "Environment variable JFROG_API_TOKEN must be set in environment." + exit 1 + fi + + curl -vL -H "Authorization: Bearer $JFROG_API_TOKEN" \ + -H "X-Checksum-Sha1: $(sha1sum $RPM | awk '{print $1}')" \ + -H "X-Checksum-Sha256: $(sha256sum $RPM | awk '{print $1}')" \ + -H "X-Checksum-MD5: $(md5sum $RPM | awk '{print $1}')" \ + -X PUT "https://artifactory.verizonmedia.com/artifactory/vespa/$OS_DISTRO/$RELEASEVER/$MATURITY/$BASEARCH/Packages/$RPM" \ + -T $RPM +} + +main "$@" |