diff options
| -rw-r--r-- | screwdriver.yaml | 18 | ||||
| -rwxr-xr-x | screwdriver/publish-unpublished-rpms-to-archive.sh | 82 | ||||
| -rwxr-xr-x | screwdriver/upload-rpm-to-cloudsmith.sh | 36 |
3 files changed, 136 insertions, 0 deletions
diff --git a/screwdriver.yaml b/screwdriver.yaml index 7661f7043dc..e1ee1bcff8c 100644 --- a/screwdriver.yaml +++ b/screwdriver.yaml @@ -528,6 +528,24 @@ jobs: dnf install -y epel-release dnf install -y vespa + mirror-copr-rpms-to-archive: + image: quay.io/centos/centos:stream8 + annotations: + screwdriver.cd/cpu: LOW + screwdriver.cd/ram: LOW + screwdriver.cd/disk: HIGH + screwdriver.cd/timeout: 60 + screwdriver.cd/buildPeriodically: H 6 * * * + secrets: + - CLOUDSMITH_API_CREDS + steps: + - install: | + dnf install -y dnf-plugins-core jq + - mirror-x86-64: | + screwdriver/publish-unpublished-rpms-to-archive.sh x86_64 + - mirror-aarch64: | + screwdriver/publish-unpublished-rpms-to-archive.sh aarch64 + mirror-copr-rpms-to-artifactory: image: quay.io/centos/centos:stream8 annotations: diff --git a/screwdriver/publish-unpublished-rpms-to-archive.sh b/screwdriver/publish-unpublished-rpms-to-archive.sh new file mode 100755 index 00000000000..1e4b74b6f78 --- /dev/null +++ b/screwdriver/publish-unpublished-rpms-to-archive.sh @@ -0,0 +1,82 @@ +#!/bin/bash +# Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +set -euo pipefail +set -x + +if (( $# < 1 )); then + echo "Usage: $0 <RPM architecture>" + exit 1 +fi + +RPMARCH=$1 +ALLOWED_ARCHS=("x86_64" "aarch64") + +if [[ ! ${ALLOWED_ARCHS[@]} =~ $RPMARCH ]]; then + echo "Architecture $RPMARCH not in allowed archs: ${ALLOWED_ARCHS[@]}" + exit 1 +fi + +readonly MYDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +# Copr repo +dnf config-manager --add-repo https://copr.fedorainfracloud.org/coprs/g/vespa/vespa/repo/epel-8/group_vespa-vespa-epel-8.repo +sed -i "s,\$basearch,$RPMARCH,g" /etc/yum.repos.d/group_vespa-vespa-epel-8.repo + +# Cloudsmith repo +rpm --import 'https://dl.cloudsmith.io/public/vespa/vespa/gpg.0F3DA3C70D35DA7B.key' +curl -1sLf 'https://dl.cloudsmith.io/public/vespa/vespa/config.rpm.txt?distro=el&codename=8' > /tmp/vespa-vespa.repo +dnf config-manager --add-repo '/tmp/vespa-vespa.repo' +rm -f /tmp/vespa-vespa.repo + +readonly COPR_PACKAGES=$(mktemp) +trap "rm -f $COPR_PACKAGES" EXIT +readonly DLDIR=$(mktemp -d) +trap "rm -rf $DLDIR" EXIT + +cd $DLDIR + +readonly DNF="dnf -y -q --forcearch $RPMARCH" + +$DNF list --disablerepo='*' --enablerepo=copr:copr.fedorainfracloud.org:group_vespa:vespa --showduplicates 'vespa*' | grep "Available Packages" -A 100000 | tail -n +2 | sed '/\.src\ */d' | sed "s/\.$RPMARCH\ */-/" | awk '{print $1}' | grep -v '.src$' > $COPR_PACKAGES + +echo "Packages on Copr:" +cat $COPR_PACKAGES +echo + +for pv in $(cat $COPR_PACKAGES); do + if ! $DNF list --disablerepo='*' --enablerepo=vespa-* $pv &> /dev/null; then + echo "$pv not found on in archive. Downloading..." + $DNF download --disablerepo='*' --enablerepo=copr:copr.fedorainfracloud.org:group_vespa:vespa $pv + echo "$pv downloaded." + fi +done +echo + +if ! ls *.rpm &> /dev/null; then + echo "All packages already in archive." + exit 0 +fi + +echo "RPMs missing in archive:" +ls -lh *.rpm +echo + +UPLOAD_FAILED=false +if [[ -n $SCREWDRIVER ]] && [[ -z $SD_PULL_REQUEST ]]; then + for rpm in $(ls *.rpm); do + echo "Uploading $rpm ..." + if ! $MYDIR/upload-rpm-to-cloudsmith.sh $rpm ; then + echo "Could not upload $rpm" + UPLOAD_FAILED=true + else + echo "$rpm uploaded" + fi + done + echo +fi + +if $UPLOAD_FAILED; then + echo "Some RPMs failed to upload" + exit 1 +fi diff --git a/screwdriver/upload-rpm-to-cloudsmith.sh b/screwdriver/upload-rpm-to-cloudsmith.sh new file mode 100755 index 00000000000..dd24185433b --- /dev/null +++ b/screwdriver/upload-rpm-to-cloudsmith.sh @@ -0,0 +1,36 @@ +#!/bin/bash +# Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +set -euo pipefail + +if (( $# < 1 )); then + echo "Usage: $0 <RPM file>" + exit 1 +fi + +if [[ -z $CLOUDSMITH_API_CREDS ]]; then + echo "Environment CLOUDSMITH_API_CREDS not set. Exiting." + exit 1 +fi + +RPM=$1 +OS_DISTRO=el +RELEASEVER=8 + +main() { + + FID=$(curl -sLf \ + --upload-file $RPM \ + -u "$CLOUDSMITH_API_CREDS" \ + -H "Content-Sha256: $(sha256sum $RPM | cut -f1 -d' ')" \ + https://upload.cloudsmith.io/vespa/vespa/$RPM | jq -re '.identifier') + + if [[ -n $FID ]]; then + curl -sLf -X POST -H "Content-Type: application/json" \ + -u "$CLOUDSMITH_API_CREDS" \ + -d "{\"package_file\": \"$FID\", \"distribution\": \"$OS_DISTRO/$RELEASEVER\"}" \ + https://api-prd.cloudsmith.io/v1/packages/vespa/vespa/upload/rpm/ + fi +} + +main "$@" |