blob: bbffbd2ecde02d648e673f7016d05560a09929bb (
plain) (
blame)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
|
# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
---
cache:
job:
build-vespa: [/main_job_cache]
shared:
image: vespaengine/vespa-build-centos7:latest
environment:
USER_SHELL_BIN: bash
jobs:
build-vespa:
requires: [~pr, ~commit]
annotations:
screwdriver.cd/cpu: 7
screwdriver.cd/ram: 16
screwdriver.cd/disk: HIGH
screwdriver.cd/timeout: 60
environment:
LOCAL_MVN_REPO: "/tmp/vespa/mvnrepo"
VESPA_MAVEN_EXTRA_OPTS: "-Dmaven.repo.local=/tmp/vespa/mvnrepo -Dmaven.javadoc.skip=true -Dmaven.source.skip=true"
CCACHE_TMP_DIR: "/tmp/ccache_tmp"
CCACHE_DATA_DIR: "/tmp/vespa/ccache"
MAIN_CACHE_FILE: "/main_job_cache/vespa.tar"
steps:
- inspect: |
set -x
env | grep -v TOKEN
cat /proc/cpuinfo
cat /proc/meminfo
df -h
uname -a
rpm -qa | grep "vespa"
- restore-cache: |
(cd /tmp && if [[ -f $MAIN_CACHE_FILE ]]; then tar xf $MAIN_CACHE_FILE; fi)
mkdir -p $CCACHE_DATA_DIR
mkdir -p $CCACHE_TMP_DIR
export CCACHE_DIR=$CCACHE_DATA_DIR
rm -f $CCACHE_DIR/ccache.conf
ccache -M 20G
ccache -o log_file=$SD_ARTIFACTS_DIR/ccache_log.txt
ccache -o temporary_dir=$CCACHE_TMP_DIR
ccache -p
ccache -z
- compile: |
screwdriver/build-vespa.sh
- save-cache: |
if [[ -z "$SD_PULL_REQUEST" ]]; then
# Remove what we have produced
rm -rf $LOCAL_MVN_REPO/com/yahoo
rm -rf $LOCAL_MVN_REPO/ai/vespa
# Tar toghether the /tmp/vespa folder containing ccache and cleaned mvn repo
mkdir -p $(dirname $MAIN_CACHE_FILE)
(cd /tmp && tar cf $MAIN_CACHE_FILE vespa)
# Wipe the cache if we exceed 2GB to avoid pulling and pusing too large files
if (( $(stat --format='%s' $MAIN_CACHE_FILE) > $(( 2*1000*1000*1000 )) )); then
tar cf $MAIN_CACHE_FILE --files-from=/dev/null;
echo "Cleaning cache file. $MAIN_CACHE_FILE is now $(stat --format='%s' $MAIN_CACHE_FILE) bytes."
fi
fi
- inspect-after: |
du -sh /tmp/vespa/*
ls -la /main_job_cache || true
df -h
publish-release:
annotations:
screwdriver.cd/cpu: 7
screwdriver.cd/ram: 16
screwdriver.cd/disk: HIGH
screwdriver.cd/timeout: 300
screwdriver.cd/dockerEnabled: true
screwdriver.cd/dockerCpu: TURBO
screwdriver.cd/dockerRam: TURBO
secrets:
- COPR_WEBHOOK
- OSSRH_USER
- OSSRH_TOKEN
- GPG_KEYNAME
- GPG_PASSPHRASE
- GPG_ENCPHRASE
- SAMPLE_APPS_DEPLOY_KEY
- VESPA_DEPLOY_KEY
- DOCKER_HUB_DEPLOY_KEY
environment:
GIT_SSH_COMMAND: "ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no"
steps:
- get-vespa-version: |
VESPA_VERSION=$(meta get vespa.version)
VESPA_REF=$(meta get vespa.ref)
if [[ $VESPA_VERSION == null ]] || [[ $VESPA_REF == null ]]; then
echo "Must have valid Vespa version and reference to continue (got VESPA_VERSION=$VESPA_VERSION, VESPA_REF=$VESPA_REF)."
exit 1
fi
- install-dependencies: |
yum-config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo
yum install -y --setopt skip_missing_names_on_install=False docker-ce docker-ce-cli containerd.io rh-git227
docker system info
source /opt/rh/rh-git227/enable
- release-rpms: |
screwdriver/release-rpms.sh $VESPA_VERSION $VESPA_REF
- release-container-image: |
screwdriver/release-container-image.sh $VESPA_VERSION
- release-java-artifacts: |
screwdriver/release-java-artifacts.sh $VESPA_VERSION $VESPA_REF
- update-sample-apps: |
screwdriver/update-vespa-version-in-sample-apps.sh $VESPA_VERSION
verify-opensource-release-7days:
annotations:
screwdriver.cd/buildPeriodically: H 0 * * *
steps:
- now-epoch: |
now_epoch=`date "+%s"`
echo "Now epoch: " $now_epoch
- calculate-current-release-age: |
current_release_date=`curl -s 'https://repo1.maven.org/maven2/com/yahoo/vespa/parent/' | \
grep '^<a href="[0-9]' | awk '{print $4}' | sort | tail -1`
echo "Current release date: " $current_release_date
current_release_epoch=`date -d "$current_release_date" "+%s"`
echo "Current release epoch: " $current_release_epoch
release_age_days=$((($now_epoch-$current_release_epoch)/86400))
echo "Release age days: " $release_age_days
- error-if-current-release-too-old: |
if [ "$release_age_days" -gt 7 ]; then
echo "Current open source release is older than 7 days"
exit 1
fi
- calculate-docker-image-age: |
image_date=`curl https://hub.docker.com/v2/repositories/vespaengine/vespa/ | \
python -m json.tool| grep last_updated | awk '{print $2}' | tr -d '",'`
echo "Docker image last_updated: " $image_date
image_epoch=`date -d "$image_date" "+%s"`
echo "Docker image epoch: " $image_epoch
docker_image_age_days=$((($now_epoch-$image_epoch)/86400))
echo "Docker image age days: " $docker_image_age_days
- error-if-docker-image-too-old: |
if [ "$docker_image_age_days" -gt 7 ]; then
echo "Current Docker image is older than 7 days"
exit 1
fi
mirror-copr-rpms-to-artifactory:
annotations:
screwdriver.cd/cpu: LOW
screwdriver.cd/ram: LOW
screwdriver.cd/disk: HIGH
screwdriver.cd/timeout: 60
screwdriver.cd/buildPeriodically: H 6 * * *
secrets:
- JFROG_API_TOKEN
steps:
- mirror: |
screwdriver/publish-unpublished-rpms-to-jfrog-cloud.sh
delete-old-versions-on-artifactory:
annotations:
screwdriver.cd/cpu: LOW
screwdriver.cd/ram: LOW
screwdriver.cd/timeout: 10
screwdriver.cd/buildPeriodically: H 6 * * 1
secrets:
- JFROG_API_TOKEN
steps:
- cleanup: |
screwdriver/delete-old-artifactory-artifacts.sh
|
