1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
|
# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
---
cache:
job:
build-vespa: [/main_job_cache]
component-build: [/main_job_cache]
shared:
image: vespaengine/vespa-build-centos7:latest
environment:
USER_SHELL_BIN: bash
annotations:
restore-cache: &restore-cache
restore-cache: |
(cd /tmp && if [[ -f $MAIN_CACHE_FILE ]]; then tar xf $MAIN_CACHE_FILE; fi)
mkdir -p $CCACHE_DATA_DIR
mkdir -p $CCACHE_TMP_DIR
export CCACHE_DIR=$CCACHE_DATA_DIR
rm -f $CCACHE_DIR/ccache.conf
ccache -M 20G
ccache -o log_file=$SD_ARTIFACTS_DIR/ccache_log.txt
ccache -o temporary_dir=$CCACHE_TMP_DIR
ccache -p
ccache -z
du -sh /tmp/*
save-cache: &save-cache
save-cache: |
ls -lart /tmp/vespa
du -sh /tmp/vespa/*
if [[ -z "$SD_PULL_REQUEST" ]]; then
# Remove what we have produced
rm -rf $LOCAL_MVN_REPO/com/yahoo
rm -rf $LOCAL_MVN_REPO/ai/vespa
# Tar toghether the /tmp/vespa folder containing ccache and cleaned mvn repo
mkdir -p $(dirname $MAIN_CACHE_FILE)
(cd /tmp && tar cf $MAIN_CACHE_FILE vespa)
# Wipe the cache if we exceed 2GB to avoid pulling and pusing too large files
if (( $(stat --format='%s' $MAIN_CACHE_FILE) > $(( 2200*1000*1000 )) )); then
tar cf $MAIN_CACHE_FILE --files-from=/dev/null;
echo "Cleaning cache file. $MAIN_CACHE_FILE is now $(stat --format='%s' $MAIN_CACHE_FILE) bytes."
fi
fi
jobs:
build-vespa:
requires: [~pr]
annotations:
screwdriver.cd/cpu: 7
screwdriver.cd/ram: 16
screwdriver.cd/disk: HIGH
screwdriver.cd/timeout: 60
screwdriver.cd/buildPeriodically: H 4 * * *
environment:
LOCAL_MVN_REPO: "/tmp/vespa/mvnrepo"
VESPA_MAVEN_EXTRA_OPTS: "-Dmaven.repo.local=/tmp/vespa/mvnrepo -Dmaven.source.skip=true"
CCACHE_TMP_DIR: "/tmp/ccache_tmp"
CCACHE_DATA_DIR: "/tmp/vespa/ccache"
MAIN_CACHE_FILE: "/main_job_cache/vespa.tar"
steps:
- inspect: |
set -x
env | grep -v TOKEN
cat /proc/cpuinfo
cat /proc/meminfo
df -h
uname -a
rpm -qa | grep "vespa"
- *restore-cache
- compile: |
screwdriver/build-vespa.sh
- *save-cache
- inspect-after: |
du -sh /tmp/vespa/*
ls -la /main_job_cache || true
df -h
component-build:
annotations:
screwdriver.cd/cpu: 7
screwdriver.cd/ram: 16
screwdriver.cd/disk: HIGH
screwdriver.cd/timeout: 90
screwdriver.cd/dockerEnabled: true
screwdriver.cd/dockerCpu: TURBO
screwdriver.cd/dockerRam: HIGH
environment:
LOCAL_MVN_REPO: "/tmp/vespa/mvnrepo"
VESPA_MAVEN_EXTRA_OPTS: "--show-version --batch-mode --no-snapshot-updates -Dmaven.repo.local=/tmp/vespa/mvnrepo"
CCACHE_TMP_DIR: "/tmp/ccache_tmp"
CCACHE_DATA_DIR: "/tmp/vespa/ccache"
MAIN_CACHE_FILE: "/main_job_cache/vespa.tar"
CCACHE_COMPRESS: "1"
secrets:
- DOCKER_HUB_DEPLOY_KEY
steps:
- get-vespa-version: |
VESPA_VERSION=$(meta get vespa.version)
VESPA_REF=$(meta get vespa.ref)
SYSTEM_TEST_REF=$(meta get system-test.ref)
if [[ $VESPA_VERSION == null ]] || [[ $VESPA_REF == null ]] || [[ $SYSTEM_TEST_REF == null ]]; then
echo "Must have valid Vespa version and reference to continue \
(got VESPA_VERSION=$VESPA_VERSION, VESPA_REF=$VESPA_REF, SYSTEM_TEST_REF=$SYSTEM_TEST_REF)."
exit 1
fi
- install-dependencies: |
yum-config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo
yum install -y --setopt skip_missing_names_on_install=False docker-ce docker-ce-cli containerd.io
docker system info
ln -sf $(which yum) /usr/local/bin/dnf
- checkout: |
mkdir -p workdir
cd workdir
WORKDIR=$(pwd)
git clone -q https://github.com/vespa-engine/vespa
(cd vespa && git checkout $VESPA_REF && git tag v$VESPA_VERSION)
git clone -q https://github.com/vespa-engine/system-test
(cd system-test && git checkout $SYSTEM_TEST_REF)
# Set correct version in pom.xml files
(cd vespa && \
find . -name "pom.xml" -exec sed -i'' \
-e "s,<version>.*SNAPSHOT.*</version>,<version>$VESPA_VERSION</version>," \
-e "s,<vespaversion>.*project.version.*</vespaversion>,<vespaversion>$VESPA_VERSION</vespaversion>," \
-e "s,<test-framework.version>.*project.version.*</test-framework.version>,<test-framework.version>$VESPA_VERSION</test-framework.version>," \
{} \;)
- make-srpm: |
make -C $WORKDIR/vespa -f .copr/Makefile srpm outdir=$WORKDIR
- *restore-cache
- build: |
cd $WORKDIR/vespa
export FACTORY_VESPA_VERSION=$VESPA_VERSION
NUM_THREADS=$(( $(nproc) + 2 ))
time make -C client/go BIN=$WORKDIR/vespa-install/opt/vespa/bin SHARE=$WORKDIR/vespa-install/usr/share install-all
time ./bootstrap.sh java
time mvn -T $NUM_THREADS $VESPA_MAVEN_EXTRA_OPTS install
cmake3 -DVESPA_UNPRIVILEGED=no .
time make -j ${NUM_THREADS}
time ctest3 --output-on-failure -j ${NUM_THREADS}
ccache --show-stats
time make -j ${NUM_THREADS} install DESTDIR=$WORKDIR/vespa-install
- verify-rpm-build: |
cd $WORKDIR
time rpmbuild --rebuild --define="_topdir $WORKDIR/vespa-rpmbuild" \
--define "_debugsource_template %{nil}" \
--define "installdir $WORKDIR/vespa-install" *.src.rpm
mkdir -p $WORKDIR/system-test/docker/rpms
mv $WORKDIR/vespa-rpmbuild/RPMS/x86_64/*.rpm $WORKDIR/system-test/docker/rpms
rm -f $WORKDIR/system-test/docker/rpms/*debuginfo*rpm
ls -la $WORKDIR/system-test/docker/rpms
- build-test-image: |
cd $WORKDIR/system-test
mkdir -p docker/vespa-systemtests
git archive HEAD --format tar | tar x -C docker/vespa-systemtests
cp -a $LOCAL_MVN_REPO docker/repository
cd docker
docker build --file Dockerfile.systemtest \
--build-arg VESPA_BASE_IMAGE=vespaengine/vespa-systemtest-base-centos7:latest \
--build-arg SYSTEMTEST_BASE_IMAGE=vespa --build-arg SKIP_M2_POPULATE=true \
--target systemtest \
--tag vespaengine/vespa-systemtest-centos7:$VESPA_VERSION \
--tag vespaengine/vespa-systemtest-centos7:latest .
- verify-test-image: |
docker run --rm -ti --entrypoint bash vespaengine/vespa-systemtest-centos7:$VESPA_VERSION -lc \
"source /opt/rh/rh-ruby27/enable; /opt/vespa-systemtests/lib/node_server.rb & sleep 3 && ruby /opt/vespa-systemtests/tests/search/basicsearch/basic_search.rb --run test_basicsearch__ELASTIC"
- publish-test-image: |
if [[ -z "$SD_PULL_REQUEST" ]]; then
OPT_STATE="$(set +o)"
set +x
docker login --username aressem --password "$DOCKER_HUB_DEPLOY_KEY"
eval "$OPT_STATE"
docker push docker.io/vespaengine/vespa-systemtest-centos7:$VESPA_VERSION
docker push docker.io/vespaengine/vespa-systemtest-centos7:latest
fi
- *save-cache
publish-release:
annotations:
screwdriver.cd/cpu: 7
screwdriver.cd/ram: 16
screwdriver.cd/disk: HIGH
screwdriver.cd/timeout: 300
screwdriver.cd/dockerEnabled: true
screwdriver.cd/dockerCpu: TURBO
screwdriver.cd/dockerRam: HIGH
secrets:
- COPR_WEBHOOK
- OSSRH_USER
- OSSRH_TOKEN
- GPG_KEYNAME
- GPG_PASSPHRASE
- GPG_ENCPHRASE
- SAMPLE_APPS_DEPLOY_KEY
- VESPA_DEPLOY_KEY
- DOCKER_HUB_DEPLOY_KEY
- GHCR_DEPLOY_KEY
environment:
GIT_SSH_COMMAND: "ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no"
steps:
- get-vespa-version: |
VESPA_VERSION=$(meta get vespa.version)
VESPA_REF=$(meta get vespa.ref)
if [[ $VESPA_VERSION == null ]] || [[ $VESPA_REF == null ]]; then
echo "Must have valid Vespa version and reference to continue (got VESPA_VERSION=$VESPA_VERSION, VESPA_REF=$VESPA_REF)."
exit 1
fi
- install-dependencies: |
yum-config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo
yum install -y --setopt skip_missing_names_on_install=False docker-ce docker-ce-cli containerd.io rh-git227
docker system info
source /opt/rh/rh-git227/enable
- release-rpms: |
screwdriver/release-rpms.sh $VESPA_VERSION $VESPA_REF
- release-container-image: |
screwdriver/release-container-image.sh $VESPA_VERSION
- release-java-artifacts: |
screwdriver/release-java-artifacts.sh $VESPA_VERSION $VESPA_REF
- update-sample-apps: |
screwdriver/update-vespa-version-in-sample-apps.sh $VESPA_VERSION
publish-cli-release:
image: homebrew/brew:latest
annotations:
# Run once an hour, in the hours 7-15 UTC, Monday-Thursday
screwdriver.cd/buildPeriodically: H 7-15 * * 1-4
secrets:
- HOMEBREW_GITHUB_API_TOKEN
- GH_TOKEN
steps:
- install-dependencies: |
export HOMEBREW_NO_INSTALL_CLEANUP=1 HOMEBREW_NO_ANALYTICS=1
brew install --quiet gh zip go
- publish-github:
make -C client/go clean dist-github
- publish-homebrew:
make -C client/go clean dist-homebrew
- verify-brew-install:
make -C client/go install-brew
verify-opensource-release-7days:
annotations:
screwdriver.cd/buildPeriodically: H 0 * * *
steps:
- now-epoch: |
now_epoch=`date "+%s"`
echo "Now epoch: " $now_epoch
- calculate-current-release-age: |
current_release_date=`curl -s 'https://repo1.maven.org/maven2/com/yahoo/vespa/parent/' | \
grep '^<a href="[0-9]' | awk '{print $4}' | sort | tail -1`
echo "Current release date: " $current_release_date
current_release_epoch=`date -d "$current_release_date" "+%s"`
echo "Current release epoch: " $current_release_epoch
release_age_days=$((($now_epoch-$current_release_epoch)/86400))
echo "Release age days: " $release_age_days
- error-if-current-release-too-old: |
if [ "$release_age_days" -gt 7 ]; then
echo "Current open source release is older than 7 days"
exit 1
fi
- calculate-docker-image-age: |
image_date=`curl https://hub.docker.com/v2/repositories/vespaengine/vespa/ | \
python -m json.tool| grep last_updated | awk '{print $2}' | tr -d '",'`
echo "Docker image last_updated: " $image_date
image_epoch=`date -d "$image_date" "+%s"`
echo "Docker image epoch: " $image_epoch
docker_image_age_days=$((($now_epoch-$image_epoch)/86400))
echo "Docker image age days: " $docker_image_age_days
- error-if-docker-image-too-old: |
if [ "$docker_image_age_days" -gt 7 ]; then
echo "Current Docker image is older than 7 days"
exit 1
fi
mirror-copr-rpms-to-artifactory:
annotations:
screwdriver.cd/cpu: LOW
screwdriver.cd/ram: LOW
screwdriver.cd/disk: HIGH
screwdriver.cd/timeout: 60
screwdriver.cd/buildPeriodically: H 6 * * *
secrets:
- JFROG_API_TOKEN
steps:
- mirror: |
screwdriver/publish-unpublished-rpms-to-jfrog-cloud.sh
delete-old-versions-on-artifactory:
annotations:
screwdriver.cd/cpu: LOW
screwdriver.cd/ram: LOW
screwdriver.cd/timeout: 10
screwdriver.cd/buildPeriodically: H 6 * * 1
secrets:
- JFROG_API_TOKEN
steps:
- cleanup: |
screwdriver/delete-old-artifactory-artifacts.sh
|