diff options
author | Arnstein Ressem <aressem@yahooinc.com> | 2022-06-07 09:24:58 +0200 |
---|---|---|
committer | Arnstein Ressem <aressem@yahooinc.com> | 2022-06-08 11:41:30 +0200 |
commit | e68334191733421bd3772b224a81b1abbb080424 (patch) | |
tree | 9a6a5523c6fdbcf3a13f290455049e5418e6e817 | |
parent | 38e71d4979792c42b0d163268ad1335cf3176b37 (diff) |
Use CentOS Stream 8 on Vespa 8
-rw-r--r-- | dist/vespa.spec | 12 | ||||
-rw-r--r-- | screwdriver.yaml | 22 | ||||
-rwxr-xr-x | screwdriver/build-vespa.sh | 35 | ||||
-rwxr-xr-x | screwdriver/delete-old-artifactory-artifacts.sh | 4 | ||||
-rwxr-xr-x | screwdriver/publish-unpublished-rpms-to-jfrog-cloud.sh | 10 | ||||
-rwxr-xr-x | screwdriver/upload-rpm-to-artifactory.sh | 2 |
6 files changed, 43 insertions, 42 deletions
diff --git a/dist/vespa.spec b/dist/vespa.spec index 54fe37aad2d..d489a205133 100644 --- a/dist/vespa.spec +++ b/dist/vespa.spec @@ -226,9 +226,9 @@ BuildRequires: zlib-devel BuildRequires: libicu-devel %endif %if 0%{?el7} && 0%{?amzn2} -BuildRequires: java-11-amazon-corretto +BuildRequires: java-17-amazon-corretto %else -BuildRequires: java-11-openjdk-devel +BuildRequires: java-17-openjdk-devel %endif BuildRequires: rpm-build BuildRequires: make @@ -365,9 +365,9 @@ Vespa - The open big data serving engine Summary: Vespa - The open big data serving engine - base %if 0%{?el7} && 0%{?amzn2} -Requires: java-11-amazon-corretto +Requires: java-17-amazon-corretto %else -Requires: java-11-openjdk-devel +Requires: java-17-openjdk-devel %endif Requires: perl Requires: perl-Getopt-Long @@ -597,7 +597,7 @@ source %{_rhgit227_enable} || true %if 0%{?_java_home:1} export JAVA_HOME=%{?_java_home} %else -export JAVA_HOME=/usr/lib/jvm/java-11-openjdk +export JAVA_HOME=/usr/lib/jvm/java-17-openjdk %endif export PATH="$JAVA_HOME/bin:$PATH" export FACTORY_VESPA_VERSION=%{version} @@ -648,7 +648,7 @@ cp %{buildroot}/%{_prefix}/etc/systemd/system/vespa.service %{buildroot}/usr/lib cp %{buildroot}/%{_prefix}/etc/systemd/system/vespa-configserver.service %{buildroot}/usr/lib/systemd/system %endif -ln -s /usr/lib/jvm/jre-11-openjdk %{buildroot}/%{_prefix}/jdk +ln -s /usr/lib/jvm/jre-17-openjdk %{buildroot}/%{_prefix}/jdk %clean rm -rf $RPM_BUILD_ROOT diff --git a/screwdriver.yaml b/screwdriver.yaml index 937abc30613..0ffc3316f95 100644 --- a/screwdriver.yaml +++ b/screwdriver.yaml @@ -6,7 +6,7 @@ cache: component-build: [/main_job_cache] shared: - image: vespaengine/vespa-build-centos7:latest + image: vespaengine/vespa-build-centos-stream8:latest environment: USER_SHELL_BIN: bash annotations: @@ -113,10 +113,9 @@ jobs: exit 1 fi - install-dependencies: | - yum-config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo - yum install -y --setopt skip_missing_names_on_install=False docker-ce docker-ce-cli containerd.io + dnf config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo + dnf -y install docker-ce docker-ce-cli containerd.io docker system info - ln -sf $(which yum) /usr/local/bin/dnf - checkout: | mkdir -p workdir cd workdir @@ -149,6 +148,7 @@ jobs: time make -j ${NUM_THREADS} install DESTDIR=$WORKDIR/vespa-install - verify-rpm-build: | cd $WORKDIR + ulimit -c 0 time rpmbuild --rebuild --define="_topdir $WORKDIR/vespa-rpmbuild" \ --define "_debugsource_template %{nil}" \ --define "installdir $WORKDIR/vespa-install" *.src.rpm @@ -163,22 +163,22 @@ jobs: cp -a $LOCAL_MVN_REPO docker/repository cd docker docker build --file Dockerfile.systemtest \ - --build-arg VESPA_BASE_IMAGE=vespaengine/vespa-systemtest-base-centos7:latest \ + --build-arg VESPA_BASE_IMAGE=vespaengine/vespa-systemtest-base-centos-stream8:latest \ --build-arg SYSTEMTEST_BASE_IMAGE=vespa --build-arg SKIP_M2_POPULATE=true \ --target systemtest \ - --tag vespaengine/vespa-systemtest-centos7:$VESPA_VERSION \ - --tag vespaengine/vespa-systemtest-centos7:latest . + --tag vespaengine/vespa-systemtest-centos-stream8:$VESPA_VERSION \ + --tag vespaengine/vespa-systemtest-centos-stream8:latest . - verify-test-image: | - docker run --rm -ti --entrypoint bash vespaengine/vespa-systemtest-centos7:$VESPA_VERSION -lc \ - "source /opt/rh/rh-ruby27/enable; /opt/vespa-systemtests/lib/node_server.rb & sleep 3 && ruby /opt/vespa-systemtests/tests/search/basicsearch/basic_search.rb --run test_basicsearch__ELASTIC" + docker run --rm -ti --entrypoint bash vespaengine/vespa-systemtest-centos-stream8:$VESPA_VERSION -lc \ + "/opt/vespa-systemtests/lib/node_server.rb & sleep 3 && ruby /opt/vespa-systemtests/tests/search/basicsearch/basic_search.rb --run test_basicsearch__ELASTIC" - publish-test-image: | if [[ -z "$SD_PULL_REQUEST" ]]; then OPT_STATE="$(set +o)" set +x docker login --username aressem --password "$DOCKER_HUB_DEPLOY_KEY" eval "$OPT_STATE" - docker push docker.io/vespaengine/vespa-systemtest-centos7:$VESPA_VERSION - docker push docker.io/vespaengine/vespa-systemtest-centos7:latest + docker push docker.io/vespaengine/vespa-systemtest-centos-stream8:$VESPA_VERSION + docker push docker.io/vespaengine/vespa-systemtest-centos-stream8:latest fi - *save-cache diff --git a/screwdriver/build-vespa.sh b/screwdriver/build-vespa.sh index c809504bef5..ca9ada81205 100755 --- a/screwdriver/build-vespa.sh +++ b/screwdriver/build-vespa.sh @@ -6,9 +6,7 @@ set -e readonly SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd )" readonly NUM_THREADS=$(( $(nproc) + 2 )) -source /etc/profile.d/enable-devtoolset-11.sh -source /etc/profile.d/enable-rh-maven35.sh -source /etc/profile.d/enable-rh-git227.sh +source /etc/profile.d/enable-gcc-toolset-11.sh export MALLOC_ARENA_MAX=1 export MAVEN_OPTS="-Xss1m -Xms128m -Xmx2g" @@ -54,21 +52,24 @@ case $SHOULD_BUILD in ;; esac -if [[ $SHOULD_BUILD == systemtest ]]; then - yum -y --setopt=skip_missing_names_on_install=False install \ - zstd \ - devtoolset-11-gcc-c++ \ - devtoolset-11-libatomic-devel \ - devtoolset-11-binutils \ - libxml2-devel \ - rh-ruby27-rubygems-devel \ - rh-ruby27-ruby-devel \ - rh-ruby27 \ - rh-ruby27-rubygem-net-telnet - - source /opt/rh/rh-ruby27/enable - gem install libxml-ruby gnuplot distribution test-unit builder concurrent-ruby ffi +if [[ $SHOULD_BUILD == systemtest ]]; then + dnf module enable -y ruby:2.7 + dnf install -y \ + gcc-toolset-11-annobin \ + gcc-toolset-11-annobin-plugin-gcc \ + gcc-toolset-11-binutils \ + gcc-toolset-11-gcc-c++ \ + gcc-toolset-11-libatomic-devel \ + libxml2-devel \ + ruby \ + ruby-devel \ + rubygems-devel \ + rubygem-net-telnet \ + zstd + source /opt/rh/gcc-toolset-11/enable + gem install libxml-ruby gnuplot distribution test-unit builder concurrent-ruby bigdecimal ffi parallel + cd $HOME git clone https://github.com/vespa-engine/system-test export SYSTEM_TEST_DIR=$(pwd)/system-test diff --git a/screwdriver/delete-old-artifactory-artifacts.sh b/screwdriver/delete-old-artifactory-artifacts.sh index 9235575baff..80cee37bd72 100755 --- a/screwdriver/delete-old-artifactory-artifacts.sh +++ b/screwdriver/delete-old-artifactory-artifacts.sh @@ -12,13 +12,13 @@ if [[ ! -f /etc/yum.repos.d/vespa.repo ]]; then cat << EOF > /etc/yum.repos.d/vespa.repo [vespa-release] name=Vespa releases -baseurl=$ARTIFACTORY_URL/vespa/centos/7/release/\$basearch +baseurl=$ARTIFACTORY_URL/vespa/centos/8/release/\$basearch gpgcheck=0 enabled=1 EOF fi -VERSIONS_TO_DELETE=$(yum list --quiet --showduplicates --disablerepo='*' --enablerepo=vespa-release vespa | awk '/[0-9].*\.[0-9].*\.[0-9].*/{print $2}' | sort -V | head -n -200) +VERSIONS_TO_DELETE=$(dnf list --quiet --showduplicates --disablerepo='*' --enablerepo=vespa-release vespa | awk '/[0-9].*\.[0-9].*\.[0-9].*/{print $2}' | sort -V | head -n -200) RPMS_TO_DELETE=$(mktemp) trap "rm -f $RPMS_TO_DELETE" EXIT diff --git a/screwdriver/publish-unpublished-rpms-to-jfrog-cloud.sh b/screwdriver/publish-unpublished-rpms-to-jfrog-cloud.sh index dd0c953b424..3bcb4e101a5 100755 --- a/screwdriver/publish-unpublished-rpms-to-jfrog-cloud.sh +++ b/screwdriver/publish-unpublished-rpms-to-jfrog-cloud.sh @@ -13,7 +13,7 @@ if [[ ! -f /etc/yum.repos.d/group_vespa-vespa-epel-7.repo ]]; then cat << 'EOF' > /etc/yum.repos.d/group_vespa-vespa-epel-7.repo [copr:copr.fedorainfracloud.org:group_vespa:vespa] name=Copr repo for vespa owned by @vespa -baseurl=https://copr-be.cloud.fedoraproject.org/results/@vespa/vespa/epel-7-$basearch/ +baseurl=https://download.copr.fedorainfracloud.org/results/@vespa/vespa/centos-stream-8-$basearch/ type=rpm-md gpgcheck=1 gpgkey=https://copr-be.cloud.fedoraproject.org/results/@vespa/vespa/pubkey.gpg @@ -27,7 +27,7 @@ if [[ ! -f /etc/yum.repos.d/vespa.repo ]]; then cat << 'EOF' > /etc/yum.repos.d/vespa.repo [vespa-release] name=Vespa releases -baseurl=https://artifactory.yahooinc.com/artifactory/vespa/centos/7/release/$basearch +baseurl=https://artifactory.yahooinc.com/artifactory/vespa/centos/8/release/$basearch gpgcheck=0 enabled=1 EOF @@ -36,16 +36,16 @@ fi readonly COPR_PACKAGES=$(mktemp) trap "rm -f $COPR_PACKAGES" EXIT -yum list -q --disablerepo='*' --enablerepo=copr:copr.fedorainfracloud.org:group_vespa:vespa --showduplicates 'vespa*' | grep "Available Packages" -A 100000 | tail -n +2 | sed "s/\.x86_64\ */-/"| awk '{print $1}' > $COPR_PACKAGES +dnf list -q --disablerepo='*' --enablerepo=copr:copr.fedorainfracloud.org:group_vespa:vespa --showduplicates 'vespa*' | grep "Available Packages" -A 100000 | tail -n +2 | sed "s/\.x86_64\ */-/"| awk '{print $1}' > $COPR_PACKAGES echo "Packages on Copr:" cat $COPR_PACKAGES echo for pv in $(cat $COPR_PACKAGES); do - if ! yum list -q --disablerepo='*' --enablerepo=vespa-release $pv &> /dev/null; then + if ! dnf list -q --disablerepo='*' --enablerepo=vespa-release $pv &> /dev/null; then echo "$pv not found on JFrog Clould. Downloading..." - yumdownloader -q $pv + dnf download -q $pv echo "$pv downloaded." fi done diff --git a/screwdriver/upload-rpm-to-artifactory.sh b/screwdriver/upload-rpm-to-artifactory.sh index 9482bc2dc6a..0c679debb48 100755 --- a/screwdriver/upload-rpm-to-artifactory.sh +++ b/screwdriver/upload-rpm-to-artifactory.sh @@ -5,7 +5,7 @@ set -euo pipefail RPM=$1 OS_DISTRO=centos -RELEASEVER=7 +RELEASEVER=8 MATURITY=release BASEARCH=x86_64 |