From 72231250ed81e10d66bfe70701e64fa5fe50f712 Mon Sep 17 00:00:00 2001 From: Jon Bratseth Date: Wed, 15 Jun 2016 23:09:44 +0200 Subject: Publish --- vespabase/.gitignore | 1 + vespabase/CMakeLists.txt | 37 +++ vespabase/OWNERS | 2 + vespabase/README | 2 + vespabase/conf/services.conf | 6 + vespabase/desc.services | 14 ++ vespabase/desc.vespa_base | 37 +++ vespabase/desc.vespa_base_dev | 40 ++++ vespabase/src/.gitignore | 2 + vespabase/src/Defaults.pm | 59 +++++ vespabase/src/common-env.sh | 243 ++++++++++++++++++++ vespabase/src/defaultManifest.mf | 5 + vespabase/src/project.properties | 45 ++++ vespabase/src/rhel-prestart.sh | 119 ++++++++++ vespabase/src/start-cbinaries.sh | 255 +++++++++++++++++++++ vespabase/src/start-tool.sh | 65 ++++++ vespabase/src/start-vespa-base.sh | 76 ++++++ vespabase/src/stop-vespa-base.sh | 67 ++++++ vespabase/src/vespa-allow-downgrade-from-6-to-5.sh | 8 + vespabase/src/vespa-core-dumper.sh | 28 +++ vespabase/src/vespa-start-configserver.sh | 73 ++++++ vespabase/src/vespa-start-services.sh | 73 ++++++ vespabase/src/vespa-stop-configserver.sh | 69 ++++++ vespabase/src/vespa-stop-services.sh | 69 ++++++ 24 files changed, 1395 insertions(+) create mode 100644 vespabase/.gitignore create mode 100644 vespabase/CMakeLists.txt create mode 100644 vespabase/OWNERS create mode 100644 vespabase/README create mode 100644 vespabase/conf/services.conf create mode 100644 vespabase/desc.services create mode 100644 vespabase/desc.vespa_base create mode 100644 vespabase/desc.vespa_base_dev create mode 100644 vespabase/src/.gitignore create mode 100644 vespabase/src/Defaults.pm create mode 100755 vespabase/src/common-env.sh create mode 100755 vespabase/src/defaultManifest.mf create mode 100644 vespabase/src/project.properties create mode 100755 vespabase/src/rhel-prestart.sh create mode 100755 vespabase/src/start-cbinaries.sh create mode 100755 vespabase/src/start-tool.sh create mode 100755 vespabase/src/start-vespa-base.sh create mode 100755 vespabase/src/stop-vespa-base.sh create mode 100755 vespabase/src/vespa-allow-downgrade-from-6-to-5.sh create mode 100755 vespabase/src/vespa-core-dumper.sh create mode 100755 vespabase/src/vespa-start-configserver.sh create mode 100755 vespabase/src/vespa-start-services.sh create mode 100755 vespabase/src/vespa-stop-configserver.sh create mode 100755 vespabase/src/vespa-stop-services.sh (limited to 'vespabase') diff --git a/vespabase/.gitignore b/vespabase/.gitignore new file mode 100644 index 00000000000..f3c7a7c5da6 --- /dev/null +++ b/vespabase/.gitignore @@ -0,0 +1 @@ +Makefile diff --git a/vespabase/CMakeLists.txt b/vespabase/CMakeLists.txt new file mode 100644 index 00000000000..3519ad9274e --- /dev/null +++ b/vespabase/CMakeLists.txt @@ -0,0 +1,37 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_install_script(src/start-cbinaries.sh getvespaconfig bin) +vespa_install_script(src/start-cbinaries.sh vds-document-statistics bin) +vespa_install_script(src/start-cbinaries.sh vdsgetclusterstate bin) +vespa_install_script(src/start-cbinaries.sh vdsgetnodestate bin) +vespa_install_script(src/start-cbinaries.sh vdsgetsystemstate bin) +vespa_install_script(src/start-cbinaries.sh vdssetnodestate bin) +vespa_install_script(src/start-cbinaries.sh vdsstat bin) +vespa_install_script(src/start-cbinaries.sh verify_ranksetup bin) +vespa_install_script(src/start-cbinaries.sh vespa-config-status bin) +vespa_install_script(src/start-cbinaries.sh vespadoclocator bin) +vespa_install_script(src/start-cbinaries.sh vespafeeder-old bin) +vespa_install_script(src/start-cbinaries.sh vespaget bin) +vespa_install_script(src/start-cbinaries.sh vespa-model-inspect bin) +vespa_install_script(src/start-cbinaries.sh vespa-proton-cmd bin) +vespa_install_script(src/start-cbinaries.sh vespa-query-profile-dump-tool bin) +vespa_install_script(src/start-cbinaries.sh vesparoute bin) +vespa_install_script(src/start-cbinaries.sh vespa-transactionlog-inspect bin) +vespa_install_script(src/start-cbinaries.sh vespavisit bin) +vespa_install_script(src/start-cbinaries.sh vespavisittarget bin) +vespa_install_script(src/start-cbinaries.sh distributord sbin) +vespa_install_script(src/start-cbinaries.sh fdispatch sbin) +vespa_install_script(src/start-cbinaries.sh filedistributor sbin) +vespa_install_script(src/start-cbinaries.sh proton sbin) +vespa_install_script(src/start-cbinaries.sh storaged sbin) + +vespa_install_script(src/vespa-allow-downgrade-from-6-to-5.sh vespa-allow-downgrade-from-6-to-5 bin) +vespa_install_script(src/start-tool.sh vespa-start-tool.sh bin) + +vespa_install_script(src/rhel-prestart.sh vespa-prestart.sh bin) +vespa_install_script(src/common-env.sh common-env.sh libexec/vespa) +vespa_install_script(src/start-vespa-base.sh start-vespa-base.sh libexec/vespa) +vespa_install_script(src/stop-vespa-base.sh stop-vespa-base.sh libexec/vespa) +vespa_install_script(src/vespa-start-configserver.sh vespa-start-configserver bin) +vespa_install_script(src/vespa-start-services.sh vespa-start-services bin) +vespa_install_script(src/vespa-stop-configserver.sh vespa-stop-configserver bin) +vespa_install_script(src/vespa-stop-services.sh vespa-stop-services bin) diff --git a/vespabase/OWNERS b/vespabase/OWNERS new file mode 100644 index 00000000000..7028eebe31a --- /dev/null +++ b/vespabase/OWNERS @@ -0,0 +1,2 @@ +musum +arnej27959 diff --git a/vespabase/README b/vespabase/README new file mode 100644 index 00000000000..c74c1ce385f --- /dev/null +++ b/vespabase/README @@ -0,0 +1,2 @@ +This CVS module will hold some vespa-base related stuff, +such as monitoring and startup scripts. diff --git a/vespabase/conf/services.conf b/vespabase/conf/services.conf new file mode 100644 index 00000000000..dd0988e185a --- /dev/null +++ b/vespabase/conf/services.conf @@ -0,0 +1,6 @@ +#!/bin/sh +cat <) { + chomp; + my ( $action, $varname, $value ) = split(' ', $_, 3); + if ( $varname !~ m{^\w+$} ) { + # print STDERR "INVALID variable name '$varname' in $deffile, skipping\n"; + next; + } + if ( $action eq 'fallback' ) { + considerFallback($varname, $value); + } + } + close(DEFFILE); + } +} diff --git a/vespabase/src/common-env.sh b/vespabase/src/common-env.sh new file mode 100755 index 00000000000..cbc096dc1bf --- /dev/null +++ b/vespabase/src/common-env.sh @@ -0,0 +1,243 @@ +#! /bin/bash +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +# common environment setup for vespa scripts + +# put these two lines in all scripts: +# cd $VESPA_HOME || { echo "Cannot cd to $VESPA_HOME" 1>&2; exit 1; } # usually wanted +# . $VESPA_HOME/libexec/vespa/common-env.sh || exit 1 + +varhasvalue () { + # does the variable named in $1 have a (non-empty) value? + eval "oldvariablevalue=\${$1}" + [ "$oldvariablevalue" ] +} + +varisempty () { + # is the variable named in $1 unset, or does it have an empty value? + eval "oldvariablevalue=\${$1}" + [ -z "$oldvariablevalue" ] +} + +consider_fallback () { + if varhasvalue $1; then + : $1 already has value $oldvariablevalue + elif [ -z "${2}" ]; then + : proposed value "${2}" is empty + elif [ `expr match "$2" ".*'"` != 0 ]; then + : proposed value "${2}" contains a single-quote + else + eval "${1}='${2}'" + export ${1} + # echo "FALLBACK: ${1}='${2}'" + fi +} + +read_conf_file () { + deffile="$VESPA_HOME/conf/vespa/default-env.txt" + if [ -f "${deffile}" ]; then + eval $(perl -ne ' + chomp; + my ($action, $varname, $value) = split(" ", $_, 3); + $varname =~ s{[.]}{__}g; + if ($varname !~ m{^\w+}) { + # print STDERR "invalid var name $varname" + next; + } + if ($action eq "fallback" || $action eq "override") { + next if ($action eq "fallback" && $ENV{$varname} ne ""); + # quote value + if ($value !~ m{^["][^"]*["]$} ) { + $value =~ s{(\W)}{\\$1}g; + } + print "$varname=$value; export $varname; " + } + if ($action eq "unset") { + print "export -n $varname; unset $varname; " + } + ' < $deffile) + fi +} + +# Check if a variable is set. Plain environment precedes prefixed vars. +get_var() { + arg=$1 + ret_val=$2 + + env_var_name=`echo $arg | tr '[:lower:]' '[:upper:]'` + prefixed_var_name1=services__${arg} + prefixed_var_name2=vespa_base__${arg} + + if varhasvalue $env_var_name ; then eval "ret_val=\${$env_var_name}" + elif varhasvalue $prefixed_var_name1 ; then eval "ret_val=\${$prefixed_var_name1}" + elif varhasvalue $prefixed_var_name2 ; then eval "ret_val=\${$prefixed_var_name2}" + fi + echo "$ret_val" +} + +populate_environment () { + export VESPA_WEB_SERVICE_PORT=4080 + + # these are the variables we want while running vespa: + # VESPA_HOME - where is Vespa installed + # VESPA_CONFIGSERVERS - the host (or list of host) where a configserver runs + # VESPA_CONFIG_SOURCES - possible override as the first place to get config + + # network settings + # VESPA_PORT_BASE - the start of the port range where vespa services should listen + # VESPA_CONFIGSERVER_RPC_PORT - the RPC port for configservers + # VESPA_CONFIGSERVER_HTTP_PORT - the webservice (REST api) port for configservers + # VESPA_CONFIG_PROTOCOL_VERSION - the RPC protocol version to use + + # debugging + # VESPA_VALGRIND_OPT - for memory leak tracking, the options to use for valgrind + # VESPA_USE_VESPAMALLOC_D - use debugging version of malloc + # VESPA_USE_VESPAMALLOC_DST - use debugging and tracking version of malloc + + # optimizations + # VESPA_USE_HUGEPAGES_LIST - list of programs that should use huge pages or memory + # VESPA_USE_MADVISE_LIST - list of programs that should use madvise() + # VESPA_USE_VESPAMALLOC - list of programs that should preload an optimized malloc + # VESPA_USE_NO_VESPAMALLOC - list of programs that should use normal system malloc + + read_conf_file + consider_fallback ROOT ${VESPA_HOME} +} + +populate_environment + +PATH=$VESPA_HOME/bin64:$VESPA_HOME/bin:/usr/local/bin:/usr/X11R6/bin:/sbin:/bin:/usr/sbin:/usr/bin +export LD_LIBRARY_PATH=$VESPA_HOME/lib64 + +# how to find the "java" program? +# should be available in $VESPA_HOME/bin or JAVA_HOME +if [ "$JAVA_HOME" ] && [ -f "${JAVA_HOME}/bin/java" ]; then + PATH="${PATH}:${JAVA_HOME}/bin" +fi + +consider_fallback VESPA_VALGRIND_OPT "--num-callers=32 \ +--run-libc-freeres=yes \ +--track-origins=yes \ +--freelist-vol=1000000000 \ +--leak-check=full \ +--show-reachable=yes \ +--suppressions=${VESPA_HOME}etc/vespa/suppressions.txt" + +consider_fallback VESPA_USE_HUGEPAGES_LIST $(get_var "hugepages_list") +consider_fallback VESPA_USE_HUGEPAGES_LIST "all" + +consider_fallback VESPA_USE_MADVISE_LIST $(get_var "madvise_list") + +consider_fallback VESPA_USE_VESPAMALLOC $(get_var "vespamalloc_list") +consider_fallback VESPA_USE_VESPAMALLOC_D $(get_var "vespamallocd_list") +consider_fallback VESPA_USE_VESPAMALLOC_DST $(get_var "vespamallocdst_list") +consider_fallback VESPA_USE_NO_VESPAMALLOC $(get_var "no_vespamalloc_list") + +# TODO: +# if [ "$VESPA_USE_HUGEPAGES_LIST" = "all" ]; then +# # Can not enable this blindly as java will emit error if not available. THP must do then. +# export VESPA_USE_HUGEPAGES="yes" +# fi + +if [ -z "$YINST_RUNNING" ]; then + export VESPA_RUN_STANDALONE=true +fi + +fixlimits () { + # number of open files: + if varhasvalue file_descriptor_limit; then + ulimit -n ${file_descriptor_limit} || exit 1 + elif [ `ulimit -n` -lt 16384 ]; then + ulimit -n 16384 || exit 1 + fi + ulimit -c unlimited # core file size + ulimit -u 409600 # number of processes/threads +} + +checkjava () { + if java -version 2>&1 | grep "Java HotSpot.* 64-Bit Server VM" >/dev/null ; then + : OK + else + echo + echo "java must invoke the 64-bit Sun Java VM" + echo "Got:" + java -version + echo "Path: $PATH" + echo + exit 1 + fi +} + +runvalidation() { + run=$vespa_base__validationscript + if [ "$run" ]; then + if [ -x "$run" ]; then + if $run ; then + : OK + else + echo "validation script '$run' failed" + exit 1 + fi + else + echo "validation script '$run' must be executable" + exit 1 + fi + else + : no validation script + fi +} + +disable_vm_zone_reclaim_mode () { + # Should probably also be done on host. + dn=/proc/sys/vm/zone_reclaim_mode + if [ -w $dn ]; then + echo 0 > $dn + fi +} + +drop_caches () { + dn=/proc/sys/vm/drop_caches + if [ -w $dn ]; then + echo 3 > $dn + fi +} + +no_transparent_hugepages () { + # Should probably also be done on host. + dn=/sys/kernel/mm/redhat_transparent_hugepage + if [ -w $dn/enabled ]; then + echo always > $dn/enabled + fi + if [ -w $dn/defrag ]; then + echo never > $dn/defrag + fi + if [ -w $dn/khugepaged/defrag ]; then + echo yes > $dn/khugepaged/defrag + fi +} + +use_configserver_if_needed () { + nvcs=$(perl $VESPA_HOME/libexec/vespa/vespa-config.pl -allconfigsources) + x="$VESPA_CONFIG_SOURCES" + if [ "$x" ]; then + export VESPA_CONFIG_SOURCES="$x,$nvcs" + else + export VESPA_CONFIG_SOURCES="$nvcs" + fi +} + +getJavaOptionsIPV46() { + canon_ipv4=$(hostname | perl -pe 'chomp; ($_,$rest) = gethostbyname($_);') + if [ -z "${canon_ipv4}" ]; then + echo " -Djava.net.preferIPv6Addresses=true" + fi +} + +log_debug_message () { + if [ -n "$YINST_RUNNING" ]; then + echo "debug $*" 1>&2 + fi +} + +log_warning_message () { + echo "warning $*" 1>&2 +} diff --git a/vespabase/src/defaultManifest.mf b/vespabase/src/defaultManifest.mf new file mode 100755 index 00000000000..3f9d742b657 --- /dev/null +++ b/vespabase/src/defaultManifest.mf @@ -0,0 +1,5 @@ +Manifest-Version: 1.0 +Project-name: @PROJECT_NAME@ +Version: @VERSION@ +Creation-time: @NOW@ + diff --git a/vespabase/src/project.properties b/vespabase/src/project.properties new file mode 100644 index 00000000000..8f70e96d96a --- /dev/null +++ b/vespabase/src/project.properties @@ -0,0 +1,45 @@ +################################################################### +# PROJECT DIRECTORY DEFINITIONS +################################################################### +# +# This section defines the name and location of various directories +# in the project. +# +################################################################### + +#------------------------------------------------------------------ +# PROJECT_DIR defines the root directory of the project. If the +# value is set to the magic string "[DEFAULT]", the property will +# be automatically set to be the current directory (pwd) +# +PROJECT_DIR=[DEFAULT] + +#------------------------------------------------------------------ +# PROJECT_NAME defines the name of the project. If the value is set +# to the magic string "[DEFAULT]", the property will automatically +# be set to be the name of the project directory. +# +PROJECT_NAME=vespabase + +#------------------------------------------------------------------ +# DIST_DIR location of distribution files +# +DIST_DIR=build + +################################################################### +# VARIOUS PROJECT PROPS +################################################################### + +#------------------------------------------------------------------ +# VERSION - the current major version of the project +# +VERSION=0.1 +#------------------------------------------------------------------ +# BUILD_XML defines the location of the ant target definition file +# +BUILD_XML=$PROJECT_DIR/build.xml + +#------------------------------------------------------------------ +# JAVA_HOME defines the location of the JDK +# +JAVA_HOME=[DEFAULT] diff --git a/vespabase/src/rhel-prestart.sh b/vespabase/src/rhel-prestart.sh new file mode 100755 index 00000000000..761fd6af334 --- /dev/null +++ b/vespabase/src/rhel-prestart.sh @@ -0,0 +1,119 @@ +#!/bin/sh +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +# BEGIN environment bootstrap section +# Do not edit between here and END as this section should stay identical in all scripts + +findpath () { + myname=${0} + mypath=${myname%/*} + myname=${myname##*/} + if [ "$mypath" ] && [ -d "$mypath" ]; then + return + fi + mypath=$(pwd) + if [ -f "${mypath}/${myname}" ]; then + return + fi + echo "FATAL: Could not figure out the path where $myname lives from $0" + exit 1 +} + +COMMON_ENV=libexec/vespa/common-env.sh + +source_common_env () { + if [ "$VESPA_HOME" ] && [ -d "$VESPA_HOME" ]; then + # ensure it ends with "/" : + VESPA_HOME=${VESPA_HOME%/}/ + export VESPA_HOME + common_env=$VESPA_HOME/$COMMON_ENV + if [ -f "$common_env" ]; then + . $common_env + return + fi + fi + return 1 +} + +findroot () { + source_common_env && return + if [ "$VESPA_HOME" ]; then + echo "FATAL: bad VESPA_HOME value '$VESPA_HOME'" + exit 1 + fi + if [ "$ROOT" ] && [ -d "$ROOT" ]; then + VESPA_HOME="$ROOT" + source_common_env && return + fi + findpath + while [ "$mypath" ]; do + VESPA_HOME=${mypath} + source_common_env && return + mypath=${mypath%/*} + done + echo "FATAL: missing VESPA_HOME environment variable" + echo "Could not locate $COMMON_ENV anywhere" + exit 1 +} + +findroot + +# END environment bootstrap section + +[ "$VESPA_HOME" ] || { echo "Missing VESPA_HOME variable" 1>&2; exit 1; } + +cd $VESPA_HOME || { echo "Cannot cd to $VESPA_HOME" 1>&2; exit 1; } + +fixdir () { + if [ $# != 4 ]; then + echo "fixdir: Expected 4 params, got:" "$@" + exit 1 + fi + mkdir -p "$4" + chown $1 "$4" + chgrp $2 "$4" + chmod $3 "$4" +} + +# BEGIN directory fixups + +fixdir yahoo wheel 755 libdata/yell/exception +fixdir yahoo wheel 775 libexec/vespa/modelplugins +fixdir yahoo wheel 755 libexec/vespa/plugins/qrs +fixdir root wheel 1777 logs +fixdir yahoo wheel 1777 logs/vespa +fixdir yahoo wheel 755 logs/vespa/qrs +fixdir yahoo wheel 755 logs/vespa/search +fixdir root wheel 1777 tmp +fixdir yahoo wheel 1777 tmp/vespa +fixdir yahoo wheel 755 var/cache/vespa/config +fixdir yahoo wheel 1777 var/crash +fixdir yahoo wheel 755 var/db/vespa +fixdir yahoo wheel 755 var/db/vespa/config_server +fixdir yahoo wheel 755 var/db/vespa/config_server/serverdb +fixdir yahoo wheel 755 var/db/vespa/config_server/serverdb/applications +fixdir yahoo wheel 755 var/db/vespa/config_server/serverdb/configs +fixdir yahoo wheel 755 var/db/vespa/config_server/serverdb/configs/application +fixdir yahoo wheel 755 var/db/vespa/index +fixdir yahoo wheel 755 var/db/vespa/search +fixdir yahoo wheel 755 var/db/vespa/logcontrol +fixdir root wheel 1777 var/run + +chown -hR yahoo logs/vespa +chown -hR yahoo var/db/vespa + +# END directory fixups + +# Delete temporary files created by storage when running. +rm -f /home/y/tmp/hostinfo.*.*.report + +# Add $VESPA_HOME/bin to default path +perl -pi -e 'm=^pathmunge /usr/X11R6/bin after= and s=^=pathmunge /home/y/bin after; =' /etc/profile + +#Enable core files by default +perl -pi -e 's/^# No core files by default/# Vespa: Enable core files by default/' /etc/profile +perl -pi -e 's/^ulimit -S -c 0/ulimit -S -c unlimited/' /etc/profile + +# Don't fail script if this command fails. +# * sysctl will always return error on openvz jails +sysctl kernel.core_pattern="|/home/y/bin/vespa-core-dumper /home/y/bin64/lz4 /home/y/var/crash/%e.core.%p.lz4" || true diff --git a/vespabase/src/start-cbinaries.sh b/vespabase/src/start-cbinaries.sh new file mode 100755 index 00000000000..27a2b2cf4ae --- /dev/null +++ b/vespabase/src/start-cbinaries.sh @@ -0,0 +1,255 @@ +#!/bin/sh +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +# BEGIN environment bootstrap section +# Do not edit between here and END as this section should stay identical in all scripts + +findpath () { + myname=${0} + mypath=${myname%/*} + myname=${myname##*/} + if [ "$mypath" ] && [ -d "$mypath" ]; then + return + fi + mypath=$(pwd) + if [ -f "${mypath}/${myname}" ]; then + return + fi + echo "FATAL: Could not figure out the path where $myname lives from $0" + exit 1 +} + +COMMON_ENV=libexec/vespa/common-env.sh + +source_common_env () { + if [ "$VESPA_HOME" ] && [ -d "$VESPA_HOME" ]; then + # ensure it ends with "/" : + VESPA_HOME=${VESPA_HOME%/}/ + export VESPA_HOME + common_env=$VESPA_HOME/$COMMON_ENV + if [ -f "$common_env" ]; then + . $common_env + return + fi + fi + return 1 +} + +findroot () { + source_common_env && return + if [ "$VESPA_HOME" ]; then + echo "FATAL: bad VESPA_HOME value '$VESPA_HOME'" + exit 1 + fi + if [ "$ROOT" ] && [ -d "$ROOT" ]; then + VESPA_HOME="$ROOT" + source_common_env && return + fi + findpath + while [ "$mypath" ]; do + VESPA_HOME=${mypath} + source_common_env && return + mypath=${mypath%/*} + done + echo "FATAL: missing VESPA_HOME environment variable" + echo "Could not locate $COMMON_ENV anywhere" + exit 1 +} + +findroot + +# END environment bootstrap section + +bname=`basename $0` +no_valgrind=true +use_callgrind=false + +if [ "$bname" = "storaged" ] || [ "$bname" = "distributord" ]; then + memorySize=${vespa_storageserver__virtual_memory_size} + if [ "$memorySize" ]; then + log_debug_message "vespa_storageserver.virtual_memory_size set to $memorySize, running: ulimit -v $memorySize" + ulimit -v $memorySize + fi +fi + +if [ "$bname" = "filedistributor" ]; then + VESPA_USE_VALGRIND="" + VESPA_VALGRIND_OPT="" +fi + +if [ "$bname" = "vespafeeder-old" ] || [ "$bname" = "vdsfeeder" ]; then + export VESPA_LOG_LEVEL="all -spam -debug -config -info -event" +fi + +case $VESPA_VALGRIND_OPT in + *callgrind*) use_callgrind=true;; +esac + +for f in $VESPA_USE_VALGRIND +do + # log_debug_message "Testing '$f'" + if [ "$f" = "$bname" ]; then + # log_debug_message "I ($bname) shall use valgrind, as set in VESPA_USE_VALGRIND" + no_valgrind=false + break + fi +done + +if [ "$VESPA_USE_VALGRIND" = "all" ]; then + # log_debug_message "I shall use valgrind, as VESPA_USE_VALGRIND is 'all'" + no_valgrind=false +fi + +# special malloc setup; we should make some better mechanism for this +# +export GLIBCXX_FORCE_NEW=1 +# +unset LD_PRELOAD +p64=/home/y/lib64 +p32=/home/y/lib32 +pre=/home/y/lib +suf=vespa/malloc/libvespamalloc.so +# +# should really check that format is same as binary below +# +for f in $VESPA_USE_HUGEPAGES_LIST +do + # log_debug_message "Testing '$f'" + if [ "$f" = "$bname" ]; then + # log_debug_message "I ($bname) shall use huge pages, as set in VESPA_USE_HUGEPAGES_LIST" + export VESPA_USE_HUGEPAGES="yes" + break + fi +done + +if [ "$VESPA_USE_HUGEPAGES_LIST" = "all" ]; then + # log_debug_message "I shall use huge pages, as VESPA_USE_HUGEPAGES_LIST is 'all'" + export VESPA_USE_HUGEPAGES="yes" +fi + +for f in $VESPA_USE_MADVISE_LIST +do + # log_debug_message "Testing '$f'" + app=`echo $f | cut -d '=' -f1` + limit=`echo $f | cut -d '=' -f2` + if [ "$app" = "$bname" ]; then + log_debug_message "I ($bname) shall use madvise with limit $limit, as set in VESPA_USE_MADVISE_LIST" + export VESPA_MALLOC_MADVISE_LIMIT="$limit" + break + fi + + if [ "$app" = "all" ]; then + log_debug_message "I shall use madvise with limit $limit, as VESPA_USE_MADVISE_LIST is 'all'" + export VESPA_MALLOC_MADVISE_LIMIT="$limit" + fi +done + +for f in $VESPA_USE_NO_VESPAMALLOC +do + # log_debug_message "Testing '$f'" + if [ "$f" = "$bname" ]; then + # log_debug_message "I ($bname) shall not use vespamalloc, as set in VESPA_USE_NO_VESPAMALLOC" + suf=vespa/malloc/libvespamalloc_notexisting.so + break + fi +done + +if [ "$VESPA_USE_NO_VESPAMALLOC" = "all" ]; then + # log_debug_message "I shall not use, as VESPA_USE_NO_VESPAMALLOC is 'all'" + suf=vespa/malloc/libvespamalloc_notexisting.so +fi + +for f in $VESPA_USE_VESPAMALLOC +do + # log_debug_message "Testing '$f'" + if [ "$f" = "$bname" ]; then + # log_debug_message "I ($bname) shall use libvespamalloc, as set in VESPA_USE_VESPAMALLOC" + suf=vespa/malloc/libvespamalloc.so + break + fi +done + +if [ "$VESPA_USE_VESPAMALLOC" = "all" ]; then + # log_debug_message "I shall use libvespamalloc, as VESPA_USE_VESPAMALLOC is 'all'" + suf=vespa/malloc/libvespamalloc.so +fi + +for f in $VESPA_USE_VESPAMALLOC_D +do + # log_debug_message "Testing '$f'" + if [ "$f" = "$bname" ]; then + # log_debug_message "I ($bname) shall use libvespamallocd, as set in VESPA_USE_VESPAMALLOC_D" + suf=vespa/malloc/libvespamallocd.so + break + fi +done + +if [ "$VESPA_USE_VESPAMALLOC_D" = "all" ]; then + # log_debug_message "I shall use libvespamallocd, as VESPA_USE_VESPAMALLOC_D is 'all'" + suf=vespa/malloc/libvespamallocd.so +fi + +for f in $VESPA_USE_VESPAMALLOC_DST +do + # log_debug_message "Testing '$f'" + if [ "$f" = "$bname" ]; then + # log_debug_message "I ($bname) shall use libvespamallocdst16, as set in VESPA_USE_VESPAMALLOC_DST" + suf=vespa/malloc/libvespamallocdst16.so + break + fi +done + +if [ "$VESPA_USE_VESPAMALLOC_DST" = "all" ]; then + # log_debug_message "I shall use libvespamallocdst16, as VESPA_USE_VESPAMALLOC_DST is 'all'" + suf=vespa/malloc/libvespamallocdst16.so +fi + +if [[ ($no_valgrind || $use_callgrind) && -z $VESPA_RUN_STANDALONE ]]; then + for tryfile in $p64/$suf $p32/$suf $pre/$suf ; do + if [ -f $tryfile ]; then + LD_PRELOAD=$tryfile + log_debug_message "Using LD_PRELOAD=$tryfile" + if [ "$VESPA_USE_HUGEPAGES" ]; then + export VESPA_MALLOC_HUGEPAGES="$VESPA_USE_HUGEPAGES" + log_debug_message "enabling hugepages for '$0-bin'." + fi + break + fi + done +fi + +# log_debug_message "VESPA_USE_VALGRIND='$VESPA_USE_VALGRIND'; bname='$bname'; no_valgrind='$no_valgrind'" + +if $no_valgrind || ! which valgrind >/dev/null ; then + # log_debug_message $0-bin $@ + if [ -z $VESPA_RUN_STANDALONE ]; then + ulimit -c unlimited + fi + if numactl --interleave all true &> /dev/null; then + # We are allowed to use numactl + if [ "$VESPA_AFFINITY_CPU_SOCKET" ]; then + numcpu=`numactl --hardware | grep available | cut -d' ' -f2` + node=$(($VESPA_AFFINITY_CPU_SOCKET % $numcpu)) + log_debug_message "Starting $0 with affinity on cpu $VESPA_AFFINITY_CPU_SOCKET out of $numcpu : " \ + numactl --cpunodebind=$node --membind=$node env LD_PRELOAD=$LD_PRELOAD $0-bin "$@" + exec numactl --cpunodebind=$node --membind=$node env LD_PRELOAD=$LD_PRELOAD $0-bin "$@" + else + log_debug_message "Starting $0 with memory interleaving on all nodes : " \ + numactl --interleave all env LD_PRELOAD=$LD_PRELOAD $0-bin "$@" + exec numactl --interleave all env LD_PRELOAD=$LD_PRELOAD $0-bin "$@" + fi + else + log_debug_message "Starting $0 without numactl (no permission or not available) : " \ + env LD_PRELOAD=$LD_PRELOAD $0-bin "$@" + exec env LD_PRELOAD=$LD_PRELOAD $0-bin "$@" + fi +else + if $use_callgrind ; then + export LD_PRELOAD + else + export LD_PRELOAD="" + fi + log_debug_message "Starting : " \ + valgrind $VESPA_VALGRIND_OPT --log-file=/home/y/tmp/valgrind.$bname.log.$$ $0-bin "$@" + exec valgrind $VESPA_VALGRIND_OPT --log-file=/home/y/tmp/valgrind.$bname.log.$$ $0-bin "$@" +fi diff --git a/vespabase/src/start-tool.sh b/vespabase/src/start-tool.sh new file mode 100755 index 00000000000..73ec408ad7d --- /dev/null +++ b/vespabase/src/start-tool.sh @@ -0,0 +1,65 @@ +#!/bin/sh +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +# BEGIN environment bootstrap section +# Do not edit between here and END as this section should stay identical in all scripts + +findpath () { + myname=${0} + mypath=${myname%/*} + myname=${myname##*/} + if [ "$mypath" ] && [ -d "$mypath" ]; then + return + fi + mypath=$(pwd) + if [ -f "${mypath}/${myname}" ]; then + return + fi + echo "FATAL: Could not figure out the path where $myname lives from $0" + exit 1 +} + +COMMON_ENV=libexec/vespa/common-env.sh + +source_common_env () { + if [ "$VESPA_HOME" ] && [ -d "$VESPA_HOME" ]; then + # ensure it ends with "/" : + VESPA_HOME=${VESPA_HOME%/}/ + export VESPA_HOME + common_env=$VESPA_HOME/$COMMON_ENV + if [ -f "$common_env" ]; then + . $common_env + return + fi + fi + return 1 +} + +findroot () { + source_common_env && return + if [ "$VESPA_HOME" ]; then + echo "FATAL: bad VESPA_HOME value '$VESPA_HOME'" + exit 1 + fi + if [ "$ROOT" ] && [ -d "$ROOT" ]; then + VESPA_HOME="$ROOT" + source_common_env && return + fi + findpath + while [ "$mypath" ]; do + VESPA_HOME=${mypath} + source_common_env && return + mypath=${mypath%/*} + done + echo "FATAL: missing VESPA_HOME environment variable" + echo "Could not locate $COMMON_ENV anywhere" + exit 1 +} + +findroot + +# END environment bootstrap section + +use_configserver_if_needed + +exec $0-bin "$@" diff --git a/vespabase/src/start-vespa-base.sh b/vespabase/src/start-vespa-base.sh new file mode 100755 index 00000000000..ad6b58f61b1 --- /dev/null +++ b/vespabase/src/start-vespa-base.sh @@ -0,0 +1,76 @@ +#!/bin/sh +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +# BEGIN environment bootstrap section +# Do not edit between here and END as this section should stay identical in all scripts + +findpath () { + myname=${0} + mypath=${myname%/*} + myname=${myname##*/} + if [ "$mypath" ] && [ -d "$mypath" ]; then + return + fi + mypath=$(pwd) + if [ -f "${mypath}/${myname}" ]; then + return + fi + echo "FATAL: Could not figure out the path where $myname lives from $0" + exit 1 +} + +COMMON_ENV=libexec/vespa/common-env.sh + +source_common_env () { + if [ "$VESPA_HOME" ] && [ -d "$VESPA_HOME" ]; then + # ensure it ends with "/" : + VESPA_HOME=${VESPA_HOME%/}/ + export VESPA_HOME + common_env=$VESPA_HOME/$COMMON_ENV + if [ -f "$common_env" ]; then + . $common_env + return + fi + fi + return 1 +} + +findroot () { + source_common_env && return + if [ "$VESPA_HOME" ]; then + echo "FATAL: bad VESPA_HOME value '$VESPA_HOME'" + exit 1 + fi + if [ "$ROOT" ] && [ -d "$ROOT" ]; then + VESPA_HOME="$ROOT" + source_common_env && return + fi + findpath + while [ "$mypath" ]; do + VESPA_HOME=${mypath} + source_common_env && return + mypath=${mypath%/*} + done + echo "FATAL: missing VESPA_HOME environment variable" + echo "Could not locate $COMMON_ENV anywhere" + exit 1 +} + +findroot + +# END environment bootstrap section + +cd $VESPA_HOME || { echo "Cannot cd to $VESPA_HOME" 1>&2; exit 1; } + +fixlimits +checkjava +runvalidation +no_transparent_hugepages +disable_vm_zone_reclaim_mode +drop_caches + +if [ -f /tmp/.ylock-arena.shm ]; then + chown yahoo:wheel /tmp/.ylock-arena.shm +fi + +exec run-as-yahoo ${VESPA_HOME}/bin/config-ctl start diff --git a/vespabase/src/stop-vespa-base.sh b/vespabase/src/stop-vespa-base.sh new file mode 100755 index 00000000000..6e2fccc5664 --- /dev/null +++ b/vespabase/src/stop-vespa-base.sh @@ -0,0 +1,67 @@ +#!/bin/sh +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +# BEGIN environment bootstrap section +# Do not edit between here and END as this section should stay identical in all scripts + +findpath () { + myname=${0} + mypath=${myname%/*} + myname=${myname##*/} + if [ "$mypath" ] && [ -d "$mypath" ]; then + return + fi + mypath=$(pwd) + if [ -f "${mypath}/${myname}" ]; then + return + fi + echo "FATAL: Could not figure out the path where $myname lives from $0" + exit 1 +} + +COMMON_ENV=libexec/vespa/common-env.sh + +source_common_env () { + if [ "$VESPA_HOME" ] && [ -d "$VESPA_HOME" ]; then + # ensure it ends with "/" : + VESPA_HOME=${VESPA_HOME%/}/ + export VESPA_HOME + common_env=$VESPA_HOME/$COMMON_ENV + if [ -f "$common_env" ]; then + . $common_env + return + fi + fi + return 1 +} + +findroot () { + source_common_env && return + if [ "$VESPA_HOME" ]; then + echo "FATAL: bad VESPA_HOME value '$VESPA_HOME'" + exit 1 + fi + if [ "$ROOT" ] && [ -d "$ROOT" ]; then + VESPA_HOME="$ROOT" + source_common_env && return + fi + findpath + while [ "$mypath" ]; do + VESPA_HOME=${mypath} + source_common_env && return + mypath=${mypath%/*} + done + echo "FATAL: missing VESPA_HOME environment variable" + echo "Could not locate $COMMON_ENV anywhere" + exit 1 +} + +findroot + +# END environment bootstrap section + +cd ${VESPA_HOME} || { echo "Cannot cd to ${VESPA_HOME}" 1>&2; exit 1; } + +fixlimits + +exec run-as-yahoo ${VESPA_HOME}/bin/config-ctl stop diff --git a/vespabase/src/vespa-allow-downgrade-from-6-to-5.sh b/vespabase/src/vespa-allow-downgrade-from-6-to-5.sh new file mode 100755 index 00000000000..c01333bdd79 --- /dev/null +++ b/vespabase/src/vespa-allow-downgrade-from-6-to-5.sh @@ -0,0 +1,8 @@ +#!/bin/sh +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +for f in /home/y/var/db/vespa/search/*/*/documents/*/config/config-* +do + echo $f + touch $f/smartsummary.cfg +done diff --git a/vespabase/src/vespa-core-dumper.sh b/vespabase/src/vespa-core-dumper.sh new file mode 100755 index 00000000000..5f9baadf4fc --- /dev/null +++ b/vespabase/src/vespa-core-dumper.sh @@ -0,0 +1,28 @@ +#!/bin/bash +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +log_message () { + echo "warning $*" | logger -t vespa-core-dumper +} + +compressor=$1 +corefile=$2 +option=$3 + +log_message "Starting $compressor > $corefile" + + +if [ -f "$corefile" ] +then + if [ "$option" != "overwrite" ] + then + log_message "$corefile is read-only. Core is not dumped." + exit 1 + else + log_message "Overwriting $corefile" + fi +fi + +$compressor > $corefile +chmod 444 $corefile +log_message "Finished $compressor > $corefile" diff --git a/vespabase/src/vespa-start-configserver.sh b/vespabase/src/vespa-start-configserver.sh new file mode 100755 index 00000000000..1af506d7db1 --- /dev/null +++ b/vespabase/src/vespa-start-configserver.sh @@ -0,0 +1,73 @@ +#!/bin/sh +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +# BEGIN environment bootstrap section +# Do not edit between here and END as this section should stay identical in all scripts + +findpath () { + myname=${0} + mypath=${myname%/*} + myname=${myname##*/} + if [ "$mypath" ] && [ -d "$mypath" ]; then + return + fi + mypath=$(pwd) + if [ -f "${mypath}/${myname}" ]; then + return + fi + echo "FATAL: Could not figure out the path where $myname lives from $0" + exit 1 +} + +COMMON_ENV=libexec/vespa/common-env.sh + +source_common_env () { + if [ "$VESPA_HOME" ] && [ -d "$VESPA_HOME" ]; then + # ensure it ends with "/" : + VESPA_HOME=${VESPA_HOME%/}/ + export VESPA_HOME + common_env=$VESPA_HOME/$COMMON_ENV + if [ -f "$common_env" ]; then + . $common_env + return + fi + fi + return 1 +} + +findroot () { + source_common_env && return + if [ "$VESPA_HOME" ]; then + echo "FATAL: bad VESPA_HOME value '$VESPA_HOME'" + exit 1 + fi + if [ "$ROOT" ] && [ -d "$ROOT" ]; then + VESPA_HOME="$ROOT" + source_common_env && return + fi + findpath + while [ "$mypath" ]; do + VESPA_HOME=${mypath} + source_common_env && return + mypath=${mypath%/*} + done + echo "FATAL: missing VESPA_HOME environment variable" + echo "Could not locate $COMMON_ENV anywhere" + exit 1 +} + +findroot + +# END environment bootstrap section + +${VESPA_HOME}bin/vespa-prestart.sh || exit 1 +willrun=${VESPA_HOME}libexec/vespa/start-configserver +if [ -x $willrun ]; then + echo "Starting $willrun as a background process." >&2 + exec /dev/null 2>&1 + nohup $willrun & + exit 0 +fi +echo "FATAL cannot run: $willrun" +exit 1 diff --git a/vespabase/src/vespa-start-services.sh b/vespabase/src/vespa-start-services.sh new file mode 100755 index 00000000000..f2a043247b4 --- /dev/null +++ b/vespabase/src/vespa-start-services.sh @@ -0,0 +1,73 @@ +#!/bin/sh +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +# BEGIN environment bootstrap section +# Do not edit between here and END as this section should stay identical in all scripts + +findpath () { + myname=${0} + mypath=${myname%/*} + myname=${myname##*/} + if [ "$mypath" ] && [ -d "$mypath" ]; then + return + fi + mypath=$(pwd) + if [ -f "${mypath}/${myname}" ]; then + return + fi + echo "FATAL: Could not figure out the path where $myname lives from $0" + exit 1 +} + +COMMON_ENV=libexec/vespa/common-env.sh + +source_common_env () { + if [ "$VESPA_HOME" ] && [ -d "$VESPA_HOME" ]; then + # ensure it ends with "/" : + VESPA_HOME=${VESPA_HOME%/}/ + export VESPA_HOME + common_env=$VESPA_HOME/$COMMON_ENV + if [ -f "$common_env" ]; then + . $common_env + return + fi + fi + return 1 +} + +findroot () { + source_common_env && return + if [ "$VESPA_HOME" ]; then + echo "FATAL: bad VESPA_HOME value '$VESPA_HOME'" + exit 1 + fi + if [ "$ROOT" ] && [ -d "$ROOT" ]; then + VESPA_HOME="$ROOT" + source_common_env && return + fi + findpath + while [ "$mypath" ]; do + VESPA_HOME=${mypath} + source_common_env && return + mypath=${mypath%/*} + done + echo "FATAL: missing VESPA_HOME environment variable" + echo "Could not locate $COMMON_ENV anywhere" + exit 1 +} + +findroot + +# END environment bootstrap section + +${VESPA_HOME}bin/vespa-prestart.sh || exit 1 +willrun=${VESPA_HOME}libexec/vespa/start-vespa-base.sh +if [ -x $willrun ]; then + echo "Starting $willrun as a background process." >&2 + exec /dev/null 2>&1 + nohup $willrun & + exit 0 +fi +echo "FATAL cannot run: $willrun" +exit 1 diff --git a/vespabase/src/vespa-stop-configserver.sh b/vespabase/src/vespa-stop-configserver.sh new file mode 100755 index 00000000000..3face7ccbde --- /dev/null +++ b/vespabase/src/vespa-stop-configserver.sh @@ -0,0 +1,69 @@ +#!/bin/sh +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +# BEGIN environment bootstrap section +# Do not edit between here and END as this section should stay identical in all scripts + +findpath () { + myname=${0} + mypath=${myname%/*} + myname=${myname##*/} + if [ "$mypath" ] && [ -d "$mypath" ]; then + return + fi + mypath=$(pwd) + if [ -f "${mypath}/${myname}" ]; then + return + fi + echo "FATAL: Could not figure out the path where $myname lives from $0" + exit 1 +} + +COMMON_ENV=libexec/vespa/common-env.sh + +source_common_env () { + if [ "$VESPA_HOME" ] && [ -d "$VESPA_HOME" ]; then + # ensure it ends with "/" : + VESPA_HOME=${VESPA_HOME%/}/ + export VESPA_HOME + common_env=$VESPA_HOME/$COMMON_ENV + if [ -f "$common_env" ]; then + . $common_env + return + fi + fi + return 1 +} + +findroot () { + source_common_env && return + if [ "$VESPA_HOME" ]; then + echo "FATAL: bad VESPA_HOME value '$VESPA_HOME'" + exit 1 + fi + if [ "$ROOT" ] && [ -d "$ROOT" ]; then + VESPA_HOME="$ROOT" + source_common_env && return + fi + findpath + while [ "$mypath" ]; do + VESPA_HOME=${mypath} + source_common_env && return + mypath=${mypath%/*} + done + echo "FATAL: missing VESPA_HOME environment variable" + echo "Could not locate $COMMON_ENV anywhere" + exit 1 +} + +findroot + +# END environment bootstrap section + +willrun=${VESPA_HOME}libexec/vespa/stop-configserver +if [ -x $willrun ]; then + echo "Executing $willrun" >&2 + exec $willrun +fi +echo "FATAL cannot run: $willrun" +exit 1 diff --git a/vespabase/src/vespa-stop-services.sh b/vespabase/src/vespa-stop-services.sh new file mode 100755 index 00000000000..03250b61a88 --- /dev/null +++ b/vespabase/src/vespa-stop-services.sh @@ -0,0 +1,69 @@ +#!/bin/sh +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +# BEGIN environment bootstrap section +# Do not edit between here and END as this section should stay identical in all scripts + +findpath () { + myname=${0} + mypath=${myname%/*} + myname=${myname##*/} + if [ "$mypath" ] && [ -d "$mypath" ]; then + return + fi + mypath=$(pwd) + if [ -f "${mypath}/${myname}" ]; then + return + fi + echo "FATAL: Could not figure out the path where $myname lives from $0" + exit 1 +} + +COMMON_ENV=libexec/vespa/common-env.sh + +source_common_env () { + if [ "$VESPA_HOME" ] && [ -d "$VESPA_HOME" ]; then + # ensure it ends with "/" : + VESPA_HOME=${VESPA_HOME%/}/ + export VESPA_HOME + common_env=$VESPA_HOME/$COMMON_ENV + if [ -f "$common_env" ]; then + . $common_env + return + fi + fi + return 1 +} + +findroot () { + source_common_env && return + if [ "$VESPA_HOME" ]; then + echo "FATAL: bad VESPA_HOME value '$VESPA_HOME'" + exit 1 + fi + if [ "$ROOT" ] && [ -d "$ROOT" ]; then + VESPA_HOME="$ROOT" + source_common_env && return + fi + findpath + while [ "$mypath" ]; do + VESPA_HOME=${mypath} + source_common_env && return + mypath=${mypath%/*} + done + echo "FATAL: missing VESPA_HOME environment variable" + echo "Could not locate $COMMON_ENV anywhere" + exit 1 +} + +findroot + +# END environment bootstrap section + +willrun=${VESPA_HOME}libexec/vespa/stop-vespa-base.sh +if [ -x $willrun ]; then + echo "Executing $willrun" >&2 + exec $willrun +fi +echo "FATAL cannot run: $willrun" +exit 1 -- cgit v1.2.3