diff options
| -rwxr-xr-x | dist/build-rpm.sh | 124 | ||||
| -rw-r--r-- | node-admin/src/main/application/services.xml | 23 | ||||
| -rwxr-xr-x | node-admin/src/main/sh/node-admin.sh | 82 | ||||
| -rw-r--r-- | node-admin/vespa-node-admin.spec | 49 | ||||
| -rwxr-xr-x | standalone-container/src/main/sh/standalone-container.sh | 309 | ||||
| -rw-r--r-- | standalone-container/vespa-standalone-container.spec | 95 |
6 files changed, 682 insertions, 0 deletions
diff --git a/dist/build-rpm.sh b/dist/build-rpm.sh new file mode 100755 index 00000000000..e86eebe9380 --- /dev/null +++ b/dist/build-rpm.sh @@ -0,0 +1,124 @@ +#!/bin/bash + +set -e + +Usage() { + cat <<EOF +Usage: ${0##*/} [OPTIONS]...SPECFILE +Run rpmbuild with the given specfile macros, creating TOPDIR if necessary. + +Options: + -b BUILDDIR Overrides %_builddir. + -d DIST The %dist to build for (e.g. .el7 for RHEL 7). Can be specified + multiple time to build multiple RPMs. The default %dist is used + if no -d options have been specified. + -h Print this help text and exit. + -t TOPDIR Overrides %_topdir. + -v VERSION [Required] The version of the RPM. +EOF + + exit 1 +} + +Fail() { + printf "%s\n" "$*" + exit 1 +} + +Run() { + local command="$1" + shift + printf "%q" "$command" + + local arg + for arg in "$@"; do + printf " %q" "$arg" + done + printf "\n" + + "$command" "$@" +} + +Main() { + local -a dists=() + local version= topdir= builddir= + while (( $# > 0 )); do + case "$1" in + -b|--builddir) + builddir="$2" + shift 2 + if ! test -d "$builddir"; then + Fail "BUILDDIR '$builddir' does not exist" + # Make builddir an absolute path + elif ! builddir=$(readlink -e "$builddir"); then + Fail "Failed to resolve BUILDDIR '$builddir'" + fi + ;; + -d|--dist) + local dist="$2" + shift 2 + case "$dist" in + .el6|.el7) : ;; + *) Fail "Bad DIST value '$dist'" ;; + esac + dists+=("$dist") + ;; + -t|--topdir) + topdir="$2" + shift 2 + if ! [[ "$topdir" =~ ^/ ]]; then + Fail "TOPDIR must be an absolute path" + fi + ;; + -v|--version) + version="$2" + shift 2 + if ! [[ "$version" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + Fail "VERSION must be a version of the form X.Y.Z" + fi + ;; + *) break ;; + esac + done + + if (( $# == 0 )); then + Fail "Missing SPECFILE" + elif (( $# > 1 )); then + Fail "Too many arguments" + else + case "$1" in + help|-h|--help) Usage ;; + esac + fi + local specfile="$1" + + local -a defines=() + + if test -n "$builddir"; then + defines+=(--define "_builddir $builddir") + fi + + if test -n "$topdir"; then + if ! mkdir -p "$topdir"; then + Fail "Failed to create TOPDIR directory '$topdir'" + fi + defines+=(--define "_topdir $topdir") + fi + + if test -n "$version"; then + defines+=(--define "version $version") + else + Fail "VERSION is required" + fi + + if (( ${#dists[@]} == 0 )); then + Run rpmbuild -bb "${defines[@]}" "$specfile" + else + local dist + for dist in "${dists[@]}"; do + Run rpmbuild -bb "${defines[@]}" --define "dist $dist" "$specfile" + done + fi +} + +Main "$@" diff --git a/node-admin/src/main/application/services.xml b/node-admin/src/main/application/services.xml new file mode 100644 index 00000000000..030c42ac8c5 --- /dev/null +++ b/node-admin/src/main/application/services.xml @@ -0,0 +1,23 @@ +<?xml version="1.0" encoding="utf-8" ?> +<!-- Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<jdisc id="node-admin" jetty="true" version="1.0"> + <!-- Please update container test when changing this file --> + <accesslog type="vespa" fileNamePattern="logs/vespa/node-admin/access.log.%Y%m%d%H%M%S" rotationScheme="date" symlinkName="access.log" /> + <handler id="com.yahoo.vespa.hosted.node.admin.restapi.RestApiHandler" bundle="node-admin"> + <binding>http://*/rest/*</binding> + </handler> + <component id="node-admin" class="com.yahoo.vespa.hosted.node.admin.provider.NodeAdminProvider" bundle="node-admin"/> + <component id="docker-api" class="com.yahoo.vespa.hosted.dockerapi.DockerImpl" bundle="docker-api"/> + <component id="metrics-wrapper" class="com.yahoo.vespa.hosted.dockerapi.metrics.MetricReceiverWrapper" bundle="docker-api"/> + + <config name='vespa.hosted.dockerapi.docker'> + <isRunningLocally>false</isRunningLocally> + </config> + + <config name='vespa.hosted.node.admin.node-admin'> + <isRunningLocally>false</isRunningLocally> + <restartOnDeploy>true</restartOnDeploy> + </config> + + <nodes type="host"/> +</jdisc> diff --git a/node-admin/src/main/sh/node-admin.sh b/node-admin/src/main/sh/node-admin.sh new file mode 100755 index 00000000000..ff0ea492318 --- /dev/null +++ b/node-admin/src/main/sh/node-admin.sh @@ -0,0 +1,82 @@ +#!/bin/bash +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +# BEGIN environment bootstrap section +# Do not edit between here and END as this section should stay identical in all scripts + +findpath () { + myname=${0} + mypath=${myname%/*} + myname=${myname##*/} + if [ "$mypath" ] && [ -d "$mypath" ]; then + return + fi + mypath=$(pwd) + if [ -f "${mypath}/${myname}" ]; then + return + fi + echo "FATAL: Could not figure out the path where $myname lives from $0" + exit 1 +} + +COMMON_ENV=libexec/vespa/common-env.sh + +source_common_env () { + if [ "$VESPA_HOME" ] && [ -d "$VESPA_HOME" ]; then + export VESPA_HOME + common_env=$VESPA_HOME/$COMMON_ENV + if [ -f "$common_env" ]; then + . $common_env + return + fi + fi + return 1 +} + +findroot () { + source_common_env && return + if [ "$VESPA_HOME" ]; then + echo "FATAL: bad VESPA_HOME value '$VESPA_HOME'" + exit 1 + fi + if [ "$ROOT" ] && [ -d "$ROOT" ]; then + VESPA_HOME="$ROOT" + source_common_env && return + fi + findpath + while [ "$mypath" ]; do + VESPA_HOME=${mypath} + source_common_env && return + mypath=${mypath%/*} + done + echo "FATAL: missing VESPA_HOME environment variable" + echo "Could not locate $COMMON_ENV anywhere" + exit 1 +} + +findroot + +# END environment bootstrap section + +Usage() { + cat <<EOF +Usage: ${0##*/} [start|stop] +Manage standalone node admin +EOF + + exit 1 +} + +if (( $# != 1 )); then + Usage +fi + +case "$1" in + start) + "$VESPA_HOME"/libexec/vespa/standalone-container.sh start -s node-admin -u root + ;; + stop) + "$VESPA_HOME"/libexec/vespa/standalone-container.sh stop -s node-admin -u root + ;; + *) Usage ;; +esac diff --git a/node-admin/vespa-node-admin.spec b/node-admin/vespa-node-admin.spec new file mode 100644 index 00000000000..ecdd9a02e95 --- /dev/null +++ b/node-admin/vespa-node-admin.spec @@ -0,0 +1,49 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +# Force special prefix for Vespa +%define _prefix /opt/vespa + +# Hack to speed up jar packing for now. This does not affect the rpm size. +%define __jar_repack %{nil} + +Name: vespa-node-admin +Version: %version +Release: 1%{?dist} +BuildArch: noarch +Summary: Vespa Node Admin +Group: Applications/Databases +License: Commercial +URL: http://vespa.ai + +Requires: bash +Requires: java-1.8.0-openjdk-headless +Requires: vespa-standalone-container + +Conflicts: vespa + +%description +The Node Admin manages the machine so it is a suitable host for one or more +Vespa nodes. + +%install +app_dir=%?buildroot%_prefix/conf/node-admin-app +mkdir -p "$app_dir"/components +cp node-admin/src/main/resources/services.xml "$app_dir" + +declare -a jar_components=( + node-admin/target/node-admin-jar-with-dependencies.jar + docker-api/target/docker-api-jar-with-dependencies.jar +) +for path in "${jar_components[@]}"; do + cp "$path" "$app_dir"/components +done + +mkdir -p %buildroot%_prefix/libexec/vespa +cp node-admin/src/main/sh/node-admin %buildroot%_prefix/libexec/vespa + +%clean +rm -rf %buildroot + +%files +%defattr(-,vespa,vespa,-) +%_prefix/* diff --git a/standalone-container/src/main/sh/standalone-container.sh b/standalone-container/src/main/sh/standalone-container.sh new file mode 100755 index 00000000000..dd0693f6f85 --- /dev/null +++ b/standalone-container/src/main/sh/standalone-container.sh @@ -0,0 +1,309 @@ +#!/bin/bash +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +# BEGIN environment bootstrap section +# Do not edit between here and END as this section should stay identical in all scripts + +findpath () { + myname=${0} + mypath=${myname%/*} + myname=${myname##*/} + if [ "$mypath" ] && [ -d "$mypath" ]; then + return + fi + mypath=$(pwd) + if [ -f "${mypath}/${myname}" ]; then + return + fi + echo "FATAL: Could not figure out the path where $myname lives from $0" + exit 1 +} + +COMMON_ENV=libexec/vespa/common-env.sh + +source_common_env () { + if [ "$VESPA_HOME" ] && [ -d "$VESPA_HOME" ]; then + export VESPA_HOME + common_env=$VESPA_HOME/$COMMON_ENV + if [ -f "$common_env" ]; then + . $common_env + return + fi + fi + return 1 +} + +findroot () { + source_common_env && return + if [ "$VESPA_HOME" ]; then + echo "FATAL: bad VESPA_HOME value '$VESPA_HOME'" + exit 1 + fi + if [ "$ROOT" ] && [ -d "$ROOT" ]; then + VESPA_HOME="$ROOT" + source_common_env && return + fi + findpath + while [ "$mypath" ]; do + VESPA_HOME=${mypath} + source_common_env && return + mypath=${mypath%/*} + done + echo "FATAL: missing VESPA_HOME environment variable" + echo "Could not locate $COMMON_ENV anywhere" + exit 1 +} + +findroot + +# END environment bootstrap section + +Usage() { + cat <<EOF +Usage: ${0##*/} start [OPTION]... +Usage: ${0##*/} stop [OPTION]... +Manage Vespa standalone jdisc container service. + +Options: + -u USER Run as USER. Overrides any VESPA_USER environment variable. + -s SERVICE The service name. +EOF + + exit 1 +} + +Fail() { + printf "%s\n" "$*" + exit 1 +} + +FixDataDirectory() { + if ! [ -d "$1" ]; then + echo "Creating data directory '$1'" + mkdir -p "$1" || exit 1 + fi + chown "${VESPA_USER}" "$1" + chmod 755 "$1" +} + +StartCommand() { + local service="$1" + shift + + if (( $# > 0 )); then + Fail "Too many arguments" + fi + + local service_regex='^[0-9a-zA-Z_-]+$' + if ! [[ "$service" =~ $service_regex ]]; then + Fail "Service must match regex '$service_regex'" + fi + + # common setup + export VESPA_SERVICE_NAME="$service" + + # stuff for the process: + local appdir="$VESPA_HOME/conf/$service-app" + local pidfile="$VESPA_HOME/var/run/$service.pid" + local cfpfile="$VESPA_HOME/var/jdisc_core/$service.properties" + local bundlecachedir="$VESPA_HOME/var/vespa/bundlecache/$service" + + cd "$VESPA_HOME" || Fail "Cannot cd to $VESPA_HOME" + + fixlimits + checkjava + + local vespa_log="$VESPA_HOME/logs/vespa/vespa.log" + export VESPA_LOG_TARGET="file:$vespa_log" + FixDataDirectory "$(dirname "$vespa_log")" + + export VESPA_LOG_CONTROL_FILE="$VESPA_HOME/var/db/vespa/logcontrol/$service.logcontrol" + export VESPA_LOG_CONTROL_DIR="$(dirname "$VESPA_LOG_CONTROL_FILE")" + FixDataDirectory "$VESPA_LOG_CONTROL_DIR" + + # Does not need fast allocation + export MALLOC_ARENA_MAX=1 + + # will be picked up by standalone-container: + export standalone_jdisc_container__app_location="$appdir" + + # class path + CP="$VESPA_HOME/lib/jars/jdisc_core-jar-with-dependencies.jar" + + FixDataDirectory "$(dirname "$cfpfile")" + printenv > "$cfpfile" + FixDataDirectory "$bundlecachedir" + + java \ + -Xms128m -Xmx2048m \ + -XX:+PreserveFramePointer \ + -XX:+HeapDumpOnOutOfMemoryError \ + -XX:HeapDumpPath="$VESPA_HOME/var/crash" \ + -XX:OnOutOfMemoryError="kill -9 %p" \ + -Djava.library.path="$VESPA_HOME/lib64" \ + -Djava.awt.headless=true \ + -Dsun.rmi.dgc.client.gcInterval=3600000 \ + -Dsun.net.client.defaultConnectTimeout=5000 \ + -Dsun.net.client.defaultReadTimeout=60000 \ + -Djavax.net.ssl.keyStoreType=JKS \ + -Djdisc.config.file="$cfpfile" \ + -Djdisc.export.packages= \ + -Djdisc.cache.path="$bundlecachedir" \ + -Djdisc.debug.resources=false \ + -Djdisc.bundle.path="$VESPA_HOME/lib/jars" \ + -Djdisc.logger.enabled=true \ + -Djdisc.logger.level=ALL \ + -Djdisc.logger.tag="jdisc/$service" \ + -Dfile.encoding=UTF-8 \ + -cp "$CP" \ + com.yahoo.jdisc.core.StandaloneMain standalone-container-jar-with-dependencies.jar & + + local pid="$!" + echo "$pid" > "$pidfile" +} + +Kill() { + local force="$1" + local expected_user="$2" + local expected_comm="$3" # Executable name only + local pid="$4" + + local -i now + if ! now=$(date +%s); then + Fail "Failed to get the current date in seconds since epoch" + fi + local -i timeout=$(( now + 300 )) + + local has_killed=false + + while true; do + local ps_output="" + if ! ps_output=$(ps -p "$pid" -o user= -o comm=); then + # success + return + fi + + local user comm + read -r user comm <<< "$ps_output" + + if test "$user" != "$expected_user"; then + echo "Warning: Pid collision ($pid): Expected user $expected_user but found $user." + echo "Will assume original process has died." + return + fi + + if test "$comm" != "$expected_comm"; then + echo "Warning: Pid collision ($pid): Expected program $expected_comm but found $comm." + echo "Will assume original process has died." + return + fi + + if ! "$has_killed"; then + if $force; then + if ! kill -KILL "$pid"; then + Fail "Failed to kill $pid" + fi + else + if ! kill "$pid"; then + Fail "Failed to kill $pid" + fi + fi + + has_killed=true + fi + + sleep 1 + + now=$(date +%s) + if (( now >= timeout )); then + Fail "Process $pid still exists after $timeout seconds, giving up" + fi + done +} + +StopCommand() { + local user="$1" + shift + + local force=false + while (( $# > 0 )); do + case "$1" in + -f|--force) + force=true + shift + ;; + *) break ;; + esac + done + + if (( $# != 1 )); then + Fail "Stop command takes exactly one argument" + fi + + local service="$1" + + local pidfile="$VESPA_HOME/var/run/$service.pid" + if ! test -r "$pidfile"; then + echo "$service is not running" + return + fi + + local pid=$(< "$pidfile") + if ! [[ "$pid" =~ ^[0-9]+$ ]]; then + Fail "Pid file '$pidfile' does not contain a valid pid: $pid" + fi + + Kill "$force" "$user" java "$pid" + rm -f "$pidfile" +} + +Main() { + if (( $# == 0 )); then + Usage + fi + + local command="$1" + shift + + local service="standalone/container" + local user="$VESPA_USER" + + while (( $# > 0 )); do + case "$1" in + --help|-h) Usage ;; + --service|-s) + service="$2" + shift 2 + ;; + --user|-u) + user="$2" + shift 2 + ;; + *) break ;; + esac + done + + # Service name will be included in paths and possibly environment variable + # names, so be restrictive. + local service_regex='^[a-zA-Z0-9_-]+$' + if test -z "$service"; then + Fail "SERVICE not specified" + elif ! [[ "$service" =~ $service_regex ]]; then + Fail "Service must math the regex '$service_regex'" + fi + + if ! getent passwd "$user" &> /dev/null; then + Fail "Bad user ($user): not found in passwd" + elif test "$(id -un)" != "$user"; then + Fail "${0##*/} must be started by $user" + fi + + case "$command" in + help) Usage ;; + start) StartCommand "$service" "$@" ;; + stop) StopCommand "$user" "$service" "$@" ;; + *) Fail "Unknown command '$command'" ;; + esac +} + +Main "$@" diff --git a/standalone-container/vespa-standalone-container.spec b/standalone-container/vespa-standalone-container.spec new file mode 100644 index 00000000000..5ab7e9409a5 --- /dev/null +++ b/standalone-container/vespa-standalone-container.spec @@ -0,0 +1,95 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +# Force special prefix for Vespa +%define _prefix /opt/vespa + +# Hack to speed up jar packing for now. This does not affect the rpm size. +%define __jar_repack %{nil} + +Name: vespa-standalone-container +Version: %version +BuildArch: noarch +Release: 1%{?dist} +Summary: Vespa standalone JDisc container +Group: Applications/Databases +License: Commercial +URL: http://vespa.ai + +Requires: bash +Requires: java-1.8.0-openjdk-headless + +Conflicts: vespa + +%description +The Vespa standalone JDisc container is a runtime environment for Java +applications. + +%install +declare jars_dir=%buildroot%_prefix/lib/jars +mkdir -p "$jars_dir" + +declare -a dirs=( + jdisc_jetty/target/dependency + vespa_jersey2/target/dependency +) +for dir in "${dirs[@]}"; do + cp "$dir"/* "$jars_dir" +done + +declare -a modules=( + component + config-bundle + config-model-api + config-model + config-provisioning + configdefinitions + container-disc + container-jersey2 + container-search-and-docproc + defaults + docprocs + jdisc_core + jdisc_http_service + simplemetrics + standalone-container + vespaclient-container-plugin + zkfacade +) +for module in "${modules[@]}"; do + cp "$module"/target/"$module"-jar-with-dependencies.jar "$jars_dir" +done + +# vespajlib must be installed _without_ dependencies. +cp vespajlib/target/vespajlib.jar "$jars_dir" + +declare -a libexec_files=( + vespabase/src/common-env.sh + standalone-container/src/main/sh/standalone-container.sh +) +declare libexec_dir=%buildroot%_prefix/libexec/vespa +mkdir -p "$libexec_dir" +for file in "${libexec_files[@]}"; do + cp "$file" "$libexec_dir" +done + +%clean +rm -rf %buildroot + +%pre +getent group vespa >/dev/null || groupadd -r vespa +getent passwd vespa >/dev/null || \ + useradd -r -g vespa -d %_prefix -s /sbin/nologin \ + -c "Create owner of all Vespa data files" vespa +echo "pathmunge %_prefix/bin" > /etc/profile.d/vespa.sh +echo "export VESPA_HOME=%_prefix" >> /etc/profile.d/vespa.sh +chmod +x /etc/profile.d/vespa.sh + +%postun +if [ $1 -eq 0 ]; then # this is an uninstallation + rm -f /etc/profile.d/vespa.sh + userdel vespa +fi + +%files +%defattr(-,vespa,vespa,-) +%_prefix/* |