diff options
-rwxr-xr-x | build-rpm.sh | 124 | ||||
-rw-r--r-- | node-admin/src/main/resources/services.xml | 23 | ||||
-rwxr-xr-x | node-admin/src/main/sh/node-admin | 82 | ||||
-rw-r--r-- | node-admin/vespa-node-admin.spec | 49 | ||||
-rw-r--r-- | pom.xml | 1 | ||||
-rwxr-xr-x | standalone-container/src/main/sh/jdisc-container | 218 | ||||
-rw-r--r-- | standalone-container/vespa-jdisc-container.spec | 92 |
7 files changed, 589 insertions, 0 deletions
diff --git a/build-rpm.sh b/build-rpm.sh new file mode 100755 index 00000000000..e86eebe9380 --- /dev/null +++ b/build-rpm.sh @@ -0,0 +1,124 @@ +#!/bin/bash + +set -e + +Usage() { + cat <<EOF +Usage: ${0##*/} [OPTIONS]...SPECFILE +Run rpmbuild with the given specfile macros, creating TOPDIR if necessary. + +Options: + -b BUILDDIR Overrides %_builddir. + -d DIST The %dist to build for (e.g. .el7 for RHEL 7). Can be specified + multiple time to build multiple RPMs. The default %dist is used + if no -d options have been specified. + -h Print this help text and exit. + -t TOPDIR Overrides %_topdir. + -v VERSION [Required] The version of the RPM. +EOF + + exit 1 +} + +Fail() { + printf "%s\n" "$*" + exit 1 +} + +Run() { + local command="$1" + shift + printf "%q" "$command" + + local arg + for arg in "$@"; do + printf " %q" "$arg" + done + printf "\n" + + "$command" "$@" +} + +Main() { + local -a dists=() + local version= topdir= builddir= + while (( $# > 0 )); do + case "$1" in + -b|--builddir) + builddir="$2" + shift 2 + if ! test -d "$builddir"; then + Fail "BUILDDIR '$builddir' does not exist" + # Make builddir an absolute path + elif ! builddir=$(readlink -e "$builddir"); then + Fail "Failed to resolve BUILDDIR '$builddir'" + fi + ;; + -d|--dist) + local dist="$2" + shift 2 + case "$dist" in + .el6|.el7) : ;; + *) Fail "Bad DIST value '$dist'" ;; + esac + dists+=("$dist") + ;; + -t|--topdir) + topdir="$2" + shift 2 + if ! [[ "$topdir" =~ ^/ ]]; then + Fail "TOPDIR must be an absolute path" + fi + ;; + -v|--version) + version="$2" + shift 2 + if ! [[ "$version" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + Fail "VERSION must be a version of the form X.Y.Z" + fi + ;; + *) break ;; + esac + done + + if (( $# == 0 )); then + Fail "Missing SPECFILE" + elif (( $# > 1 )); then + Fail "Too many arguments" + else + case "$1" in + help|-h|--help) Usage ;; + esac + fi + local specfile="$1" + + local -a defines=() + + if test -n "$builddir"; then + defines+=(--define "_builddir $builddir") + fi + + if test -n "$topdir"; then + if ! mkdir -p "$topdir"; then + Fail "Failed to create TOPDIR directory '$topdir'" + fi + defines+=(--define "_topdir $topdir") + fi + + if test -n "$version"; then + defines+=(--define "version $version") + else + Fail "VERSION is required" + fi + + if (( ${#dists[@]} == 0 )); then + Run rpmbuild -bb "${defines[@]}" "$specfile" + else + local dist + for dist in "${dists[@]}"; do + Run rpmbuild -bb "${defines[@]}" --define "dist $dist" "$specfile" + done + fi +} + +Main "$@" diff --git a/node-admin/src/main/resources/services.xml b/node-admin/src/main/resources/services.xml new file mode 100644 index 00000000000..57bdf8b7af2 --- /dev/null +++ b/node-admin/src/main/resources/services.xml @@ -0,0 +1,23 @@ +<?xml version="1.0" encoding="utf-8" ?> +<!-- Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<jdisc id="node-admin" jetty="true" version="1.0"> + <!-- Please update container test when changing this file --> + <accesslog type="vespa" fileNamePattern="logs/vespa/node-admin/access.log.%Y%m%d%H%M%S" rotationScheme="date" symlinkName="access.log" /> + <handler id="com.yahoo.vespa.hosted.node.admin.restapi.RestApiHandler" bundle="node-admin"> + <binding>http://*/rest/*</binding> + </handler> + <component id="node-admin" class="com.yahoo.vespa.hosted.node.admin.provider.NodeAdminProvider" bundle="node-admin"/> + <component id="docker-api" class="com.yahoo.vespa.hosted.dockerapi.DockerImpl" bundle="docker-api"/> + <component id="metrics-wrapper" class="com.yahoo.vespa.hosted.dockerapi.metrics.MetricReceiverWrapper" bundle="docker-api"/> + + <config name='vespa.hosted.dockerapi.docker'> + <isRunningLocally>false</isRunningLocally> + </config> + + <config name='vespa.hosted.node.admin.node-admin'> + <isRunningLocally>false</isRunningLocally> + <restartOnDeploy>true</restartOnDeploy> + </config> + + <nodes jvmargs="-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=localhost:5555 -Dvespa.freezedetector.disable=true" type="host"/> +</jdisc> diff --git a/node-admin/src/main/sh/node-admin b/node-admin/src/main/sh/node-admin new file mode 100755 index 00000000000..8a8963eeed0 --- /dev/null +++ b/node-admin/src/main/sh/node-admin @@ -0,0 +1,82 @@ +#!/bin/bash +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +# BEGIN environment bootstrap section +# Do not edit between here and END as this section should stay identical in all scripts + +findpath () { + myname=${0} + mypath=${myname%/*} + myname=${myname##*/} + if [ "$mypath" ] && [ -d "$mypath" ]; then + return + fi + mypath=$(pwd) + if [ -f "${mypath}/${myname}" ]; then + return + fi + echo "FATAL: Could not figure out the path where $myname lives from $0" + exit 1 +} + +COMMON_ENV=libexec/vespa/common-env.sh + +source_common_env () { + if [ "$VESPA_HOME" ] && [ -d "$VESPA_HOME" ]; then + export VESPA_HOME + common_env=$VESPA_HOME/$COMMON_ENV + if [ -f "$common_env" ]; then + . $common_env + return + fi + fi + return 1 +} + +findroot () { + source_common_env && return + if [ "$VESPA_HOME" ]; then + echo "FATAL: bad VESPA_HOME value '$VESPA_HOME'" + exit 1 + fi + if [ "$ROOT" ] && [ -d "$ROOT" ]; then + VESPA_HOME="$ROOT" + source_common_env && return + fi + findpath + while [ "$mypath" ]; do + VESPA_HOME=${mypath} + source_common_env && return + mypath=${mypath%/*} + done + echo "FATAL: missing VESPA_HOME environment variable" + echo "Could not locate $COMMON_ENV anywhere" + exit 1 +} + +findroot + +# END environment bootstrap section + +Usage() { + cat <<EOF +Usage: ${0##*/} [start|stop] +Manage standalone node admin +EOF + + exit 1 +} + +if (( $# != 1 )); then + Usage +fi + +case "$1" in + start) + "$VESPA_HOME"/libexec/vespa/jdisc-container start -s node-admin -u root + ;; + stop) + "$VESPA_HOME"/libexec/vespa/jdisc-container stop -s node-admin -u root + ;; + *) Usage ;; +esac diff --git a/node-admin/vespa-node-admin.spec b/node-admin/vespa-node-admin.spec new file mode 100644 index 00000000000..8c0eb23533c --- /dev/null +++ b/node-admin/vespa-node-admin.spec @@ -0,0 +1,49 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +# Force special prefix for Vespa +%define _prefix /opt/vespa + +# Hack to speed up jar packing for now. This does not affect the rpm size. +%define __jar_repack %{nil} + +Name: vespa-node-admin +Version: %version +Release: 1%{?dist} +BuildArch: noarch +Summary: Vespa Node Admin +Group: Applications/Databases +License: Commercial +URL: http://vespa.ai + +Requires: bash +Requires: java-1.8.0-openjdk-headless +Requires: vespa-jdisc-container + +Conflicts: vespa + +%description +The Node Admin manages the machine so it is a suitable host for one or more +Vespa nodes. + +%install +app_dir=%?buildroot%_prefix/conf/node-admin-app +mkdir -p "$app_dir"/components +cp node-admin/src/main/resources/services.xml "$app_dir" + +declare -a jar_components=( + node-admin/target/node-admin-jar-with-dependencies.jar + docker-api/target/docker-api-jar-with-dependencies.jar +) +for path in "${jar_components[@]}"; do + cp "$path" "$app_dir"/components +done + +mkdir -p %buildroot%_prefix/libexec/vespa +cp node-admin/src/main/sh/node-admin %buildroot%_prefix/libexec/vespa + +%clean +rm -rf %buildroot + +%files +%defattr(-,vespa,vespa,-) +%_prefix/* @@ -111,6 +111,7 @@ <module>simplemetrics</module> <module>socket_test</module> <module>standalone-container</module> + <module>standalone-container-deps</module> <module>statistics</module> <module>storage</module> <module>testutil</module> diff --git a/standalone-container/src/main/sh/jdisc-container b/standalone-container/src/main/sh/jdisc-container new file mode 100755 index 00000000000..5d023e5de4d --- /dev/null +++ b/standalone-container/src/main/sh/jdisc-container @@ -0,0 +1,218 @@ +#!/bin/bash +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +# BEGIN environment bootstrap section +# Do not edit between here and END as this section should stay identical in all scripts + +findpath () { + myname=${0} + mypath=${myname%/*} + myname=${myname##*/} + if [ "$mypath" ] && [ -d "$mypath" ]; then + return + fi + mypath=$(pwd) + if [ -f "${mypath}/${myname}" ]; then + return + fi + echo "FATAL: Could not figure out the path where $myname lives from $0" + exit 1 +} + +COMMON_ENV=libexec/vespa/common-env.sh + +source_common_env () { + if [ "$VESPA_HOME" ] && [ -d "$VESPA_HOME" ]; then + export VESPA_HOME + common_env=$VESPA_HOME/$COMMON_ENV + if [ -f "$common_env" ]; then + . $common_env + return + fi + fi + return 1 +} + +findroot () { + source_common_env && return + if [ "$VESPA_HOME" ]; then + echo "FATAL: bad VESPA_HOME value '$VESPA_HOME'" + exit 1 + fi + if [ "$ROOT" ] && [ -d "$ROOT" ]; then + VESPA_HOME="$ROOT" + source_common_env && return + fi + findpath + while [ "$mypath" ]; do + VESPA_HOME=${mypath} + source_common_env && return + mypath=${mypath%/*} + done + echo "FATAL: missing VESPA_HOME environment variable" + echo "Could not locate $COMMON_ENV anywhere" + exit 1 +} + +findroot + +# END environment bootstrap section + +Usage() { + cat <<EOF +Usage: ${0##*/} start [OPTION]... +Usage: ${0##*/} stop [OPTION]... +Manage Vespa standalone jdisc container service. + +Options: + -u USER Run as USER. Overrides any VESPA_USER environment variable. + -s SERVICE The service name. +EOF + + exit 1 +} + +Fail() { + printf "%s\n" "$*" + exit 1 +} + +FixDataDirectory() { + if ! [ -d "$1" ]; then + echo "Creating data directory '$1'" + mkdir -p "$1" || exit 1 + fi + chown "${VESPA_USER}" "$1" + chmod 755 "$1" +} + +StartCommand() { + local service="$1" + shift + + if (( $# > 0 )); then + Fail "Too many arguments" + fi + + # common setup + export VESPA_SERVICE_NAME="$service" + + # stuff for the process: + local appdir="$VESPA_HOME/conf/$service-app" + local pidfile="$VESPA_HOME/var/run/$service.pid" + local cfpfile="$VESPA_HOME/var/jdisc_core/$service.properties" + local bundlecachedir="$VESPA_HOME/var/vespa/bundlecache/$service" + + cd "$VESPA_HOME" || Fail "Cannot cd to $VESPA_HOME" + + fixlimits + checkjava + + local vespa_log="$VESPA_HOME/logs/vespa/vespa.log" + export VESPA_LOG_TARGET="file:$vespa_log" + mkdir -p "$(dirname "$vespa_log")" + + export VESPA_LOG_CONTROL_FILE="$VESPA_HOME/var/db/vespa/logcontrol/$service.logcontrol" + export VESPA_LOG_CONTROL_DIR="$(dirname "$VESPA_LOG_CONTROL_FILE")" + mkdir -p "$VESPA_LOG_CONTROL_DIR" + + # Does not need fast allocation + export MALLOC_ARENA_MAX=1 + + # will be picked up by standalone-container: + export standalone_jdisc_container__app_location="$appdir" + + # class path + CP="$VESPA_HOME/lib/jars/jdisc_core-jar-with-dependencies.jar" + + mkdir -p "$(dirname "$cfpfile")" + printenv > "$cfpfile" + FixDataDirectory "$bundlecachedir" + + java \ + -Xms128m -Xmx2048m \ + -XX:+PreserveFramePointer \ + -XX:+HeapDumpOnOutOfMemoryError \ + -XX:HeapDumpPath="$VESPA_HOME/var/crash" \ + -XX:OnOutOfMemoryError="kill -9 %p" \ + -Djava.library.path="$VESPA_HOME/lib64" \ + -Djava.awt.headless=true \ + -Dsun.rmi.dgc.client.gcInterval=3600000 \ + -Dsun.net.client.defaultConnectTimeout=5000 \ + -Dsun.net.client.defaultReadTimeout=60000 \ + -Djavax.net.ssl.keyStoreType=JKS \ + -Djdisc.config.file="$cfpfile" \ + -Djdisc.export.packages= \ + -Djdisc.cache.path="$bundlecachedir" \ + -Djdisc.debug.resources=false \ + -Djdisc.bundle.path="$VESPA_HOME/lib/jars" \ + -Djdisc.logger.enabled=true \ + -Djdisc.logger.level=ALL \ + -Djdisc.logger.tag="jdisc/$service" \ + -Dfile.encoding=UTF-8 \ + -cp "$CP" \ + com.yahoo.jdisc.core.StandaloneMain standalone-container-jar-with-dependencies.jar +} + +StopCommand() { + local service="$1" + shift + + if (( $# > 0 )); then + Fail "Too many arguments" + fi + + Fail "Not implemented" +} + +Main() { + if (( $# == 0 )); then + Usage + fi + + local command="$1" + shift + + local service="standalone/container" + local user="$VESPA_USER" + + while (( $# > 0 )); do + case "$1" in + --help|-h) Usage ;; + --service|-s) + service="$2" + shift 2 + ;; + --user|-u) + user="$2" + shift 2 + ;; + *) break ;; + esac + done + + # Service name will be included in paths and possibly environment variable + # names, so be restrictive. + local service_regex='^[a-zA-Z0-9_-]+$' + if test -z "$service"; then + Fail "SERVICE not specified" + elif ! [[ "$service" =~ $service_regex ]]; then + Fail "Service must math the regex '$service_regex'" + fi + + if ! getent passwd "$user" &> /dev/null; then + Fail "Bad user ($user): not found in passwd" + elif test "$(id -un)" != "$user"; then + Fail "${0##*/} must be started by $user" + fi + export VESPA_USER="$user" + + case "$command" in + help) Usage ;; + start) StartCommand "$service" "$@" ;; + stop) StopCommand "$service" "$@" ;; + *) Fail "Unknown command '$command'" ;; + esac +} + +Main "$@" diff --git a/standalone-container/vespa-jdisc-container.spec b/standalone-container/vespa-jdisc-container.spec new file mode 100644 index 00000000000..84f5a832ad4 --- /dev/null +++ b/standalone-container/vespa-jdisc-container.spec @@ -0,0 +1,92 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +# Force special prefix for Vespa +%define _prefix /opt/vespa + +# Hack to speed up jar packing for now. This does not affect the rpm size. +%define __jar_repack %{nil} + +Name: vespa-jdisc-container +Version: %version +BuildArch: noarch +Release: 1%{?dist} +Summary: Vespa standalone JDisc container +Group: Applications/Databases +License: Commercial +URL: http://vespa.ai + +Requires: bash +Requires: java-1.8.0-openjdk-headless + +Conflicts: vespa + +%description +The Vespa standalone JDisc container is a runtime environment for Java +applications. + +%install +declare jars_dir=%buildroot%_prefix/lib/jars +mkdir -p "$jars_dir" + +declare -a dirs=( + jdisc_jetty/target/dependency + vespa_jersey2/target/dependency +) +for dir in "${dirs[@]}"; do + cp "$dir"/* "$jars_dir" +done + +declare -a files=( + component/target/component-jar-with-dependencies.jar + config-bundle/target/config-bundle-jar-with-dependencies.jar + config-model-api/target/config-model-api-jar-with-dependencies.jar + config-model/target/config-model-jar-with-dependencies.jar + config-provisioning/target/config-provisioning-jar-with-dependencies.jar + configdefinitions/target/configdefinitions-jar-with-dependencies.jar + container-disc/target/container-disc-jar-with-dependencies.jar + container-jersey2/target/container-jersey2-jar-with-dependencies.jar + container-search-and-docproc/target/container-search-and-docproc-jar-with-dependencies.jar + defaults/target/defaults-jar-with-dependencies.jar + docprocs/target/docprocs-jar-with-dependencies.jar + jdisc_core/target/dependency/vespajlib.jar + jdisc_core/target/jdisc_core-jar-with-dependencies.jar + jdisc_http_service/target/jdisc_http_service-jar-with-dependencies.jar + simplemetrics/target/simplemetrics-jar-with-dependencies.jar + standalone-container/target/standalone-container-jar-with-dependencies.jar + vespaclient-container-plugin/target/vespaclient-container-plugin-jar-with-dependencies.jar + vespajlib/target/vespajlib.jar + zkfacade/target/zkfacade-jar-with-dependencies.jar +) +for file in "${files[@]}"; do + cp "$file" "$jars_dir" +done + +declare -a libexec_files=( + vespabase/src/common-env.sh + standalone-container/src/main/sh/jdisc-container +) +declare libexec_dir=%buildroot%_prefix/libexec/vespa +mkdir -p "$libexec_dir" +for file in "${libexec_files[@]}"; do + cp "$file" "$libexec_dir" +done + +%clean +rm -rf %buildroot + +%pre +getent group vespa >/dev/null || groupadd -r vespa +getent passwd vespa >/dev/null || \ + useradd -r -g vespa -d %_prefix -s /sbin/nologin \ + -c "Create owner of all Vespa data files" vespa +echo "pathmunge %_prefix/bin" > /etc/profile.d/vespa.sh +echo "export VESPA_HOME=%_prefix" >> /etc/profile.d/vespa.sh +chmod +x /etc/profile.d/vespa.sh + +%postun +rm -f /etc/profile.d/vespa.sh +userdel vespa + +%files +%defattr(-,vespa,vespa,-) +%_prefix/* |