diff options
52 files changed, 571 insertions, 115 deletions
diff --git a/README-documentation.md b/README-documentation.md index 81d5d93199e..01a95ba6ad4 100644 --- a/README-documentation.md +++ b/README-documentation.md @@ -62,6 +62,10 @@ Make the text as short, clear, and easy to read as possible: * Avoid superfluous words such as "very". * Avoid filler sentences intended to improve the flow of the text - documents are usually browsed, not read anyway. * Use consistent terminology even when it leads to repetition which would be bad in other kind of writing. +* Use active form "index the documents", not passive "indexing the documents" +* Avoid making it personal - do not use "we", "you", "our" +* Do not use &quot; , &mdash; and the likes - makes the document harder to edit, and no need to use it. +* Less is more - <em> and <strong> is sufficient formatting in most cases ### Links @@ -72,4 +76,4 @@ Example: <h2 id="my-nice-heading">My nice Heading</h2> If this algorithmic transformation is followed it is possible to link to this section using <a href="doc.html#my-nice-heading"> without having to consult the html source of the page to find the right id. -*By Jon Bratseth in June 2016*
\ No newline at end of file +*By Jon Bratseth in June 2016* diff --git a/README.md b/README.md index 11a2e6145c4..722041c294d 100644 --- a/README.md +++ b/README.md @@ -4,8 +4,11 @@ Vespa is an engine for low-latency computation over large data sets. It stores and indexes your data such that queries, selection and processing over the data can be performed at serving time. -This README describes how to build and develop the Vespa engine. -For user documentation see TODO: Github pages link +This README describes how to build and develop the Vespa engine. If you want to use Vespa +you can go to the +[quick start guide](http://yahoo.github.io/vespa/vespa-quick-start.html), or take a +look at our +[user documentation](http://yahoo.github.io/vespa/vespatoc.html). ## Getting started developing @@ -14,7 +17,6 @@ For user documentation see TODO: Github pages link git config --global user.name "John Doe" git config --global user.email johndoe@host.com - ### Setting up build environment C++ building is supported on CentOS 7. @@ -42,8 +44,7 @@ Java modules can be built on any environment having Java and Maven: ### Building C++ modules source /opt/rh/devtoolset-4/enable - sh bootstrap.sh - mvn install -DskipTests -Dmaven.javadoc.skip=true + sh bootstrap.sh full cmake . make make test @@ -57,7 +58,6 @@ Java modules can be built on any environment having Java and Maven: * OS X : See [node-admin/README_MAC.md](node-admin/README_MAC.md) * Linux : See [node-admin/README_LINUX.md](node-admin/README_LINUX.md) - Code licensed under the Apache 2.0 license. See LICENSE file for terms. ## Documenting your features diff --git a/bootstrap.sh b/bootstrap.sh index 60145b39077..66b399fac13 100755 --- a/bootstrap.sh +++ b/bootstrap.sh @@ -1,6 +1,16 @@ #!/bin/bash -e # Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +FULL=false + +if [ "$1" = "full" ]; then + FULL=true +fi + +mvn_install() { + mvn install -Dmaven.test.skip=true -Dmaven.javadoc.skip=true $@ +} + # Generate vtag map top=$(dirname $0) $top/dist/getversion.pl -M $top > $top/dist/vtag.map @@ -9,15 +19,20 @@ $top/dist/getversion.pl -M $top > $top/dist/vtag.map # Vespa java dependency graph. MODULES=" parent - configgen annotations scalalib bundle-plugin - config-class-plugin - yolean - vespajlib - filedistributionmanager" + " for module in $MODULES; do - (cd $module && mvn install -DskipTests -Dmaven.javadoc.skip=true) + (cd $module && mvn_install) done + +mvn_install -am -pl config-class-plugin -rf configgen +if $FULL; then + # Build all java modules required by C++ testing + mvn_install -am -pl filedistributionmanager,jrt,linguistics,messagebus -rf yolean +else + # Build minimal set of java modules required to run cmake + mvn_install -am -pl filedistributionmanager -rf yolean +fi diff --git a/container-core/pom.xml b/container-core/pom.xml index 650c5e87074..e9b2d18628d 100644 --- a/container-core/pom.xml +++ b/container-core/pom.xml @@ -144,7 +144,6 @@ <groupId>com.yahoo.vespa</groupId> <artifactId>vespajlib</artifactId> <version>${project.version}</version> - <scope>provided</scope> <exclusions> <exclusion> <groupId>log4j</groupId> diff --git a/container-dev/pom.xml b/container-dev/pom.xml index 22cc10d36e4..ca21d564ed4 100644 --- a/container-dev/pom.xml +++ b/container-dev/pom.xml @@ -118,10 +118,5 @@ <artifactId>config-bundle</artifactId> <version>${project.version}</version> </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>searchlib</artifactId> - <version>${project.version}</version> - </dependency> </dependencies> </project> diff --git a/container-di/pom.xml b/container-di/pom.xml index 0cab901ab9f..f78cae8f434 100644 --- a/container-di/pom.xml +++ b/container-di/pom.xml @@ -73,7 +73,6 @@ <groupId>com.yahoo.vespa</groupId> <artifactId>vespajlib</artifactId> <version>${project.version}</version> - <scope>provided</scope> <exclusions> <exclusion> <groupId>log4j</groupId> diff --git a/container-disc/pom.xml b/container-disc/pom.xml index a6a6c78a662..f320eb371bb 100644 --- a/container-disc/pom.xml +++ b/container-disc/pom.xml @@ -94,7 +94,6 @@ <groupId>com.yahoo.vespa</groupId> <artifactId>vespajlib</artifactId> <version>${project.version}</version> - <scope>provided</scope> </dependency> <dependency> <groupId>com.yahoo.vespa</groupId> @@ -118,7 +117,6 @@ <groupId>com.yahoo.vespa</groupId> <artifactId>searchlib</artifactId> <version>${project.version}</version> - <scope>provided</scope> <exclusions> <exclusion> <groupId>com.yahoo.vespa</groupId> @@ -156,18 +154,17 @@ <Bundle-Activator>com.yahoo.container.jdisc.osgi.ContainerBundleActivator</Bundle-Activator> <discApplicationClass>com.yahoo.container.jdisc.ConfiguredApplication</discApplicationClass> <discPreInstallBundle> - component-jar-with-dependencies.jar, config-bundle-jar-with-dependencies.jar, configdefinitions-jar-with-dependencies.jar, container-jersey2-jar-with-dependencies.jar, container-search-and-docproc-jar-with-dependencies.jar, - defaults-jar-with-dependencies.jar, docprocs-jar-with-dependencies.jar, jdisc_http_service-jar-with-dependencies.jar, persistence-jar-with-dependencies.jar, - searchlib-jar-with-dependencies.jar, - simplemetrics-jar-with-dependencies.jar, vespaclient-container-plugin-jar-with-dependencies.jar, + simplemetrics-jar-with-dependencies.jar, + defaults-jar-with-dependencies.jar, + component-jar-with-dependencies.jar, <!-- jersey2 --> aopalliance-repackaged-2.3.0-b05.jar, hk2-api-2.3.0-b05.jar, diff --git a/container-disc/src/main/sh/vespa-start-container-daemon.sh b/container-disc/src/main/sh/vespa-start-container-daemon.sh new file mode 100755 index 00000000000..1c63eac805c --- /dev/null +++ b/container-disc/src/main/sh/vespa-start-container-daemon.sh @@ -0,0 +1,81 @@ +#!/bin/sh +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +if [ -z "${VESPA_HOME}" ]; then + echo "Missing VESPA_HOME variable" + exit 1 +fi +if [ -z "${VESPA_SERVICE_NAME}" ]; then + echo "Missing VESPA_SERVICE_NAME variable" + exit 1 +fi +if [ -z "${VESPA_CONFIG_ID}" ]; then + echo "Missing VESPA_CONFIG_ID variable" + exit 1 +fi +cd ${VESPA_HOME} || { echo "Cannot cd to ${VESPA_HOME}" 1>&2; exit 1; } + +DISCRIMINATOR=`echo ${VESPA_CONFIG_ID} | md5sum | cut -d' ' -f1` +CONTAINER_HOME="${VESPA_HOME}var/jdisc_container/${DISCRIMINATOR}/" + +ZOOKEEPER_LOG_FILE="${VESPA_HOME}logs/vespa/zookeeper.${VESPA_SERVICE_NAME}.log" +rm -f $ZOOKEEPER_LOG_FILE*lck + +# common setup +export VESPA_LOG_TARGET=file:${VESPA_HOME}logs/vespa/vespa.log +export VESPA_LOG_CONTROL_DIR=${VESPA_HOME}var/db/vespa/logcontrol +export LD_LIBRARY_PATH=${VESPA_HOME}lib64 + +cfpfile=${CONTAINER_HOME}/jdisc.properties +bundlecachedir=${CONTAINER_HOME}/bundlecache + +export JAVAVM_LD_PRELOAD= +unset LD_PRELOAD + +# class path +CP="${VESPA_HOME}lib/jars/jdisc_core-jar-with-dependencies.jar" + +mkdir -p $bundlecachedir || exit 1 +printenv > $cfpfile || exit 1 + +# ??? TODO ??? XXX ??? +# LANG=en_US.utf8 +# LC_ALL=C +# later, somewhere: +# export YELL_MA_EURO=INXIGHT + +if [ "$PRELOAD" ]; then + export JAVAVM_LD_PRELOAD="$PRELOAD" + export LD_PRELOAD="$PRELOAD" +fi + +exec java \ + -Xms1536m -Xmx1536m \ + -XX:MaxDirectMemorySize=267m \ + -XX:ThreadStackSize=512 \ + -XX:+UseConcMarkSweepGC \ + -XX:MaxTenuringThreshold=15 \ + -XX:NewRatio=1 \ + -XX:MaxJavaStackTraceDepth=-1 \ + -Dconfig.id="${VESPA_CONFIG_ID}" \ + -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath="${VESPA_HOME}var/crash" \ + -XX:OnOutOfMemoryError='kill -9 %p' \ + -Djava.library.path="${VESPA_HOME}lib64" \ + -Djava.awt.headless=true \ + -Dsun.rmi.dgc.client.gcInterval=3600000 \ + -Dsun.net.client.defaultConnectTimeout=5000 -Dsun.net.client.defaultReadTimeout=60000 \ + -Djavax.net.ssl.keyStoreType=JKS \ + -Djdisc.config.file="$cfpfile" \ + -Djdisc.export.packages= \ + -Djdisc.cache.path="$bundlecachedir" \ + -Djdisc.debug.resources=false \ + -Djdisc.bundle.path="${VESPA_HOME}lib/jars" \ + -Djdisc.logger.enabled=true \ + -Djdisc.logger.level=ALL \ + -Djdisc.logger.tag="${VESPA_CONFIG_ID}" \ + -Dorg.apache.commons.logging.Log=org.apache.commons.logging.impl.Jdk14Logger \ + -Dvespa.log.control.dir="${VESPA_LOG_CONTROL_DIR}" \ + -Dfile.encoding=UTF-8 \ + -Dzookeeperlogfile="${ZOOKEEPER_LOG_FILE}" \ + -cp "$CP" \ + com.yahoo.jdisc.core.StandaloneMain file:${VESPA_HOME}lib/jars/container-disc-jar-with-dependencies.jar diff --git a/container-search-and-docproc/pom.xml b/container-search-and-docproc/pom.xml index 36a5cc0c586..8c567c189ac 100644 --- a/container-search-and-docproc/pom.xml +++ b/container-search-and-docproc/pom.xml @@ -82,6 +82,10 @@ <groupId>com.yahoo.vespa</groupId> <artifactId>jrt</artifactId> </exclusion> + <exclusion> + <groupId>com.yahoo.vespa</groupId> + <artifactId>searchlib</artifactId> + </exclusion> </exclusions> </dependency> <dependency> @@ -118,6 +122,10 @@ <groupId>com.yahoo.vespa</groupId> <artifactId>jrt</artifactId> </exclusion> + <exclusion> + <groupId>com.yahoo.vespa</groupId> + <artifactId>searchlib</artifactId> + </exclusion> </exclusions> </dependency> <dependency> diff --git a/container-search/pom.xml b/container-search/pom.xml index 933d1128e01..eaab9079b89 100644 --- a/container-search/pom.xml +++ b/container-search/pom.xml @@ -106,7 +106,6 @@ <groupId>com.yahoo.vespa</groupId> <artifactId>searchlib</artifactId> <version>${project.version}</version> - <scope>provided</scope> </dependency> <dependency> <groupId>com.yahoo.vespa</groupId> diff --git a/container-search/src/main/java/com/yahoo/search/yql/FieldFiller.java b/container-search/src/main/java/com/yahoo/search/yql/FieldFiller.java index f6e8ee1f27a..653589a8e91 100644 --- a/container-search/src/main/java/com/yahoo/search/yql/FieldFiller.java +++ b/container-search/src/main/java/com/yahoo/search/yql/FieldFiller.java @@ -25,8 +25,8 @@ import edu.umd.cs.findbugs.annotations.NonNull; * Ensure the fields specified in {@link Presentation#getSummaryFields()} are * available after filling phase. * - * @author <a href="mailto:stiankri@yahoo-inc.com">Stian Kristoffersen</a> - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + * @author stiankri + * @author Steinar Knutsen */ @Beta @After(MinimalQueryInserter.EXTERNAL_YQL) diff --git a/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java b/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java index a7cc06c95f7..bcdc6ba060f 100644 --- a/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java +++ b/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java @@ -84,9 +84,9 @@ import edu.umd.cs.findbugs.annotations.NonNull; * VespaSerializer. * </p> * - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> - * @author <a href="mailto:stiankri@yahoo-inc.com">Stian Kristoffersen</a> - * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + * @author Steinar Knutsen + * @author stiankri + * @author Simon Thoresen */ @Beta public class YqlParser implements Parser { diff --git a/dist/vespa.spec b/dist/vespa.spec index 290472987fc..4a37716759f 100644 --- a/dist/vespa.spec +++ b/dist/vespa.spec @@ -130,6 +130,10 @@ ln -s %{_prefix}/lib/jars/zkfacade-jar-with-dependencies.jar %{buildroot}/%{_pre ln -s %{_prefix}/conf/configserver-app/components %{buildroot}/%{_prefix}/lib/jars/config-models ln -s storaged-bin %{buildroot}/%{_prefix}/sbin/distributord-bin +mkdir -p %{buildroot}/usr/lib/systemd/system +cp %{buildroot}/%{_prefix}/etc/systemd/system/vespa.service %{buildroot}/usr/lib/systemd/system +cp %{buildroot}/%{_prefix}/etc/systemd/system/vespa-configserver.service %{buildroot}/usr/lib/systemd/system + %clean rm -rf $RPM_BUILD_ROOT @@ -203,5 +207,7 @@ exit 0 %{_prefix}/etc/* %{_prefix}/conf/* %{_prefix}/share/vespa/schema/* +/usr/lib/systemd/system/vespa.service +/usr/lib/systemd/system/vespa-configserver.service %changelog diff --git a/docker/Dockerfile.build b/docker/Dockerfile.build new file mode 100644 index 00000000000..0f7f6935042 --- /dev/null +++ b/docker/Dockerfile.build @@ -0,0 +1,34 @@ +FROM centos:7 + +# Needed to build vespa +RUN yum -y install epel-release +RUN yum -y install centos-release-scl +RUN yum -y install devtoolset-4-gcc-c++ +RUN yum -y install devtoolset-4-libatomic-devel +RUN yum -y install Judy-devel +RUN yum -y install cmake3 +RUN yum -y install ccache +RUN yum -y install lz4-devel +RUN yum -y install zlib-devel +RUN yum -y install maven +RUN yum -y install libicu-devel +RUN yum -y install llvm-devel +RUN yum -y install llvm-static +RUN yum -y install java-1.8.0-openjdk-devel +RUN yum -y install openssl-devel +RUN yum -y install rpm-build +RUN yum -y install make + +# Install vespa dependencies +RUN yum-config-manager --add-repo https://copr.fedorainfracloud.org/coprs/g/vespa/vespa/repo/epel-7/group_vespa-vespa-epel-7.repo +RUN yum -y install vespa-boost-devel +RUN yum -y install vespa-libtorrent-devel +RUN yum -y install vespa-zookeeper-c-client-devel +RUN yum -y install vespa-cppunit-devel + +# Install utilities +RUN yum -y install sudo + +# Enable devtoolset-4 by default +RUN echo "source /opt/rh/devtoolset-4/enable" > /etc/profile.d/devtoolset-4.sh + diff --git a/docker/Dockerfile.run b/docker/Dockerfile.run new file mode 100644 index 00000000000..d82297ce676 --- /dev/null +++ b/docker/Dockerfile.run @@ -0,0 +1,27 @@ +FROM centos:7 + +# Needed to build vespa +RUN yum -y install epel-release +RUN yum -y install centos-release-scl +RUN yum -y install Judy +RUN yum -y install lz4 +RUN yum -y install zlib +RUN yum -y install libicu +RUN yum -y install llvm +RUN yum -y install java-1.8.0-openjdk +RUN yum -y install openssl +RUN yum -y install perl +RUN yum -y install perl-Env +RUN yum -y install perl-JSON +RUN yum -y install libatomic + +# Install vespa dependencies +RUN yum-config-manager --add-repo https://copr.fedorainfracloud.org/coprs/g/vespa/vespa/repo/epel-7/group_vespa-vespa-epel-7.repo +RUN yum -y install vespa-boost +RUN yum -y install vespa-libtorrent +RUN yum -y install vespa-zookeeper-c-client +RUN yum -y install vespa-cppunit # Should not be needed ? + +# Utilities +RUN yum -y install net-tools less + diff --git a/docker/README.md b/docker/README.md new file mode 100644 index 00000000000..899d9e78be9 --- /dev/null +++ b/docker/README.md @@ -0,0 +1,33 @@ + +# Building and running Vespa on Docker (OS X and Linux) + +## Installing docker +[Docker installation](https://docs.docker.com/engine/installation/) + +*On OS X, the native Docker engine (Beta) has NOT been tested. Please use the [Docker Toolbox](https://www.docker.com/products/docker-toolbox).* + +*On Linux, the default storage device is devicemapper with loopback device and max 10GB container size. This size is too small for a full build. Please see [here](http://www.projectatomic.io/blog/2016/03/daemon_option_basedevicesize/) and [here](http://www.projectatomic.io/blog/2015/06/notes-on-fedora-centos-and-docker-storage-drivers/) to overcome this limitation.* + +## Building the Vespa RPM +*On OS X, execute ```source osx-setup-docker-machine.sh``` to setup the Docker VM in which to run Docker.* + +Execute ```./build-vespa.sh <Vespa version number>``` to build Vespa from this source code. + +The produced rpms will be available in this folder after compiliation. + +## Running Vespa +*On OS X, execute ```source osx-setup-docker-machine.sh``` to setup the Docker VM in which to run Docker.* + +Execute ```./run-vespa.sh <Vespa version number>``` to start Vespa. + +This will create a Docker image which has the rpms from the build step installed. Vespa will be started inside the container. + +*On OS X, the container runs inside the Docker VM. Execute ```docker-machine ssh vespa-docker-machine``` to enter the VM.* + +## Building Vespa inside a Docker container +*On OS X, execute ```source osx-setup-docker-machine.sh``` to setup the Docker VM in which to run Docker.* + +Execute ```./enter-build-container.sh``` to enter the Vespa build environment inside a Docker container. + +The container is entered at the root of the Vespa source repository. Follow the build sections in [README.md](https://github.com/yahoo/vespa/blob/master/README.md) to build and test. + diff --git a/docker/build-vespa-internal.sh b/docker/build-vespa-internal.sh new file mode 100755 index 00000000000..f79e936c800 --- /dev/null +++ b/docker/build-vespa-internal.sh @@ -0,0 +1,18 @@ +#!/bin/bash +set -e + +if [ $# -ne 3 ]; then + echo "Usage: $0 <vespa version> <caller uid> <caller gid>" + echo "This script should not be called manually." + exit 1 +fi +VESPA_VERSION=$1 +CALLER_UID=$2 +CALLER_GID=$3 + +cd /vespa +./dist.sh ${VESPA_VERSION} +rpmbuild -bb ~/rpmbuild/SPECS/vespa-${VESPA_VERSION}.spec +chown ${CALLER_UID}:${CALLER_GID} ~/rpmbuild/RPMS/x86_64/*.rpm +mv ~/rpmbuild/RPMS/x86_64/*.rpm /vespa/docker + diff --git a/docker/build-vespa.sh b/docker/build-vespa.sh new file mode 100755 index 00000000000..6d3b1699bc5 --- /dev/null +++ b/docker/build-vespa.sh @@ -0,0 +1,17 @@ +#!/bin/bash +set -e + +if [ $# -ne 1 ]; then + echo "Usage: $0 <vespa version>" + exit 1 +fi + +DIR=$(cd $(dirname "${BASH_SOURCE[0]}") && pwd) +cd $DIR + +VESPA_VERSION=$1 +DOCKER_IMAGE="vespabuild" + +docker build -t "$DOCKER_IMAGE" -f Dockerfile.build . +docker run --rm -v $(pwd)/..:/vespa --entrypoint /vespa/docker/build-vespa-internal.sh "$DOCKER_IMAGE" "$VESPA_VERSION" "$(id -u)" "$(id -g)" + diff --git a/docker/enter-build-container-internal.sh b/docker/enter-build-container-internal.sh new file mode 100755 index 00000000000..7da96fde376 --- /dev/null +++ b/docker/enter-build-container-internal.sh @@ -0,0 +1,29 @@ +#!/bin/bash +set -e + +if [ $# -ne 0 ]; then + echo "Usage: $0" + echo "This script should not be called manually." + exit 1 +fi + +USERNAME=builder +DIR=$(cd $(dirname "${BASH_SOURCE[0]}") && pwd) +cd $DIR + +CALLER_UID=$(stat -c "%u" $DIR) +CALLER_GID=$(stat -c "%g" $DIR) + +groupadd -f -g $CALLER_GID $USERNAME +useradd -u $CALLER_UID -g $CALLER_GID $USERNAME +echo "$USERNAME ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers + +su -c "mkdir -p $DIR/../.ccache" $USERNAME +su -c "ln -sf $DIR/../.ccache /home/$USERNAME/.ccache" $USERNAME + +su -c "mkdir -p $DIR/../.m2" $USERNAME +su -c "ln -sf $DIR/../.m2 /home/$USERNAME/.m2" $USERNAME + +cd $DIR/.. +su $USERNAME + diff --git a/docker/enter-build-container.sh b/docker/enter-build-container.sh new file mode 100755 index 00000000000..ed7a2b4a130 --- /dev/null +++ b/docker/enter-build-container.sh @@ -0,0 +1,16 @@ +#!/bin/bash +set -e + +if [ $# -ne 0 ]; then + echo "Usage: $0" + exit 1 +fi + +DIR=$(cd $(dirname "${BASH_SOURCE[0]}") && pwd) +cd $DIR + +DOCKER_IMAGE="vespabuild" + +docker build -t "$DOCKER_IMAGE" -f Dockerfile.build . +docker run -ti --rm -v $(pwd)/..:/vespa --entrypoint /vespa/docker/enter-build-container-internal.sh "$DOCKER_IMAGE" + diff --git a/docker/osx-setup-docker-machine.sh b/docker/osx-setup-docker-machine.sh new file mode 100755 index 00000000000..ce27012ab46 --- /dev/null +++ b/docker/osx-setup-docker-machine.sh @@ -0,0 +1,42 @@ +#!/bin/bash + +DIR=$(cd $(dirname "${BASH_SOURCE[0]}") && pwd) +cd $DIR + +DOCKER_VM_NAME=vespa-docker-machine +DOCKER_VM_DISK_SIZE_IN_MB=40000 +DOCKER_VM_MEMORY_SIZE_IN_MB=4000 +DOCKER_VM_CPU_COUNT=4 + +DOCKER_VM_WAS_STARTED=false + +if ! docker-machine status "$DOCKER_VM_NAME" &> /dev/null; then + # Machine does not exist and we have to create and start + docker-machine create -d virtualbox \ + --virtualbox-disk-size "$DOCKER_VM_DISK_SIZE_IN_MB" \ + --virtualbox-memory "$DOCKER_VM_MEMORY_SIZE_IN_MB" \ + --virtualbox-cpu-count "$DOCKER_VM_CPU_COUNT" \ + "$DOCKER_VM_NAME" + + eval $(docker-machine env "$DOCKER_VM_NAME") + DOCKER_VM_WAS_STARTED=true +fi + + +VESPA_VM_STATUS=$(docker-machine status "$DOCKER_VM_NAME") +if [ "$VESPA_VM_STATUS" = "Stopped" ]; then + docker-machine start "$DOCKER_VM_NAME" + DOCKER_VM_WAS_STARTED=true + VESPA_VM_STATUS=$(docker-machine status "$DOCKER_VM_NAME") +fi + +if [ "$VESPA_VM_STATUS" != "Running" ]; then + echo "Unable to get Docker machine $DOCKER_VM_NAME up and running." + echo "You can try to manually remove the machine: docker-machine rm -y $DOCKER_VM_NAME " + echo " and then rerun this script." + echo "Exiting." + exit 1 +fi + +eval $(docker-machine env "$DOCKER_VM_NAME") + diff --git a/docker/run-vespa-internal.sh b/docker/run-vespa-internal.sh new file mode 100755 index 00000000000..5cbeb8ac2f8 --- /dev/null +++ b/docker/run-vespa-internal.sh @@ -0,0 +1,25 @@ +#!/bin/bash +set -e + +if [ $# -ne 1 ]; then + echo "Usage: $0 <vespa version>" + echo "This script should not be called manually." + exit 1 +fi + +DIR=$(cd $(dirname "${BASH_SOURCE[0]}") && pwd) +cd $DIR + +VESPA_VERSION=$1 + +rpm -i "vespa-${VESPA_VERSION}-1.el7.centos.x86_64.rpm" +rpm -i "vespa-debuginfo-${VESPA_VERSION}-1.el7.centos.x86_64.rpm" + +# Workaround until we figure out why rpm does not set the ownership. +chown -R vespa:vespa /opt/vespa + +/opt/vespa/bin/vespa-start-configserver +/opt/vespa/bin/vespa-start-services + +# Output log until killed +/opt/vespa/bin/logfmt -f diff --git a/docker/run-vespa.sh b/docker/run-vespa.sh new file mode 100755 index 00000000000..2ff511cc70b --- /dev/null +++ b/docker/run-vespa.sh @@ -0,0 +1,17 @@ +#!/bin/bash +set -e + +if [ $# -ne 1 ]; then + echo "Usage: $0 <vespa version>" + exit 1 +fi + +DIR=$(cd $(dirname "${BASH_SOURCE[0]}") && pwd) +cd $DIR + +VESPA_VERSION=$1 +DOCKER_IMAGE=vesparun + +docker build -t "$DOCKER_IMAGE" -f Dockerfile.run . +docker run -d -v $(pwd)/..:/vespa --entrypoint /vespa/docker/run-vespa-internal.sh "$DOCKER_IMAGE" "$VESPA_VERSION" + diff --git a/jdisc_akamai/OWNERS b/jdisc_akamai/OWNERS index 90fdb511ae3..6c536000692 100644 --- a/jdisc_akamai/OWNERS +++ b/jdisc_akamai/OWNERS @@ -1 +1,3 @@ bakksjo +gjoranv +bjorncs diff --git a/jdisc_container_maven_archetype_application/OWNERS b/jdisc_container_maven_archetype_application/OWNERS index 3b2ba1ede81..78b92e411b4 100644 --- a/jdisc_container_maven_archetype_application/OWNERS +++ b/jdisc_container_maven_archetype_application/OWNERS @@ -1 +1,2 @@ gjoranv +bjorncs diff --git a/jdisc_core/OWNERS b/jdisc_core/OWNERS index 90fdb511ae3..6c536000692 100644 --- a/jdisc_core/OWNERS +++ b/jdisc_core/OWNERS @@ -1 +1,3 @@ bakksjo +gjoranv +bjorncs diff --git a/jdisc_core/pom.xml b/jdisc_core/pom.xml index 7bd1a54d330..315a46a65c4 100644 --- a/jdisc_core/pom.xml +++ b/jdisc_core/pom.xml @@ -121,13 +121,7 @@ <scope>compile</scope> </dependency> <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>vespajlib</artifactId> - <version>${project.version}</version> - <scope>compile</scope> - </dependency> - <dependency> - <!-- Used for export-package parsing. Ideally, that should have been a separate artifact. --> + <!-- This seems odd. Used for export-package parsing. Lazy stuff. Should be separated out. --> <groupId>com.yahoo.vespa</groupId> <artifactId>bundle-plugin</artifactId> <version>${project.version}</version> @@ -212,7 +206,6 @@ <argument>${project.build.directory}/dependency/log4j-over-slf4j.jar</argument> <argument>${project.build.directory}/dependency/config-lib.jar</argument> <argument>${project.build.directory}/dependency/yolean.jar</argument> - <argument>${project.build.directory}/dependency/vespajlib.jar</argument> </arguments> </configuration> </execution> diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/core/StandaloneMain.java b/jdisc_core/src/main/java/com/yahoo/jdisc/core/StandaloneMain.java index 552d423f8cb..19b6c220f07 100644 --- a/jdisc_core/src/main/java/com/yahoo/jdisc/core/StandaloneMain.java +++ b/jdisc_core/src/main/java/com/yahoo/jdisc/core/StandaloneMain.java @@ -1,7 +1,7 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.jdisc.core; -import com.yahoo.system.CatchSigTerm; +import com.yahoo.yolean.system.CatchSigTerm; import java.util.Arrays; import java.util.concurrent.atomic.AtomicBoolean; @@ -36,16 +36,17 @@ public class StandaloneMain { void run(String bundleLocation) { try { - log.info("Initializing application without privileges."); - loader.init(bundleLocation, false); - loader.start(); - setupSigTermHandler(); - waitForShutdown(); - // Event.stopping(APPNAME, "shutdown"); - loader.stop(); - // Event.stopped(APPNAME, 0, 0); - loader.destroy(); - // System.exit(0); + System.out.println("debug\tInitializing application without privileges."); + loader.init(bundleLocation, false); + loader.start(); + setupSigTermHandler(); + waitForShutdown(); + System.out.println("debug\tTrying to shutdown in a controlled manner."); + loader.stop(); + System.out.println("debug\tTrying to clean up in a controlled manner."); + loader.destroy(); + System.out.println("debug\tStopped ok."); + System.exit(0); } catch (Exception e) { log.log(Level.WARNING, "Unexpected: ", e); System.exit(6); diff --git a/jdisc_core_test/OWNERS b/jdisc_core_test/OWNERS index 90fdb511ae3..6c536000692 100644 --- a/jdisc_core_test/OWNERS +++ b/jdisc_core_test/OWNERS @@ -1 +1,3 @@ bakksjo +gjoranv +bjorncs diff --git a/jdisc_http_service/OWNERS b/jdisc_http_service/OWNERS index 5255d2560bb..6c536000692 100644 --- a/jdisc_http_service/OWNERS +++ b/jdisc_http_service/OWNERS @@ -1,2 +1,3 @@ bakksjo gjoranv +bjorncs diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/HttpRequestDispatch.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/HttpRequestDispatch.java index e9aba0cb6c9..c16ac589332 100644 --- a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/HttpRequestDispatch.java +++ b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/HttpRequestDispatch.java @@ -101,6 +101,13 @@ class HttpRequestDispatch { HttpRequestDispatch parent = this; //used to avoid binding uninitialized variables completeRequestCallback = (result, error) -> { + boolean alreadyCalled = completeRequestCalled.getAndSet(true); + if (alreadyCalled) { + AssertionError e = new AssertionError("completeRequest called more than once"); + log.log(Level.WARNING, "Assertion failed.", e); + throw e; + } + boolean reportedError = false; if (error != null) { @@ -113,14 +120,6 @@ class HttpRequestDispatch { parent.metricReporter.successfulResponse(); } - - boolean alreadyCalled = completeRequestCalled.getAndSet(true); - if (alreadyCalled) { - AssertionError e = new AssertionError("completeRequest called more than once"); - log.log(Level.WARNING, "Assertion failed.", e); - throw e; - } - try { parent.async.complete(); log.finest(() -> "Request completed successfully: " + parent.servletRequest.getRequestURI()); diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/ServletOutputStreamWriter.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/ServletOutputStreamWriter.java index 271805765c2..7a15107a3eb 100644 --- a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/ServletOutputStreamWriter.java +++ b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/ServletOutputStreamWriter.java @@ -41,6 +41,10 @@ public class ServletOutputStreamWriter { private static final Logger log = Logger.getLogger(ServletOutputStreamWriter.class.getName()); + // TODO: This reference is not guaranteed to be unique; ByteBuffer.allocate(0) MAY in principle return a singleton! + // If so, application code could fake a close by writing such a byte buffer. + // The problem can be solved by filtering out zero-length byte buffers from application code. + // Other ways to express this are also possible, e.g. with a 'closed' state checked when queue goes empty. private static final ByteBuffer CLOSE_STREAM_BUFFER = ByteBuffer.allocate(0); private final Object monitor = new Object(); @@ -74,6 +78,7 @@ public class ServletOutputStreamWriter { public void setSendingError() { synchronized (monitor) { + // TODO: This assert seems fishy. Investigate. assertStateIs(state, State.NOT_STARTED); state = State.FINISHED_OR_ERROR; } @@ -109,6 +114,7 @@ public class ServletOutputStreamWriter { } if (thisThreadShouldWrite) { + // TODO: Consider refactoring to avoid multiple monitor entry-exit. writeBuffersInQueueToOutputStream(); } } diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/ServletRequestReader.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/ServletRequestReader.java index 5bea01bd104..a763a03d39d 100644 --- a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/ServletRequestReader.java +++ b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/ServletRequestReader.java @@ -133,9 +133,8 @@ class ServletRequestReader implements ReadListener { numberOfOutstandingUserCalls += 2; } try { - requestContentChannel.write(buf, writeCompletionHandler); - int bytesReceived = buf.remaining(); + requestContentChannel.write(buf, writeCompletionHandler); metricReporter.successfulRead(bytesReceived); } catch (final Throwable t) { finishedFuture.completeExceptionally(t); diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/ServletResponseController.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/ServletResponseController.java index b0781c402d5..126e4fee9e6 100644 --- a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/ServletResponseController.java +++ b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/ServletResponseController.java @@ -115,6 +115,8 @@ public class ServletResponseController { } try { + // TODO: sendError() is a synchronous call. Refactor. (Also, we should control the response content - + // this method generates a response body based on Jetty's own response templates ("Powered by Jetty"). servletResponse.sendError( statusCode, reasonPhrase); @@ -166,10 +168,14 @@ public class ServletResponseController { private static void setStatus_holdingLock(Response jdiscResponse, HttpServletResponse servletResponse) { if (jdiscResponse instanceof HttpResponse) { + // TODO: Figure out what this does to the response (with Jetty), and move to non-deprecated APIs. + // Deprecate our own code as necessary. servletResponse.setStatus(jdiscResponse.getStatus(), ((HttpResponse) jdiscResponse).getMessage()); } else { Optional<String> errorMessage = getErrorMessage(jdiscResponse); if (errorMessage.isPresent()) { + // TODO: Figure out what this does to the response (with Jetty), and move to non-deprecated APIs. + // Deprecate our own code as necessary. servletResponse.setStatus(jdiscResponse.getStatus(), errorMessage.get()); } else { servletResponse.setStatus(jdiscResponse.getStatus()); diff --git a/jdisc_jetty/OWNERS b/jdisc_jetty/OWNERS index 5255d2560bb..6c536000692 100644 --- a/jdisc_jetty/OWNERS +++ b/jdisc_jetty/OWNERS @@ -1,2 +1,3 @@ bakksjo gjoranv +bjorncs diff --git a/jdisc_jmx_metrics/OWNERS b/jdisc_jmx_metrics/OWNERS index 3b2ba1ede81..78b92e411b4 100644 --- a/jdisc_jmx_metrics/OWNERS +++ b/jdisc_jmx_metrics/OWNERS @@ -1 +1,2 @@ gjoranv +bjorncs diff --git a/jdisc_maven_archetype_component/OWNERS b/jdisc_maven_archetype_component/OWNERS index 3b2ba1ede81..78b92e411b4 100644 --- a/jdisc_maven_archetype_component/OWNERS +++ b/jdisc_maven_archetype_component/OWNERS @@ -1 +1,2 @@ gjoranv +bjorncs diff --git a/jdisc_messagebus_service/OWNERS b/jdisc_messagebus_service/OWNERS index 90fdb511ae3..6c536000692 100644 --- a/jdisc_messagebus_service/OWNERS +++ b/jdisc_messagebus_service/OWNERS @@ -1 +1,3 @@ bakksjo +gjoranv +bjorncs diff --git a/jdisc_status/OWNERS b/jdisc_status/OWNERS index d24d7c7860d..f02be9b1f86 100644 --- a/jdisc_status/OWNERS +++ b/jdisc_status/OWNERS @@ -1,2 +1,3 @@ gjoranv bakksjo +bjorncs diff --git a/node-admin/scripts/node-admin.sh b/node-admin/scripts/node-admin.sh index ae0fa94029b..cae3cf54d0d 100755 --- a/node-admin/scripts/node-admin.sh +++ b/node-admin/scripts/node-admin.sh @@ -123,7 +123,6 @@ function Start { --volume "/home/docker/container-storage/node-admin$VESPA_HOME/var/vespa:$VESPA_HOME/var/vespa" \ --volume "/home/docker/container-storage/node-admin$VESPA_HOME/var/yca:$VESPA_HOME/var/yca" \ --volume "/home/docker/container-storage/node-admin$VESPA_HOME/var/ycore++:$VESPA_HOME/var/ycore++" \ - --volume "/home/docker/container-storage/node-admin$VESPA_HOME/var/ymon:$VESPA_HOME/var/ymon" \ --volume "/home/docker/container-storage/node-admin$VESPA_HOME/var/zookeeper:$VESPA_HOME/var/zookeeper" \ --env "CONFIG_SERVER_ADDRESS=$CONFIG_SERVER_HOSTNAME" \ --env "NETWORK_TYPE=$NETWORK_TYPE" \ diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerImpl.java index 4ef7b1b0705..7d449c6d7f8 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerImpl.java @@ -51,8 +51,6 @@ import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; import java.util.logging.Level; import java.util.logging.Logger; -import java.util.regex.Matcher; -import java.util.regex.Pattern; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -65,8 +63,7 @@ public class DockerImpl implements Docker { private static final int SECONDS_TO_WAIT_BEFORE_KILLING = 10; private static final String FRAMEWORK_CONTAINER_PREFIX = "/"; - private static final String[] COMMAND_YINST_LS_VESPA = new String[]{"yinst", "ls", "vespa"}; - private static final Pattern VESPA_PACKAGE_VERSION_PATTERN = Pattern.compile("^vespa-(\\S+)", Pattern.MULTILINE); + static final String[] COMMAND_GET_VESPA_VERSION = new String[]{"vespa-nodectl", "vespa-version"}; private static final String LABEL_NAME_MANAGEDBY = "com.yahoo.vespa.managedby"; private static final String LABEL_VALUE_MANAGEDBY = "node-admin"; @@ -79,7 +76,7 @@ public class DockerImpl implements Docker { } private static final Path RELATIVE_APPLICATION_STORAGE_PATH = Paths.get("home/docker/container-storage"); - private static final Path RELATIVE_CLEANUP_APPLICATION_STORAGE_PATH = RELATIVE_APPLICATION_STORAGE_PATH.resolve("../container-storage-cleanup"); + private static final Path RELATIVE_CLEANUP_APPLICATION_STORAGE_PATH = RELATIVE_APPLICATION_STORAGE_PATH.resolve("cleanup"); private static final Path APPLICATION_STORAGE_PATH_FOR_NODE_ADMIN = Paths.get("/host").resolve(RELATIVE_APPLICATION_STORAGE_PATH); private static final Path CLEANUP_APPLICATION_STORAGE_PATH_FOR_NODE_ADMIN = Paths.get("/host").resolve(RELATIVE_CLEANUP_APPLICATION_STORAGE_PATH); private static final Path APPLICATION_STORAGE_PATH_FOR_HOST = Paths.get("/").resolve(RELATIVE_APPLICATION_STORAGE_PATH); @@ -101,7 +98,6 @@ public class DockerImpl implements Docker { getDefaults().underVespaHome("var/vespa"), getDefaults().underVespaHome("var/yca"), getDefaults().underVespaHome("var/ycore++"), - getDefaults().underVespaHome("var/ymon"), getDefaults().underVespaHome("var/zookeeper")); private final DockerClient docker; @@ -259,16 +255,13 @@ public class DockerImpl implements Docker { @Override public String getVespaVersion(final ContainerName containerName) { - ProcessResult result = executeInContainer(containerName, COMMAND_YINST_LS_VESPA); + ProcessResult result = executeInContainer(containerName, COMMAND_GET_VESPA_VERSION); if (!result.isSuccess()) { throw new RuntimeException("Container " + containerName.asString() + ": Command " - + Arrays.toString(COMMAND_YINST_LS_VESPA) + " failed: " + result); + + Arrays.toString(COMMAND_GET_VESPA_VERSION) + " failed: " + result); } - return parseVespaVersion(result.getOutput()) - .orElseThrow(() -> new RuntimeException( - "Container " + containerName.asString() + ": Failed to parse vespa version from " - + result.getOutput())); + return result.getOutput(); } @Override @@ -298,12 +291,6 @@ public class DockerImpl implements Docker { } } - // Returns empty if vespa version cannot be parsed. - static Optional<String> parseVespaVersion(final String outputFromYinstLsVespa) { - final Matcher matcher = VESPA_PACKAGE_VERSION_PATTERN.matcher(outputFromYinstLsVespa); - return matcher.find() ? Optional.of(matcher.group(1)) : Optional.empty(); - } - private void setupContainerNetworking(ContainerName containerName, HostName hostName, int containerPid) throws UnknownHostException { diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/ProcessResult.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/ProcessResult.java index 75d6f641feb..41312c28df7 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/ProcessResult.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/ProcessResult.java @@ -1,7 +1,6 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.node.admin.docker; -import java.io.IOException; import java.util.Objects; public class ProcessResult { diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/docker/DockerImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/docker/DockerImplTest.java index 9351e80cdab..9066019b2a2 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/docker/DockerImplTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/docker/DockerImplTest.java @@ -17,7 +17,6 @@ import java.util.Arrays; import java.util.Collections; import java.util.LinkedList; import java.util.List; -import java.util.Optional; import java.util.Set; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CountDownLatch; @@ -86,35 +85,6 @@ public class DockerImplTest { } @Test - public void vespaVersionIsParsed() { - assertThat(DockerImpl.parseVespaVersion("vespa-5.119.53"), is(Optional.of("5.119.53"))); - } - - @Test - public void vespaVersionIsParsedWithTrailingNewline() { - assertThat(DockerImpl.parseVespaVersion("vespa-5.119.53\n"), is(Optional.of("5.119.53"))); - } - - @Test(expected = NullPointerException.class) - public void vespaVersionIsNotParsedFromNull() { - assertThat(DockerImpl.parseVespaVersion(null), is(Optional.empty())); - } - - @Test - public void vespaVersionIsNotParsedFromEmptyString() { - assertThat(DockerImpl.parseVespaVersion(""), is(Optional.empty())); - } - - @Test - public void vespaVersionIsNotParsedFromUnexpectedContent() { - assertThat(DockerImpl.parseVespaVersion("honda-5.119.53"), is(Optional.empty())); - assertThat(DockerImpl.parseVespaVersion("vespa 5.119.53"), is(Optional.empty())); - assertThat(DockerImpl.parseVespaVersion("vespa- 5.119.53"), is(Optional.empty())); - assertThat(DockerImpl.parseVespaVersion("vespa-"), is(Optional.empty())); - assertThat(DockerImpl.parseVespaVersion("No such command 'yinst'"), is(Optional.empty())); - } - - @Test public void testExecuteCompletes() throws Exception { final DockerClient dockerClient = mock(DockerClient.class); final String containerId = "container-id"; diff --git a/sample-apps/basic-search/music-data-feed.json b/sample-apps/basic-search/music-data-feed.json index f896cdad75d..f38cbef9136 100644 --- a/sample-apps/basic-search/music-data-feed.json +++ b/sample-apps/basic-search/music-data-feed.json @@ -1,6 +1,6 @@ [ { - "put": "id:music:music::http://mymusic.com/Michael-Jackson-Bad", + "put": "id:music:music::Michael-Jackson-Bad", "fields": { "album": "Bad", "artist": "Michael Jackson", @@ -10,7 +10,7 @@ } }, { - "put": "id:music:music::http://mymusic.com/Eminem-Recovery", + "put": "id:music:music::Eminem-Recovery", "fields": { "album": "Recovery", "artist": "Eminem", diff --git a/standalone-container/pom.xml b/standalone-container/pom.xml index b211d196160..eb5665760c4 100644 --- a/standalone-container/pom.xml +++ b/standalone-container/pom.xml @@ -85,7 +85,8 @@ config-bundle-jar-with-dependencies.jar, config-model-api-jar-with-dependencies.jar, config-model-jar-with-dependencies.jar, - container-disc-jar-with-dependencies.jar + container-disc-jar-with-dependencies.jar, + vespajlib.jar </discPreInstallBundle> <bundleActivator>com.yahoo.container.standalone.StandaloneContainerActivator</bundleActivator> <jdiscPrivilegedActivator>true</jdiscPrivilegedActivator> diff --git a/vespabase/CMakeLists.txt b/vespabase/CMakeLists.txt index d37902f4258..75e3537506d 100644 --- a/vespabase/CMakeLists.txt +++ b/vespabase/CMakeLists.txt @@ -35,6 +35,10 @@ vespa_install_script(src/vespa-start-configserver.sh vespa-start-configserver bi vespa_install_script(src/vespa-start-services.sh vespa-start-services bin) vespa_install_script(src/vespa-stop-configserver.sh vespa-stop-configserver bin) vespa_install_script(src/vespa-stop-services.sh vespa-stop-services bin) -vespa_install_script(vespa-core-dumper.sh vespa-core-dumper bin) +vespa_install_script(src/vespa-core-dumper.sh vespa-core-dumper bin) + +configure_file(src/vespa.service.in src/vespa.service @ONLY) +configure_file(src/vespa-configserver.service.in src/vespa-configserver.service @ONLY) +install(FILES src/vespa.service src/vespa-configserver.service DESTINATION etc/systemd/system) install(FILES src/Defaults.pm DESTINATION lib/perl5/site_perl/Yahoo/Vespa) diff --git a/vespabase/src/.gitignore b/vespabase/src/.gitignore index a76bc1497b9..63aee122f89 100644 --- a/vespabase/src/.gitignore +++ b/vespabase/src/.gitignore @@ -1,2 +1,4 @@ build lib +/vespa-configserver.service +/vespa.service diff --git a/vespabase/src/vespa-configserver.service.in b/vespabase/src/vespa-configserver.service.in new file mode 100644 index 00000000000..8928fe87f20 --- /dev/null +++ b/vespabase/src/vespa-configserver.service.in @@ -0,0 +1,11 @@ +[Unit] +Description=Vertical Search Platform Config Server + +[Service] +Type=forking +PIDFile=@CMAKE_INSTALL_PREFIX@/var/run/configserver.pid +ExecStart=@CMAKE_INSTALL_PREFIX@/bin/vespa-start-configserver +ExecStop=@CMAKE_INSTALL_PREFIX@/bin/vespa-stop-configserver + +[Install] +WantedBy=multi-user.target
\ No newline at end of file diff --git a/vespabase/src/vespa.service.in b/vespabase/src/vespa.service.in new file mode 100644 index 00000000000..f46c7ecdcb6 --- /dev/null +++ b/vespabase/src/vespa.service.in @@ -0,0 +1,11 @@ +[Unit] +Description=Vertical Search Platform + +[Service] +Type=forking +PIDFile=@CMAKE_INSTALL_PREFIX@/var/run/sentinel.pid +ExecStart=@CMAKE_INSTALL_PREFIX@/bin/vespa-start-services +ExecStop=@CMAKE_INSTALL_PREFIX@/bin/vespa-stop-services + +[Install] +WantedBy=multi-user.target
\ No newline at end of file diff --git a/yolean/src/main/java/com/yahoo/yolean/system/CatchSigTerm.java b/yolean/src/main/java/com/yahoo/yolean/system/CatchSigTerm.java new file mode 100644 index 00000000000..204dcaacf28 --- /dev/null +++ b/yolean/src/main/java/com/yahoo/yolean/system/CatchSigTerm.java @@ -0,0 +1,69 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.yolean.system; + +import java.lang.reflect.*; + +// import sun.misc.Signal; +// import sun.misc.SignalHandler; + +import java.util.concurrent.atomic.AtomicBoolean; + +public class CatchSigTerm { + /** + * Sets up a signal handler for SIGTERM, where a given AtomicBoolean + * gets a true value when the TERM signal is caught. + * + * Callers basically have two options for acting on the TERM signal: + * + * They may choose to synchronize and wait() on this variable, + * and they will be notified when it changes state to true. To avoid + * problems with spurious wakeups, use a while loop and wait() + * again if the state is still false. As soon as the caller has been + * woken up and the state is true, the application should exit as + * soon as possible. + * + * They may also choose to poll the state of this variable. As soon + * as its state becomes true, the signal has been received, and the + * application should exit as soon as possible. + * + * @param signalCaught set to false initially, will be set to true when SIGTERM is caught. + */ + @SuppressWarnings("rawtypes") + public static void setup(final AtomicBoolean signalCaught) { + signalCaught.set(false); + try { + Class shc = Class.forName("sun.misc.SignalHandler"); + Class ssc = Class.forName("sun.misc.Signal"); + + InvocationHandler ihandler = new InvocationHandler() { + public Object invoke(Object proxy, Method method, Object[] args) throws Throwable { + synchronized (signalCaught) { + signalCaught.set(true); + signalCaught.notifyAll(); + } + return null; + } + }; + Object shandler = Proxy.newProxyInstance(CatchSigTerm.class.getClassLoader(), + new Class[] { shc }, + ihandler); + Constructor[] c = ssc.getDeclaredConstructors(); + assert c.length == 1; + Object sigterm = c[0].newInstance("TERM"); + Method m = findMethod(ssc, "handle"); + assert m != null; // "NoSuchMethodException" + m.invoke(null, sigterm, shandler); + } catch (ClassNotFoundException | InvocationTargetException | InstantiationException | IllegalAccessException e) { + System.err.println("FAILED setting up signal catching: "+e); + } + } + + private static Method findMethod(Class<?> c, String name) { + for (Method m : c.getDeclaredMethods()) { + if (m.getName().equals(name)) { + return m; + } + } + return null; + } +} diff --git a/yolean/src/main/java/com/yahoo/yolean/system/package-info.java b/yolean/src/main/java/com/yahoo/yolean/system/package-info.java new file mode 100644 index 00000000000..e727474ec52 --- /dev/null +++ b/yolean/src/main/java/com/yahoo/yolean/system/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.yolean.system; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/yolean/src/test/java/com/yahoo/yolean/system/CatchSigTermTestCase.java b/yolean/src/test/java/com/yahoo/yolean/system/CatchSigTermTestCase.java new file mode 100644 index 00000000000..d2ad84948e1 --- /dev/null +++ b/yolean/src/test/java/com/yahoo/yolean/system/CatchSigTermTestCase.java @@ -0,0 +1,19 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.yolean.system; + +import com.yahoo.yolean.system.CatchSigTerm; +import java.util.concurrent.atomic.AtomicBoolean; + +/** + * @author arnej27959 + */ +public class CatchSigTermTestCase extends junit.framework.TestCase { + + public CatchSigTermTestCase(String name) { + super(name); + } + + public void testThatSetupCompiles() { + CatchSigTerm.setup(new AtomicBoolean(false)); + } +} |