From 72231250ed81e10d66bfe70701e64fa5fe50f712 Mon Sep 17 00:00:00 2001 From: Jon Bratseth Date: Wed, 15 Jun 2016 23:09:44 +0200 Subject: Publish --- node-admin/.gitignore | 3 + node-admin/Dockerfile.template | 36 + node-admin/OWNERS | 2 + node-admin/README.md | 111 + node-admin/README_LINUX.md | 129 ++ node-admin/README_MAC.md | 73 + node-admin/build.sh | 36 + node-admin/include/config-ctl.patch | 5 + node-admin/include/deploy-music-app.sh | 85 + node-admin/include/music-on-docker-services.xml | 18 + node-admin/include/node-flavors.xml | 15 + node-admin/include/nodectl-instance.sh | 138 ++ node-admin/include/root-bashrc | 1 + node-admin/include/start-config-server.sh | 138 ++ node-admin/include/start-services.sh | 98 + node-admin/pom.xml | 162 ++ node-admin/scripts/app.sh | 217 ++ node-admin/scripts/common-vm.sh | 13 + node-admin/scripts/common.sh | 180 ++ node-admin/scripts/config-server.sh | 140 ++ .../scripts/configure-container-networking.py | 311 +++ node-admin/scripts/etc-hosts.sh | 117 + node-admin/scripts/ipaddress.py | 2411 ++++++++++++++++++++ node-admin/scripts/make-host-like-container.sh | 52 + node-admin/scripts/network-bridge.sh | 63 + node-admin/scripts/node-admin.sh | 135 ++ node-admin/scripts/node-repo.sh | 318 +++ .../scripts/populate-noderepo-with-local-nodes.sh | 44 + node-admin/scripts/pyroute2/__init__.py | 95 + node-admin/scripts/pyroute2/arp.py | 69 + node-admin/scripts/pyroute2/common.py | 288 +++ node-admin/scripts/pyroute2/config.py | 10 + node-admin/scripts/pyroute2/debugger.py | 85 + node-admin/scripts/pyroute2/dhcp/__init__.py | 300 +++ node-admin/scripts/pyroute2/dhcp/dhcp4msg.py | 60 + node-admin/scripts/pyroute2/dhcp/dhcp4socket.py | 135 ++ node-admin/scripts/pyroute2/ipdb/__init__.py | 981 ++++++++ node-admin/scripts/pyroute2/ipdb/common.py | 51 + node-admin/scripts/pyroute2/ipdb/interface.py | 709 ++++++ node-admin/scripts/pyroute2/ipdb/linkedset.py | 134 ++ node-admin/scripts/pyroute2/ipdb/route.py | 354 +++ node-admin/scripts/pyroute2/ipdb/transactional.py | 402 ++++ node-admin/scripts/pyroute2/iproute.py | 888 +++++++ node-admin/scripts/pyroute2/ipset.py | 149 ++ node-admin/scripts/pyroute2/iwutil.py | 355 +++ node-admin/scripts/pyroute2/netlink/__init__.py | 1349 +++++++++++ .../scripts/pyroute2/netlink/generic/__init__.py | 74 + .../scripts/pyroute2/netlink/ipq/__init__.py | 131 ++ .../scripts/pyroute2/netlink/nfnetlink/__init__.py | 33 + .../scripts/pyroute2/netlink/nfnetlink/ipset.py | 77 + .../scripts/pyroute2/netlink/nl80211/__init__.py | 609 +++++ node-admin/scripts/pyroute2/netlink/nlsocket.py | 856 +++++++ .../scripts/pyroute2/netlink/rtnl/__init__.py | 156 ++ node-admin/scripts/pyroute2/netlink/rtnl/errmsg.py | 11 + node-admin/scripts/pyroute2/netlink/rtnl/fibmsg.py | 60 + .../scripts/pyroute2/netlink/rtnl/ifaddrmsg.py | 96 + .../scripts/pyroute2/netlink/rtnl/ifinfmsg.py | 1068 +++++++++ .../scripts/pyroute2/netlink/rtnl/iprsocket.py | 164 ++ .../scripts/pyroute2/netlink/rtnl/iw_event.py | 85 + node-admin/scripts/pyroute2/netlink/rtnl/ndmsg.py | 61 + node-admin/scripts/pyroute2/netlink/rtnl/req.py | 182 ++ node-admin/scripts/pyroute2/netlink/rtnl/rtmsg.py | 90 + node-admin/scripts/pyroute2/netlink/rtnl/tcmsg.py | 917 ++++++++ .../scripts/pyroute2/netlink/taskstats/__init__.py | 167 ++ node-admin/scripts/pyroute2/netns/__init__.py | 123 + node-admin/scripts/pyroute2/netns/nslink.py | 310 +++ .../scripts/pyroute2/netns/process/__init__.py | 39 + .../scripts/pyroute2/netns/process/base_p2.py | 7 + .../scripts/pyroute2/netns/process/base_p3.py | 7 + node-admin/scripts/pyroute2/netns/process/proxy.py | 163 ++ node-admin/scripts/pyroute2/protocols/__init__.py | 234 ++ node-admin/scripts/pyroute2/protocols/rawsocket.py | 70 + node-admin/scripts/pyroute2/proxy.py | 65 + node-admin/scripts/route-osx.sh | 16 + node-admin/scripts/setup-docker.sh | 176 ++ node-admin/scripts/setup-route-and-hosts-osx.sh | 20 + node-admin/scripts/vm.sh | 77 + node-admin/scripts/zone.sh | 80 + node-admin/src/main/application/services.xml | 17 + .../vespa/hosted/node/admin/ContainerNodeSpec.java | 93 + .../yahoo/vespa/hosted/node/admin/NodeAdmin.java | 155 ++ .../hosted/node/admin/NodeAdminScheduler.java | 144 ++ .../yahoo/vespa/hosted/node/admin/NodeAgent.java | 33 + .../vespa/hosted/node/admin/NodeAgentImpl.java | 404 ++++ .../vespa/hosted/node/admin/docker/Container.java | 54 + .../hosted/node/admin/docker/ContainerName.java | 44 + .../vespa/hosted/node/admin/docker/Docker.java | 49 + .../hosted/node/admin/docker/DockerImage.java | 44 + .../vespa/hosted/node/admin/docker/DockerImpl.java | 589 +++++ .../hosted/node/admin/docker/ProcessResult.java | 44 + .../hosted/node/admin/docker/package-info.java | 5 + .../node/admin/noderepository/NodeRepository.java | 28 + .../admin/noderepository/NodeRepositoryImpl.java | 150 ++ .../node/admin/noderepository/NodeState.java | 7 + .../noderepository/bindings/GetNodesResponse.java | 73 + .../noderepository/bindings/NodeRepositoryApi.java | 35 + .../bindings/UpdateNodeAttributesRequestBody.java | 31 + .../bindings/UpdateNodeAttributesResponse.java | 17 + .../node/admin/orchestrator/Orchestrator.java | 21 + .../admin/orchestrator/OrchestratorException.java | 9 + .../node/admin/orchestrator/OrchestratorImpl.java | 101 + .../node/admin/restapi/NodeAdminRestAPI.java | 15 + .../hosted/node/admin/testapi/PingResource.java | 20 + .../vespa/hosted/node/admin/util/Environment.java | 44 + .../main/resources/configdefinitions/docker.def | 7 + .../vespa/hosted/node/admin/NodeAdminTest.java | 194 ++ .../vespa/hosted/node/admin/NodeAgentImplTest.java | 1073 +++++++++ .../hosted/node/admin/docker/DockerImplTest.java | 322 +++ .../node/admin/docker/ProcessResultTest.java | 25 + .../noderepository/NodeRepositoryImplTest.java | 98 + .../admin/orchestrator/OrchestratorImplTest.java | 50 + 111 files changed, 21684 insertions(+) create mode 100644 node-admin/.gitignore create mode 100644 node-admin/Dockerfile.template create mode 100644 node-admin/OWNERS create mode 100644 node-admin/README.md create mode 100644 node-admin/README_LINUX.md create mode 100644 node-admin/README_MAC.md create mode 100755 node-admin/build.sh create mode 100644 node-admin/include/config-ctl.patch create mode 100755 node-admin/include/deploy-music-app.sh create mode 100644 node-admin/include/music-on-docker-services.xml create mode 100644 node-admin/include/node-flavors.xml create mode 100755 node-admin/include/nodectl-instance.sh create mode 100644 node-admin/include/root-bashrc create mode 100755 node-admin/include/start-config-server.sh create mode 100755 node-admin/include/start-services.sh create mode 100644 node-admin/pom.xml create mode 100755 node-admin/scripts/app.sh create mode 100644 node-admin/scripts/common-vm.sh create mode 100644 node-admin/scripts/common.sh create mode 100755 node-admin/scripts/config-server.sh create mode 100755 node-admin/scripts/configure-container-networking.py create mode 100755 node-admin/scripts/etc-hosts.sh create mode 100644 node-admin/scripts/ipaddress.py create mode 100755 node-admin/scripts/make-host-like-container.sh create mode 100755 node-admin/scripts/network-bridge.sh create mode 100755 node-admin/scripts/node-admin.sh create mode 100755 node-admin/scripts/node-repo.sh create mode 100755 node-admin/scripts/populate-noderepo-with-local-nodes.sh create mode 100644 node-admin/scripts/pyroute2/__init__.py create mode 100644 node-admin/scripts/pyroute2/arp.py create mode 100644 node-admin/scripts/pyroute2/common.py create mode 100644 node-admin/scripts/pyroute2/config.py create mode 100644 node-admin/scripts/pyroute2/debugger.py create mode 100644 node-admin/scripts/pyroute2/dhcp/__init__.py create mode 100644 node-admin/scripts/pyroute2/dhcp/dhcp4msg.py create mode 100644 node-admin/scripts/pyroute2/dhcp/dhcp4socket.py create mode 100644 node-admin/scripts/pyroute2/ipdb/__init__.py create mode 100644 node-admin/scripts/pyroute2/ipdb/common.py create mode 100644 node-admin/scripts/pyroute2/ipdb/interface.py create mode 100644 node-admin/scripts/pyroute2/ipdb/linkedset.py create mode 100644 node-admin/scripts/pyroute2/ipdb/route.py create mode 100644 node-admin/scripts/pyroute2/ipdb/transactional.py create mode 100644 node-admin/scripts/pyroute2/iproute.py create mode 100644 node-admin/scripts/pyroute2/ipset.py create mode 100644 node-admin/scripts/pyroute2/iwutil.py create mode 100644 node-admin/scripts/pyroute2/netlink/__init__.py create mode 100644 node-admin/scripts/pyroute2/netlink/generic/__init__.py create mode 100644 node-admin/scripts/pyroute2/netlink/ipq/__init__.py create mode 100644 node-admin/scripts/pyroute2/netlink/nfnetlink/__init__.py create mode 100644 node-admin/scripts/pyroute2/netlink/nfnetlink/ipset.py create mode 100644 node-admin/scripts/pyroute2/netlink/nl80211/__init__.py create mode 100644 node-admin/scripts/pyroute2/netlink/nlsocket.py create mode 100644 node-admin/scripts/pyroute2/netlink/rtnl/__init__.py create mode 100644 node-admin/scripts/pyroute2/netlink/rtnl/errmsg.py create mode 100644 node-admin/scripts/pyroute2/netlink/rtnl/fibmsg.py create mode 100644 node-admin/scripts/pyroute2/netlink/rtnl/ifaddrmsg.py create mode 100644 node-admin/scripts/pyroute2/netlink/rtnl/ifinfmsg.py create mode 100644 node-admin/scripts/pyroute2/netlink/rtnl/iprsocket.py create mode 100644 node-admin/scripts/pyroute2/netlink/rtnl/iw_event.py create mode 100644 node-admin/scripts/pyroute2/netlink/rtnl/ndmsg.py create mode 100644 node-admin/scripts/pyroute2/netlink/rtnl/req.py create mode 100644 node-admin/scripts/pyroute2/netlink/rtnl/rtmsg.py create mode 100644 node-admin/scripts/pyroute2/netlink/rtnl/tcmsg.py create mode 100644 node-admin/scripts/pyroute2/netlink/taskstats/__init__.py create mode 100644 node-admin/scripts/pyroute2/netns/__init__.py create mode 100644 node-admin/scripts/pyroute2/netns/nslink.py create mode 100644 node-admin/scripts/pyroute2/netns/process/__init__.py create mode 100644 node-admin/scripts/pyroute2/netns/process/base_p2.py create mode 100644 node-admin/scripts/pyroute2/netns/process/base_p3.py create mode 100644 node-admin/scripts/pyroute2/netns/process/proxy.py create mode 100644 node-admin/scripts/pyroute2/protocols/__init__.py create mode 100644 node-admin/scripts/pyroute2/protocols/rawsocket.py create mode 100644 node-admin/scripts/pyroute2/proxy.py create mode 100755 node-admin/scripts/route-osx.sh create mode 100755 node-admin/scripts/setup-docker.sh create mode 100755 node-admin/scripts/setup-route-and-hosts-osx.sh create mode 100755 node-admin/scripts/vm.sh create mode 100755 node-admin/scripts/zone.sh create mode 100644 node-admin/src/main/application/services.xml create mode 100644 node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/ContainerNodeSpec.java create mode 100644 node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/NodeAdmin.java create mode 100644 node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/NodeAdminScheduler.java create mode 100644 node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/NodeAgent.java create mode 100644 node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/NodeAgentImpl.java create mode 100644 node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/Container.java create mode 100644 node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/ContainerName.java create mode 100644 node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/Docker.java create mode 100644 node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerImage.java create mode 100644 node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerImpl.java create mode 100644 node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/ProcessResult.java create mode 100644 node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/package-info.java create mode 100644 node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/noderepository/NodeRepository.java create mode 100644 node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/noderepository/NodeRepositoryImpl.java create mode 100644 node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/noderepository/NodeState.java create mode 100644 node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/noderepository/bindings/GetNodesResponse.java create mode 100644 node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/noderepository/bindings/NodeRepositoryApi.java create mode 100644 node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/noderepository/bindings/UpdateNodeAttributesRequestBody.java create mode 100644 node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/noderepository/bindings/UpdateNodeAttributesResponse.java create mode 100644 node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/orchestrator/Orchestrator.java create mode 100644 node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/orchestrator/OrchestratorException.java create mode 100644 node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/orchestrator/OrchestratorImpl.java create mode 100644 node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/restapi/NodeAdminRestAPI.java create mode 100644 node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/testapi/PingResource.java create mode 100644 node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/util/Environment.java create mode 100644 node-admin/src/main/resources/configdefinitions/docker.def create mode 100644 node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/NodeAdminTest.java create mode 100644 node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/NodeAgentImplTest.java create mode 100644 node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/docker/DockerImplTest.java create mode 100644 node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/docker/ProcessResultTest.java create mode 100644 node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/noderepository/NodeRepositoryImplTest.java create mode 100644 node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/orchestrator/OrchestratorImplTest.java (limited to 'node-admin') diff --git a/node-admin/.gitignore b/node-admin/.gitignore new file mode 100644 index 00000000000..51bc5191fc6 --- /dev/null +++ b/node-admin/.gitignore @@ -0,0 +1,3 @@ +/dependencies +/Dockerfile +/**/*.pyc diff --git a/node-admin/Dockerfile.template b/node-admin/Dockerfile.template new file mode 100644 index 00000000000..485518cb86a --- /dev/null +++ b/node-admin/Dockerfile.template @@ -0,0 +1,36 @@ +# NOTE: Whenever the vespa-base version changes, review pom.xml. A fixed version of vespa-base contains components +# built from a certain vespa revision, while the docker image you will build with this Dockerfile contains components +# (i.e. the node-admin jar file) built from HEAD. Since dependencies change over time, there may be mismatches. +# Therefore, one of the following may be necessary for one or more dependencies in pom.xml: +# a) Hard-coding a version override. +# b) Changing a hard-coded version override. +# c) Removing a hard-coded version override. +# +# See build.sh for the meaning of $NODE_ADMIN_FROM_IMAGE. +FROM $NODE_ADMIN_FROM_IMAGE + +# Be aware that this Dockerfile is not being used in any pipelines and has no relation to production environments etc. +# It is here for developers' convenience - it allows building and experimenting with node-admin locally. + +# Things for convenience. +RUN yum install -y tcpdump +ADD include/root-bashrc /root/.bashrc + +# Override what's in the base image with local versions: +ADD target/node-admin-jar-with-dependencies.jar $VESPA_HOME/lib/jars/node-admin-jar-with-dependencies.jar +ADD src/main/application/services.xml $VESPA_HOME/conf/node-admin-app/services.xml +ADD scripts/configure-container-networking.py $VESPA_HOME/libexec/vespa/node-admin/configure-container-networking.py + +# For deploying sample application. +ADD include/deploy-music-app.sh /usr/local/bin/deploy-music-app.sh +ADD include/music-on-docker-services.xml $VESPA_HOME/share/vespa/sampleapps/search/music/services.xml + +# Entrypoint for running config server in a container. +ADD include/start-config-server.sh /usr/local/bin/start-config-server.sh + +# Included in base image, but here overridden with local modifications. +# TODO: Update the source instead. +ADD include/start-services.sh /usr/local/bin/start-services.sh + +# Make config-server aware of node flavor 'docker'. +ADD include/node-flavors.xml $VESPA_HOME/conf/configserver-app/node-flavors.xml diff --git a/node-admin/OWNERS b/node-admin/OWNERS new file mode 100644 index 00000000000..9ecc8472a21 --- /dev/null +++ b/node-admin/OWNERS @@ -0,0 +1,2 @@ +bakksjo +hakon diff --git a/node-admin/README.md b/node-admin/README.md new file mode 100644 index 00000000000..b727dde7bb8 --- /dev/null +++ b/node-admin/README.md @@ -0,0 +1,111 @@ +# Node Admin + +## Setup + +Set up Docker on your machine according to the instructions in README_LINUX or README_MAC, depending on your hardware. + +You should have the docker daemon running and the following environment variables set: +``` +DOCKER_HOST +CONTAINER_CERT_PATH +``` + +## Building + +Build Node Admin and include it (and other local modifications) in the Docker image ```vespa-local```: +``` +mvn clean package +./build.sh +``` + +## Running + +TODO: Outdated! Update this section with info on how to run everything locally. + +Start the container for the config server (TODO). Set the CONFIG_SERVER_ADDRESS +variable to the hostname of the config server. + +Start the container +``` +docker run -t -i --privileged \ + -p 4080:4080 \ + -v $CONTAINER_CERT_PATH:/host/docker/certs \ + -e "DOCKER_HOST=$DOCKER_HOST" \ + -e "CONFIG_SERVER_ADDRESS=$CONFIG_SERVER_ADDRESS" \ + vespa-local:latest +``` + +This will map the client certificate/key files to the path where Node Admin looks for them (as configured in +services.xml), and enable Node Admin to talk to the docker daemon. You can invoke Node Admin's REST APIs on port 4080 +from both inside the container and the outside host. + +## Using + +Trigger the incredibly rich and complex node-admin REST API(s) +``` +curl localhost:4080/test/ping +``` + +## Troubleshooting + +If the container doesn't start, it can be helpful to look at the jdisc log. First, find the container id: +``` +docker ps -a +``` + +Then, find the log files: +``` +docker diff | grep $VESPA_HOME/logs +``` + +View the log file (`-L` follows the symbolic link): +``` +docker cp -L :$VESPA_HOME/logs/jdisc_core/jdisc_core.log - | less +``` + +## Developing + +We will describe how you can build a Docker image for Vespa which will be used +to set up a local Docker container with the Node Admin, and a local container +with the Config Server. + +Then, we'll show how you bring up this local zone. And finally, how you can +deploy a local Vespa application to this zone. + +### Building Local Docker Image + +A Dockerfile exists in the module's root directory. This Dockerfile is not used +in production or any pipelines, it is here for convenience so you can build +images and experiment locally. See build.sh for how to build. + +The image created by the Dockerfile will be used to run Node Admin or a Config +Server. + +### Starting a Local Zone + +To start a local zone, ensure your operating system ignores ```config-server``` +and ```node-admin``` for proxying. Then issue the following command: + +``` +scripts/zone.sh start +``` + +The Node Admin and Config Server now runs in the ```node-admin``` and +```config-server``` Docker containers. These containers have their own IP +addresses and hostnames (also ```node-admin``` and ```config-server```). + +### Deploying a Local Application + +To deploy an application, use ```scripts/app.sh```. Assuming you have checked +out ```vespa/basic-search-for-docker``` to ```~```, and packaged it with ```mvn +package```, you can deploy the application with: + +``` +scripts/app.sh deploy ~/vespa/basic-search-for-docker/target/application +``` + +You can undeploy it with + +``` +scripts/app.sh undeploy +``` diff --git a/node-admin/README_LINUX.md b/node-admin/README_LINUX.md new file mode 100644 index 00000000000..b369044eef5 --- /dev/null +++ b/node-admin/README_LINUX.md @@ -0,0 +1,129 @@ +# Setting up Docker on a linux machine + +First, install Docker. With Fedora 21 you should follow +https://docs.docker.com/installation/fedora/, which describes how to install +Docker, start the Docker daemon, verify you can download images and run them, +and making your user run docker instances. + +``` +sudo systemctl enable docker +``` + +## Set up yahoo user + +The Vespa docker containers will run images that need to access the host file +system as the user 'yahoo' with UID 1000, e.g. the Node Admin runs as this +user. If this UID is not already taken, you can create a yahoo user as follows: + +``` +sudo useradd -u 1000 -g 100 -s /dev/null yahoo +``` + +If the UID is already in use you should move the user to a new UID first. +Alternatively, it might be possible to reuse that user, but this is confusing +and may lead to errors later on (and has not been tested). In the following +example we move the 'landesk' user from UID 1000 to 1010, keeping its GID 1001. + +``` +sudo usermod -u 1010 landesk +sudo find / -user 1000 -exec chown -h 1010 {} \; +``` + +## Set up image storage (aka don't break your machine) + +Docker will by default download (huge) images to a directory under /var. On our +Fedora machines, /var is part of the root filesystem, which does not have a lot +of free space. Since docker runs as root, it is allowed to completely fill up +the filesystem, and it will happily do so. Fedora works very poorly with a full +root filesystem. You won't even be able to log in and clean up the disk usage +once it's happened. + +So you'll want to store images somewhere else. An obvious choice is /home, +which typically has a lot more room. Make Docker use directories in the docker +user's home directory. Run the following script to do this: + +``` +scripts/setup-docker.sh home +``` + +## Set up TLS + +By default, the docker CLI communicates with the docker daemon via a unix +socket. This is fine in itself, but not suffficient for our use. Node Admin, +itself running in a container, will talk to the docker daemon to start +containers for vespa nodes. Node Admin uses a Java library for communication +with the docker daemon, and this library depends on JNI (native) code for unix +socket communication. We don't want that, so that dependency has been +excluded. Therefore, Node Admin uses TLS over IP to communicate with the docker +daemon. You must therefore set up docker with TLS. Mostly, you can follow the +instructions at https://docs.docker.com/articles/https/. + +The commands can be run with + +``` +scripts/setup-docker.sh certs +``` + +Note the following: + + - You will be asked to generate a key, and will repeatedly be asked for it. + - Use your fully qualified domain name for Common Name. + +Now, you need to tell the docker daemon to use TLS. Edit the file ```/lib/systemd/system/docker.service``` and change +the ExecStart line so it includes the following arguments: +``` +--tlsverify --tlscacert=/etc/dockercert_daemon/ca_cert.pem --tlscert=/etc/dockercert_daemon/server_cert.pem --tlskey=/etc/dockercert_daemon/server_key.pem -H=0.0.0.0:2376 +``` + +Then restart docker: +``` +sudo systemctl daemon-reload +sudo systemctl restart docker +``` + +Now tell the docker CLI how to communicate with the docker daemon: +``` +export DOCKER_HOST=tcp://127.0.0.1:2376 +export DOCKER_TLS_VERIFY=1 +export DOCKER_CERT_PATH=/etc/dockercert_cli +``` + +(You might want to add this to your .bashrc file.) + +Now, test that the docker cli can talk to the docker daemon: +``` +docker version +docker run --rm hello-world +``` + +These should run without errors. Finally, to run Node Admin locally, it needs access to the certificate/key files. +``` +export CONTAINER_CERT_PATH=/etc/dockercert_container +``` + +This environment variable will be used when starting the container, which is decribed in the platform-independent +README file. + +While docker can and should be run as your user, it's nice to make it possible +to run docker under root too. To enable this you must make sure sudo doesn't +strip off the environment variables, otherwise certain docker commands may +hang. Add a file /etc/sudoers.d/passthrough-docker-env with the content: + +``` +Defaults env_keep += "DOCKER_HOST DOCKER_TLS_VERIFY DOCKER_CERT_PATH CONTAINER_CERT_PATH" +``` + +You are now done with the linux-specific setup work. + +## Other + +For more information on how to configure the docker daemon, see https://docs.docker.com/articles/systemd/. + +## Upgrade of Docker + +When Docker upgrades it may overwrite /lib/systemd/system/docker.service. The +symptom is that any docker command will fail with the error message "Cannot +connect to the Docker daemon. Is the docker daemon running on this host?". + +Once you have updated docker.service according to this document, and restarted +the Docker daemon, Docker should work again. diff --git a/node-admin/README_MAC.md b/node-admin/README_MAC.md new file mode 100644 index 00000000000..75a67f6a29c --- /dev/null +++ b/node-admin/README_MAC.md @@ -0,0 +1,73 @@ +# Setting up Docker on OS X +Install Docker Toolbox according to the procedure on [https://docs.docker.com/mac/step_one](https://docs.docker.com/mac/step_one). + +# Running Vespa on OS X + +## Starting the VM +On OS X the docker daemon is running inside a VM called boot2docker. This VM is running using the VirtualBox virtualization software. To setup and start the VM, execute the following script: + +``` +scripts/vm.sh +``` +You should now have a Docker machine up and running. This can be verified with: + +``` +docker-machine ls +``` +Which should list the running vespa machine. + +## Building the Vespa Docker image +Building node-admin requires that the vespa Docker machine is up and running. This is because the building of the Docker image actually happens inside the VM. + +First we need to make sure that some environment variables are set so that the ```docker``` command knows how to communicate with the VM: + +``` +eval $(docker-machine env vespa) +``` + +To build the image, follow the instructions in [README.md](README.md). + +The Vespa Docker image will be persisted inside the VM and it is not necessary to rebuild the image if you stop and restart the VM. However, if you remove the VM with ```docker-machine rm vespa```, the image must be rebuilt. + +## Running Vespa with the node-admin scripts +The scripts that are used for starting local zones and deploying applications in Linux can be used in OS X by prefixing them with ```scripts/vm.sh ```. + +Follow the instructions in [README.md](README.md) for starting local zones and deploying applications. + +## Accessing Vespa directly from OS X +The ```scripts/vm.sh``` script does some of the network setup inside the VM that is required for this to work. However, it is necessary set up routing and the ```/etc/hosts``` file to get this working. To automatically do this, execute the script: + +``` +scripts/setup-route-and-hosts-osx.sh +``` +The script will prompt you to continue as this will alter your routing table and /etc/hosts file. If your local zone is up and running, the config server should respond to this: + +``` +curl config-server:19071 +``` + +If you don't want your `/etc/hosts` file to be changed, the +`scripts/route-osx.sh` script can be used instead. This means that you must +inspect `/etc/hosts` inside the VM to find the IP address of each container: +`docker-machine ssh cat /etc/hosts` + +## Useful Docker commands +Obtain the values for the required environment variables with: + +``` +eval $(docker-machine env vespa) +``` + +How to log onto the Docker base host: + +``` +docker-machine ssh vespa +``` + +Regular ```docker``` commands works as in Linux when you have the environment variables set. +Look in [README.md](README.md) for useful docker commands. + +## Issues +* Accessing Vespa from OS X while on a Cisco VPN connection does not work. This is because the VPN client will protect the routing table on OS X. + * Workaround is to use ```docker-machine ssh vespa``` and then execute everything from inside the VM. + diff --git a/node-admin/build.sh b/node-admin/build.sh new file mode 100755 index 00000000000..f0476c6a30b --- /dev/null +++ b/node-admin/build.sh @@ -0,0 +1,36 @@ +#!/bin/bash +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +set -e + +function Usage { + cat <&2 +$* + +Usage: build.sh +Builds the local node-admin docker image used in the local zone. + +You must set the NODE_ADMIN_FROM_IMAGE environment variable to point to the +base image (FROM-line in Dockerfile) you'd like to build the node admin image +on. +EOF + exit 1 +} + +if [ -z "$NODE_ADMIN_FROM_IMAGE" ] +then + Usage "NODE_ADMIN_FROM_IMAGE environment variable is not set." +elif [[ "$NODE_ADMIN_FROM_IMAGE" =~ % ]] +then + Usage "NODE_ADMIN_FROM_IMAGE environment variable cannot contain the %-character." +elif [ -z "$VESPA_HOME" ] +then + Usage "VESPA_HOME environment variable is not set." +fi + +cat Dockerfile.template | \ + sed 's%$NODE_ADMIN_FROM_IMAGE%'"$NODE_ADMIN_FROM_IMAGE%g" | \ + sed 's%$VESPA_HOME%'"$VESPA_HOME%g" \ + > Dockerfile + +docker build --tag="vespa-local:latest" . diff --git a/node-admin/include/config-ctl.patch b/node-admin/include/config-ctl.patch new file mode 100644 index 00000000000..fdb6882a375 --- /dev/null +++ b/node-admin/include/config-ctl.patch @@ -0,0 +1,5 @@ +36,39d35 +< case $hname in +< *.*.*) ;; +< *) echo "The hostname must be a FQDN, was: $hname" ; exit 1 ;; +< esac diff --git a/node-admin/include/deploy-music-app.sh b/node-admin/include/deploy-music-app.sh new file mode 100755 index 00000000000..1a13725213d --- /dev/null +++ b/node-admin/include/deploy-music-app.sh @@ -0,0 +1,85 @@ +#!/bin/bash +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +# +# Usage: docker exec config-server /usr/local/bin/deploy-music-app.sh +# Deploy app to config-server running in local Docker zone +# +# You must build the vespa-local:latest (vespa/vespa/node-admin) image and +# (re)start the local zone, before running deploy-music-app.sh. +# +# See also app.sh + +# BEGIN environment bootstrap section +# Do not edit between here and END as this section should stay identical in all scripts + +findpath () { + myname=${0} + mypath=${myname%/*} + myname=${myname##*/} + if [ "$mypath" ] && [ -d "$mypath" ]; then + return + fi + mypath=$(pwd) + if [ -f "${mypath}/${myname}" ]; then + return + fi + echo "FATAL: Could not figure out the path where $myname lives from $0" + exit 1 +} + +COMMON_ENV=libexec/vespa/common-env.sh + +source_common_env () { + if [ "$VESPA_HOME" ] && [ -d "$VESPA_HOME" ]; then + # ensure it ends with "/" : + VESPA_HOME=${VESPA_HOME%/}/ + export VESPA_HOME + common_env=$VESPA_HOME/$COMMON_ENV + if [ -f "$common_env" ]; then + . $common_env + return + fi + fi + return 1 +} + +findroot () { + source_common_env && return + if [ "$VESPA_HOME" ]; then + echo "FATAL: bad VESPA_HOME value '$VESPA_HOME'" + exit 1 + fi + if [ "$ROOT" ] && [ -d "$ROOT" ]; then + VESPA_HOME="$ROOT" + source_common_env && return + fi + findpath + while [ "$mypath" ]; do + VESPA_HOME=${mypath} + source_common_env && return + mypath=${mypath%/*} + done + echo "FATAL: missing VESPA_HOME environment variable" + echo "Could not locate $COMMON_ENV anywhere" + exit 1 +} + +findroot + +# END environment bootstrap section + +set -e +set -x + +declare -r CONFIG_SERVER_HOSTNAME=config-server +declare -r CONFIG_SERVER_PORT=19071 +declare -r TENANT_NAME=localtenant +# TODO: Make it possible to deploy any app from host context, not only this app (from within container). +declare -r VESPA_APP=$VESPA_HOME/share/vespa/sampleapps/search/music + +# Create tenant +curl -X PUT $CONFIG_SERVER_HOSTNAME:$CONFIG_SERVER_PORT/application/v2/tenant/$TENANT_NAME + +# Deploy sample app +deploy -e "$TENANT_NAME" prepare $VESPA_APP +deploy -e "$TENANT_NAME" activate diff --git a/node-admin/include/music-on-docker-services.xml b/node-admin/include/music-on-docker-services.xml new file mode 100644 index 00000000000..0bb37950c98 --- /dev/null +++ b/node-admin/include/music-on-docker-services.xml @@ -0,0 +1,18 @@ + + + + + + + + + + + 1 + + + + + + + diff --git a/node-admin/include/node-flavors.xml b/node-admin/include/node-flavors.xml new file mode 100644 index 00000000000..419ea348493 --- /dev/null +++ b/node-admin/include/node-flavors.xml @@ -0,0 +1,15 @@ + + + + + + docker + DOCKER_CONTAINER + 1.0 + 3.0 + 10.0 + DOCKER_CONTAINER with 1.0 CPUs, 3.0 Gb memory and 10.0 Gb disk + + + + diff --git a/node-admin/include/nodectl-instance.sh b/node-admin/include/nodectl-instance.sh new file mode 100755 index 00000000000..5a6665dbdc7 --- /dev/null +++ b/node-admin/include/nodectl-instance.sh @@ -0,0 +1,138 @@ +#!/bin/sh +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +# WARNING: This script should be kept in sync with the file used by Chef: +# vespa-cookbooks/hosted/files/default/prepost-instance.sh +# TODO: Remove the above cookbook file (with the down-side that a new script +# requires a new vespa release, instead of just a hosted release). + +# Usage: nodectl-instance.sh [start|stop] +# +# start: Set the node "in service" by e.g. undraining container traffic. +# start can be assumed to have completed successfully. +# +# stop: Prepare for a short suspension, e.g. there's a pending upgrade. Set the +# node "out of service" by draining container traffic, and flush index for a +# quick start after the suspension. There's no need to stop. + +# BEGIN environment bootstrap section +# Do not edit between here and END as this section should stay identical in all scripts + +findpath () { + myname=${0} + mypath=${myname%/*} + myname=${myname##*/} + if [ "$mypath" ] && [ -d "$mypath" ]; then + return + fi + mypath=$(pwd) + if [ -f "${mypath}/${myname}" ]; then + return + fi + echo "FATAL: Could not figure out the path where $myname lives from $0" + exit 1 +} + +COMMON_ENV=libexec/vespa/common-env.sh + +source_common_env () { + if [ "$VESPA_HOME" ] && [ -d "$VESPA_HOME" ]; then + # ensure it ends with "/" : + VESPA_HOME=${VESPA_HOME%/}/ + export VESPA_HOME + common_env=$VESPA_HOME/$COMMON_ENV + if [ -f "$common_env" ]; then + . $common_env + return + fi + fi + return 1 +} + +findroot () { + source_common_env && return + if [ "$VESPA_HOME" ]; then + echo "FATAL: bad VESPA_HOME value '$VESPA_HOME'" + exit 1 + fi + if [ "$ROOT" ] && [ -d "$ROOT" ]; then + VESPA_HOME="$ROOT" + source_common_env && return + fi + findpath + while [ "$mypath" ]; do + VESPA_HOME=${mypath} + source_common_env && return + mypath=${mypath%/*} + done + echo "FATAL: missing VESPA_HOME environment variable" + echo "Could not locate $COMMON_ENV anywhere" + exit 1 +} + +findroot + +# END environment bootstrap section + +has_servicename() { + local name="$1" + $VESPA_HOME/bin/vespa-model-inspect host $(hostname) | grep -q "$name @ " + return $? +} + +has_container() { + has_servicename container || has_servicename qrserver +} + +has_searchnode() { + has_servicename searchnode +} + +container_drain() { + # TODO: Implement proper draining + sleep 60 +} + +start() { + # Always start vip for now + $echo $VESPA_HOME/bin/vespa-routing vip -u chef in +} + +stop() { + # Always stop vip for now + $echo $VESPA_HOME/bin/vespa-routing vip -u chef out + + if has_searchnode; then + $echo $VESPA_HOME/bin/vespa-proton-cmd --local triggerFlush + fi + + if has_container; then + $echo container_drain + fi +} + +main() { + if [ $# -lt 1 ]; then + echo "Usage: $0 [-e] start|stop" >&2 + exit 1 + fi + + echo="" + if [ "$1" = "-e" ]; then + echo=echo + shift + fi + + action="$1" + + if [ "$action" = "start" ]; then + start + elif [ "$action" = "stop" ]; then + stop + else + echo "Unknown action: $action" >&2 + exit 1 + fi +} + +main "$@" diff --git a/node-admin/include/root-bashrc b/node-admin/include/root-bashrc new file mode 100644 index 00000000000..fda1752c85f --- /dev/null +++ b/node-admin/include/root-bashrc @@ -0,0 +1 @@ +export TERM=xterm diff --git a/node-admin/include/start-config-server.sh b/node-admin/include/start-config-server.sh new file mode 100755 index 00000000000..a84b0454db3 --- /dev/null +++ b/node-admin/include/start-config-server.sh @@ -0,0 +1,138 @@ +#!/bin/bash +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +# BEGIN environment bootstrap section +# Do not edit between here and END as this section should stay identical in all scripts + +findpath () { + myname=${0} + mypath=${myname%/*} + myname=${myname##*/} + if [ "$mypath" ] && [ -d "$mypath" ]; then + return + fi + mypath=$(pwd) + if [ -f "${mypath}/${myname}" ]; then + return + fi + echo "FATAL: Could not figure out the path where $myname lives from $0" + exit 1 +} + +COMMON_ENV=libexec/vespa/common-env.sh + +source_common_env () { + if [ "$VESPA_HOME" ] && [ -d "$VESPA_HOME" ]; then + # ensure it ends with "/" : + VESPA_HOME=${VESPA_HOME%/}/ + export VESPA_HOME + common_env=$VESPA_HOME/$COMMON_ENV + if [ -f "$common_env" ]; then + . $common_env + return + fi + fi + return 1 +} + +findroot () { + source_common_env && return + if [ "$VESPA_HOME" ]; then + echo "FATAL: bad VESPA_HOME value '$VESPA_HOME'" + exit 1 + fi + if [ "$ROOT" ] && [ -d "$ROOT" ]; then + VESPA_HOME="$ROOT" + source_common_env && return + fi + findpath + while [ "$mypath" ]; do + VESPA_HOME=${mypath} + source_common_env && return + mypath=${mypath%/*} + done + echo "FATAL: missing VESPA_HOME environment variable" + echo "Could not locate $COMMON_ENV anywhere" + exit 1 +} + +findroot + +# END environment bootstrap section + +export LC_ALL=C + +function WaitUntilHostIsReachable { + # Address may be IP or hostname. + local address="$1" + + echo -n "Will wait until $address is reachable... " + while ! ping -q -c 1 -W 3 "$address" &>/dev/null + do + echo "not done (will retry)" + sleep 1 + done + echo "Done" +} + +function VerifyRequiredEnvironmentVariablesAreSet { + if [ -z "$HOSTED_VESPA_REGION" ] + then + Fail "Environment variable HOSTED_VESPA_REGION is not set" + fi + if [ -z "$CONFIG_SERVER_HOSTNAME" ] + then + Fail "Environment variable CONFIG_SERVER_HOSTNAME is not set" + fi + if [ -z "$HOST_BRIDGE_IP" ] + then + Fail "Environment variable HOST_BRIDGE_IP is not set" + fi + + case "$HOSTED_VESPA_ENVIRONMENT" in + prod|test|dev|staging|perf) : ;; + *) Fail "The HOSTED_VESPA_ENVIRONMENT environment variable must be one of prod, test, dev, staging, or perf" ;; + esac +} + +function InternalMain { + VerifyRequiredEnvironmentVariablesAreSet + + mkdir -p $VESPA_HOME/logs + chmod 1777 $VESPA_HOME/logs + mkdir -p $VESPA_HOME/logs/jdisc_core + + rm -rf $VESPA_HOME/var/vespa/bundlecache/standalone + + yinst set \ + cloudconfig_server.multitenant=true \ + cloudconfig_server.region="$HOSTED_VESPA_REGION" \ + cloudconfig_server.autostart=on \ + cloudconfig_server.default_flavor=docker \ + cloudconfig_server.environment="$HOSTED_VESPA_ENVIRONMENT" \ + cloudconfig_server.hosted_vespa=true \ + services.addr_configserver="$CONFIG_SERVER_HOSTNAME" + + # Can also set jvmargs if necessary: + # set cloudconfig_server.jvmargs=-Dvespa.freezedetector.disable=true -XX:NewRatio=1 -verbose:gc -XX:+PrintGCDateStamps -agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=5005 -Xms6g -Xmx6g + + # The network is set up asynchronously and from outside of this + # container. Wait until it's done. + WaitUntilHostIsReachable "$HOST_BRIDGE_IP" + + yinst start cloudconfig_server + + touch $VESPA_HOME/logs/jdisc_core/jdisc_core.log + $VESPA_HOME/bin/logfmt -N -f $VESPA_HOME/logs/jdisc_core/jdisc_core.log +} + +function Main { + # Prefix each line to stdout/stderr with a timestamp to make it easier to + # understand the progress. + InternalMain |& while read -r + do + printf "%s %s\n" "$(date +%FT%T)" "$REPLY" + done +} + +Main "$@" diff --git a/node-admin/include/start-services.sh b/node-admin/include/start-services.sh new file mode 100755 index 00000000000..0fe5c1c0724 --- /dev/null +++ b/node-admin/include/start-services.sh @@ -0,0 +1,98 @@ +#!/bin/bash +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +# TODO: This file duplicates a file in the base image (with local modifications). +# Update the file there. + +# BEGIN environment bootstrap section +# Do not edit between here and END as this section should stay identical in all scripts + +findpath () { + myname=${0} + mypath=${myname%/*} + myname=${myname##*/} + if [ "$mypath" ] && [ -d "$mypath" ]; then + return + fi + mypath=$(pwd) + if [ -f "${mypath}/${myname}" ]; then + return + fi + echo "FATAL: Could not figure out the path where $myname lives from $0" + exit 1 +} + +COMMON_ENV=libexec/vespa/common-env.sh + +source_common_env () { + if [ "$VESPA_HOME" ] && [ -d "$VESPA_HOME" ]; then + # ensure it ends with "/" : + VESPA_HOME=${VESPA_HOME%/}/ + export VESPA_HOME + common_env=$VESPA_HOME/$COMMON_ENV + if [ -f "$common_env" ]; then + . $common_env + return + fi + fi + return 1 +} + +findroot () { + source_common_env && return + if [ "$VESPA_HOME" ]; then + echo "FATAL: bad VESPA_HOME value '$VESPA_HOME'" + exit 1 + fi + if [ "$ROOT" ] && [ -d "$ROOT" ]; then + VESPA_HOME="$ROOT" + source_common_env && return + fi + findpath + while [ "$mypath" ]; do + VESPA_HOME=${mypath} + source_common_env && return + mypath=${mypath%/*} + done + echo "FATAL: missing VESPA_HOME environment variable" + echo "Could not locate $COMMON_ENV anywhere" + exit 1 +} + +findroot + +# END environment bootstrap section + +export LC_ALL=C + +function wait_for_network_up { + while true + do + for config_server_host in $(echo $CONFIG_SERVER_ADDRESS | tr "," " ") + do + ping -c 1 -W 3 $config_server_host && return + sleep 1 + done + done +} + +if [ -z $CONFIG_SERVER_ADDRESS ] +then + echo "CONFIG_SERVER_ADDRESS must be set." + exit -1 +fi + +chown yahoo $VESPA_HOME/var/jdisc_container + +# Local modification; fixes ownership issues for vespa node running in container. +chown yahoo $VESPA_HOME/var/zookeeper + +if [ -d "$VESPA_HOME/logs" ] +then + chmod 1777 $VESPA_HOME/logs +fi + +yinst set services.addr_configserver=$CONFIG_SERVER_ADDRESS +wait_for_network_up +yinst start services +logfmt -n -f diff --git a/node-admin/pom.xml b/node-admin/pom.xml new file mode 100644 index 00000000000..c89e8524fc3 --- /dev/null +++ b/node-admin/pom.xml @@ -0,0 +1,162 @@ + + + + 4.0.0 + + com.yahoo.vespa + parent + 6-SNAPSHOT + ../parent/pom.xml + + + node-admin + 6-SNAPSHOT + container-plugin + ${project.artifactId} + + + + com.yahoo.vespa + node-repository + ${project.version} + test + + + com.yahoo.vespa + defaults + ${project.version} + provided + + + org.hamcrest + hamcrest-junit + 2.0.0.0 + test + + + com.yahoo.vespa + container-dev + ${project.version} + provided + + + com.spotify + docker-client + 3.5.12 + + + org.slf4j + slf4j-api + + + com.google.guava + guava + + + com.fasterxml.jackson.jaxrs + jackson-jaxrs-json-provider + + + com.fasterxml.jackson.core + jackson-databind + + + org.glassfish.jersey.core + jersey-client + + + org.glassfish.jersey.core + jersey-common + + + javax.ws.rs + javax.ws.rs-api + + + com.fasterxml.jackson.core + jackson-core + + + org.glassfish.jersey.media + jersey-media-json-jackson + + + jnr-unixsocket + com.github.jnr + + + + + org.apache.httpcomponents + httpcore + 4.4.1 + + + org.apache.httpcomponents + httpclient + 4.5 + + + com.yahoo.vespa + orchestrator-restapi + ${project.version} + compile + + + com.yahoo.vespa + jaxrs_utils + ${project.version} + compile + + + com.yahoo.vespa + jaxrs_client_utils + ${project.version} + compile + + + com.yahoo.vespa + application-model + ${project.version} + + + junit + junit + test + + + com.yahoo.vespa + application + test + ${project.version} + + + org.mockito + mockito-core + test + + + + + + + com.yahoo.vespa + bundle-plugin + true + + + org.apache.maven.plugins + maven-compiler-plugin + + + -Xlint:all + -Werror + + + + + + diff --git a/node-admin/scripts/app.sh b/node-admin/scripts/app.sh new file mode 100755 index 00000000000..8f1787118ed --- /dev/null +++ b/node-admin/scripts/app.sh @@ -0,0 +1,217 @@ +#!/bin/bash +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +# BEGIN environment bootstrap section +# Do not edit between here and END as this section should stay identical in all scripts + +findpath () { + myname=${0} + mypath=${myname%/*} + myname=${myname##*/} + if [ "$mypath" ] && [ -d "$mypath" ]; then + return + fi + mypath=$(pwd) + if [ -f "${mypath}/${myname}" ]; then + return + fi + echo "FATAL: Could not figure out the path where $myname lives from $0" + exit 1 +} + +COMMON_ENV=libexec/vespa/common-env.sh + +source_common_env () { + if [ "$VESPA_HOME" ] && [ -d "$VESPA_HOME" ]; then + # ensure it ends with "/" : + VESPA_HOME=${VESPA_HOME%/}/ + export VESPA_HOME + common_env=$VESPA_HOME/$COMMON_ENV + if [ -f "$common_env" ]; then + . $common_env + return + fi + fi + return 1 +} + +findroot () { + source_common_env && return + if [ "$VESPA_HOME" ]; then + echo "FATAL: bad VESPA_HOME value '$VESPA_HOME'" + exit 1 + fi + if [ "$ROOT" ] && [ -d "$ROOT" ]; then + VESPA_HOME="$ROOT" + source_common_env && return + fi + findpath + while [ "$mypath" ]; do + VESPA_HOME=${mypath} + source_common_env && return + mypath=${mypath%/*} + done + echo "FATAL: missing VESPA_HOME environment variable" + echo "Could not locate $COMMON_ENV anywhere" + exit 1 +} + +findroot + +# END environment bootstrap section + +set -e + +source "${0%/*}"/common.sh + +declare SCRIPTS_DIR="${0%/*}" + +declare -r APP_DIR_NAME_UNDER_SHARED=app + +function Usage { + UsageHelper "$@" < [] +Deploy (or undeploy) application rooted at on localhost Config Server. + +The local zone must be up and running. should point to +e.g. vespa/basic-search-on-docker/target/application. +EOF +} + +function RunOnConfigServer { + docker exec config-server "$@" +} + +function VerifyApp { + local app_dir="$1" + + # Sanity-check app_dir + if ! [ -d "$app_dir" ] + then + Fail " '$app_dir' is not a directory" + fi + + local services_xml="$app_dir"/services.xml + if ! [ -f "$services_xml" ] + then + Fail "Failed to find services.xml in '$app_dir'" + fi + + # Verify there's no element. + if grep -qE ']' "$services_xml" + then + Fail "services.xml cannot contain an element in hosted Vespa" + fi + + # Verify seems to be correctly specified (warning: this test is + # incomplete). + if grep -qE "" "$services_xml" || + ! grep -qE " element in the following form" \ + "in hosted Vespa w/Docker:" \ + " " \ + "where IMAGE is e.g. vespa-local:latest." + fi +} + +# Copies the application rooted at $1 to a directory tree shared with the +# Config Server. +function CopyToSharedDir { + local app_dir="$1" + + local shared_dir_on_localhost="$APPLICATION_STORAGE_ROOT/$CONFIG_SERVER_CONTAINER_NAME/$ROOT_DIR_SHARED_WITH_HOST" + if ! [ -d "$shared_dir_on_localhost" ] + then + Fail "Failed to find the Config Server's shared directory on" \ + "localhost '$shared_dir_on_localhost', has the" \ + "$CONFIG_SERVER_CONTAINER_NAME container been started?" + fi + + + local shared_app_dir_on_localhost="$shared_dir_on_localhost/$APP_DIR_NAME_UNDER_SHARED" + if [ "$shared_app_dir_on_localhost" != /home/docker/container-storage/config-server/shared/app ] + then + # This duplication of code is a safety-guard against 'rm -rf' unknown + # directories. + Fail "We're about to remove '$shared_app_dir_on_localhost', but it's" \ + "pointing to something unexpected, refusing to proceed..." + fi + + echo -n "Copying application to '$shared_app_dir_on_localhost'... " + rm -rf "$shared_app_dir_on_localhost" + cp -r "$app_dir" "$shared_app_dir_on_localhost" + echo done +} + +function DeployApp { + if (($# != 1)) + then + Usage + fi + + local app_dir="$1" + + VerifyApp "$app_dir" + + CopyToSharedDir "$app_dir" + + # Create tenant + echo -n "Creating tenant... " + local create_tenant_response + if create_tenant_response=$(curl --silent --show-error -X PUT "http://$CONFIG_SERVER_HOSTNAME:$CONFIG_SERVER_PORT/application/v2/tenant/$TENANT_NAME" 2>&1) + then + if ! [[ "$create_tenant_response" =~ "Tenant $TENANT_NAME created" ]] && + ! [[ "$create_tenant_response" =~ "already exists" ]] + then + echo + Fail "May have failed to create the tenant: '$create_tenant_response'" + fi + else + echo + Fail "Failed to create the tenant: $?: '$create_tenant_response'" + fi + echo done + + # Deploy app + local app_dir_on_config_server="/$ROOT_DIR_SHARED_WITH_HOST/$APP_DIR_NAME_UNDER_SHARED" + RunOnConfigServer $VESPA_HOME/bin/deploy -e "$TENANT_NAME" prepare "$app_dir_on_config_server" + echo "Activating application" + RunOnConfigServer $VESPA_HOME/bin/deploy -e "$TENANT_NAME" activate +} + +function UndeployApp { + if (($# != 0)) + then + Usage "undeploy takes no arguments" + fi + + local app_name=default + local output + echo -n "Removing application $TENANT_NAME:$app_name... " + if ! output=$(curl --silent --show-error -X DELETE "http://$CONFIG_SERVER_HOSTNAME:$CONFIG_SERVER_PORT/application/v2/tenant/$TENANT_NAME/application/$app_name") + then + echo + Fail "Failed to remove application: $output" + fi + + echo done +} + +function Main { + if (($# == 0)) + then + Usage "Missing command" + fi + local command="$1" + shift + + case "$command" in + deploy) DeployApp "$@" ;; + undeploy) UndeployApp "$@" ;; + *) Usage "Unknown command '$command'" ;; + esac +} + +Main "$@" diff --git a/node-admin/scripts/common-vm.sh b/node-admin/scripts/common-vm.sh new file mode 100644 index 00000000000..c91c75e1404 --- /dev/null +++ b/node-admin/scripts/common-vm.sh @@ -0,0 +1,13 @@ +#!/bin/bash +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +set -e + +source "${0%/*}/common.sh" + +# VM configuration +declare -r DOCKER_VM_NAME=vespa # Don't put spaces in the name +declare -r DOCKER_VM_DISK_SIZE_IN_MB=40000 +declare -r DOCKER_VM_MEMORY_SIZE_IN_MB=4096 +declare -r DOCKER_VM_CPU_COUNT=1 +declare -r DOCKER_VM_HOST_CIDR=172.21.46.1/24 diff --git a/node-admin/scripts/common.sh b/node-admin/scripts/common.sh new file mode 100644 index 00000000000..d07b4adcc5a --- /dev/null +++ b/node-admin/scripts/common.sh @@ -0,0 +1,180 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +# Common variables and functions that may be useful for scripts IN THIS +# DIRECTORY. Should be sourced as follows: +# +# source "${0%/*}/common.sh" +# +# WARNING: Some system variables, like the Config Server's, are also hardcoded +# in the Docker image startup scripts. + +declare -r SCRIPT_NAME="${0##*/}" +declare -r SCRIPT_DIR="${0%/*}" + +# TODO: Find a better name. Consider having separate images for config-server +# and node-admin. +declare -r DOCKER_IMAGE="vespa-local:latest" +declare -r APPLICATION_STORAGE_ROOT="/home/docker/container-storage" +declare -r ROOT_DIR_SHARED_WITH_HOST=shared + +# The 172.18.0.0/16 network is in IPDB. +declare -r NETWORK_PREFIX=172.18 +declare -r NETWORK_PREFIX_BITLENGTH=16 + +# Hostnames, IP addresses, names, etc of the infrastructure containers. +declare -r HOST_BRIDGE_INTERFACE=vespa +declare -r HOST_BRIDGE_IP="$NETWORK_PREFIX.0.1" +declare -r HOST_BRIDGE_NETWORK="$NETWORK_PREFIX.0.0/$NETWORK_PREFIX_BITLENGTH" +declare -r NODE_ADMIN_CONTAINER_NAME=node-admin +declare -r CONFIG_SERVER_CONTAINER_NAME=config-server +declare -r CONFIG_SERVER_HOSTNAME="$CONFIG_SERVER_CONTAINER_NAME" +declare -r CONFIG_SERVER_IP="$NETWORK_PREFIX.1.1" +declare -r CONFIG_SERVER_PORT=19071 + +declare -r DEFAULT_HOSTED_VESPA_REGION=local-region +declare -r DEFAULT_HOSTED_VESPA_ENVIRONMENT=prod + +# Hostnames, IP addresses, names, etc of the application containers. Hostname +# and container names are of the form $PREFIX$N, where N is a number between 1 +# and $NUM_APP_CONTAINERS. The IP is $APP_NETWORK_PREFIX.$N. +declare -r APP_NETWORK_PREFIX="$NETWORK_PREFIX.2" +declare -r APP_CONTAINER_NAME_PREFIX=cnode- +declare -r APP_HOSTNAME_PREFIX="$APP_CONTAINER_NAME_PREFIX" +declare -r DEFAULT_NUM_APP_CONTAINERS=20 # Statically allocated number of nodes. +declare -r TENANT_NAME=localtenant + +# May be 'vm' if docker hosts runs within a VM (osx). Default is native/Fedora. +declare -r NETWORK_TYPE="${NETWORK_TYPE:-local}" + +# Allowed program opions +declare OPTION_NUM_NODES # Set from --num-nodes or DEFAULT_NUM_APP_CONTAINERS, see Main. +declare OPTION_WAIT # Set from --wait or true, see Main. +declare OPTION_HV_REGION # Set from --hv-region or DEFAULT_HOSTED_VESPA_REGION, see Main. +declare OPTION_HV_ENV # Set from --hv-env or DEFAULT_HOSTED_VESPA_ENVIRONMENT, see Main. + +declare NUM_APP_CONTAINERS # Set from OPTION_NUM_NODES or DEFAULT_NUM_APP_CONTAINERS, see Main. + +function Fail { + printf "%s\n" "$@" >&2 + exit 1 +} + +# Used to help scripts with implementing the Usage function. The intended usage +# is: +# +# function Usage { +# UsageHelper "$@" <&2 + + if (($# > 0)) + then + printf "%s\n\n" "$*" + fi + + # Print to stdout (which has been redirected to stderr) what's on + # stdin. This will print the usage-string. + cat + + exit 1 +} + +# See Main +function Restart { + Stop + Start "$@" +} + +# Use Main as follows: +# +# Pass all script arguments to Main: +# +# Main "$@" +# +# Main will parse the arguments as follows. It assumes the arguments have +# the following form: +# +# script.sh [ |