diff options
| -rw-r--r-- | README.md | 3 | ||||
| -rw-r--r-- | docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/DockerTestUtils.java | 5 | ||||
| -rw-r--r-- | node-admin/README.md | 108 | ||||
| -rw-r--r-- | node-admin/README_LINUX.md | 123 | ||||
| -rw-r--r-- | node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/docker/RunVespaLocal.java | 149 |
5 files changed, 145 insertions, 243 deletions
diff --git a/README.md b/README.md index c1ab45629f0..54abdfeda10 100644 --- a/README.md +++ b/README.md @@ -54,8 +54,7 @@ Java modules can be built on any environment having Java and Maven: ## Running Vespa on a local machine -* OS X : See [node-admin/README_MAC.md](node-admin/README_MAC.md) -* Linux : See [node-admin/README_LINUX.md](node-admin/README_LINUX.md) +See [node-admin/README.md](node-admin/README.md) Code licensed under the Apache 2.0 license. See LICENSE file for terms. diff --git a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/DockerTestUtils.java b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/DockerTestUtils.java index d2281f7e800..5c4a721b092 100644 --- a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/DockerTestUtils.java +++ b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/DockerTestUtils.java @@ -16,11 +16,6 @@ import java.util.concurrent.ExecutionException; * 1. Install Docker Toolbox, and start it (Docker Quickstart Terminal) (you can close terminal window afterwards) * 2. For network test, we need to make docker containers visible for Mac: sudo route add 172.18.0.0/16 192.168.99.100 * - * GENERAL TIPS: - * For cleaning up your local docker machine (DON'T DO THIS ON PROD) - * docker stop $(docker ps -a -q) - * docker rm $(docker ps -a -q) - * * @author freva */ public class DockerTestUtils { diff --git a/node-admin/README.md b/node-admin/README.md index f4c5f5be4a5..d93f8a831ba 100644 --- a/node-admin/README.md +++ b/node-admin/README.md @@ -2,47 +2,65 @@ ## Setup -Set up Docker on your machine according to the instructions in README_LINUX or README_MAC, depending on your hardware. +Set up Docker on your machine according to the instructions in [LINUX](README_LINUX.md) or [Mac](README_MAC.md), depending on your hardware. You should have the docker daemon running and the following environment variables set: ``` -DOCKER_HOST -CONTAINER_CERT_PATH VESPA_HOME +VESPA_WEB_SERVICE_PORT ``` -## Building +To add update `/etc/hosts` with the required hostnames for the local containers, run +``` +sudo ./scripts/etc-hosts.sh +``` + +## Developing -Build Node Admin and include it (and other local modifications) in the Docker image ```vespa-local```: +We will describe how you can build a Docker image for Vespa which will be used +to set up a local Docker container with the Node Admin, and a local container +with the Config Server. + +Then, we'll show how you bring up this local zone. And finally, how you can +deploy a local Vespa application to this zone. + +[RunVespaLocal.java](src/test/java/com/yahoo/vespa/hosted/node/admin/docker/RunVespaLocal.java) +implements all of the basic methods you need to get started. + +### Starting a Local Zone + +To start a local zone, simply run: ``` -mvn clean package -./build.sh + DockerImage vespaDockerBase = new DockerImage("docker-registry.ops.yahoo.com:4443/vespa/ci:6.53.134"); + Path pathToContainerStorage = Paths.get("/home/docker/container-storage"); + + RunVespaLocal runVespaLocal = new RunVespaLocal(); + runVespaLocal.buildVespaLocalImage(vespaDockerBase); + runVespaLocal.startLocalZoneWithNodes(5); + runVespaLocal.startNodeAdminAsContainer(pathToContainerStorage); ``` -## Running +### Deploying a Local Application + +To deploy an application, check out `vespa/basic-search-for-docker` to `~`, and +package it with ```mvn package```, then deploy it with: -TODO: Outdated! Update this section with info on how to run everything locally. +``` + Path pathToApp = Paths.get("/home/<username>/basic-search-for-docker/target/application.zip"); + runVespaLocal.deployApplication(pathToApp); +``` -Start the container for the config server (TODO). Set the CONFIG_SERVER_ADDRESS -variable to the hostname of the config server. +You can undeploy it with -Start the container ``` -docker run -t -i --privileged \ - -p 4080:4080 \ - -v $CONTAINER_CERT_PATH:/host/docker/certs \ - -e "DOCKER_HOST=$DOCKER_HOST" \ - -e "CONFIG_SERVER_ADDRESS=$CONFIG_SERVER_ADDRESS" \ - vespa-local:latest + runVespaLocal.undeployApplication(); ``` -This will map the client certificate/key files to the path where Node Admin looks for them (as configured in -services.xml), and enable Node Admin to talk to the docker daemon. You can invoke Node Admin's REST APIs on port 4080 -from both inside the container and the outside host. + ## Using -Trigger the incredibly rich and complex node-admin REST API(s) +Trigger the incredibly rich and complex `node-admin` REST API(s) ``` curl localhost:4080/test/ping ``` @@ -64,49 +82,3 @@ View the log file (`-L` follows the symbolic link): docker cp -L <container id>:$VESPA_HOME/logs/jdisc_core/jdisc_core.log - | less ``` -## Developing - -We will describe how you can build a Docker image for Vespa which will be used -to set up a local Docker container with the Node Admin, and a local container -with the Config Server. - -Then, we'll show how you bring up this local zone. And finally, how you can -deploy a local Vespa application to this zone. - -### Building Local Docker Image - -A Dockerfile exists in the module's root directory. This Dockerfile is not used -in production or any pipelines, it is here for convenience so you can build -images and experiment locally. See build.sh for how to build. - -The image created by the Dockerfile will be used to run Node Admin or a Config -Server. - -### Starting a Local Zone - -To start a local zone, ensure your operating system ignores ```config-server``` -and ```node-admin``` for proxying. Then issue the following command: - -``` -scripts/zone.sh start -``` - -The Node Admin and Config Server now runs in the ```node-admin``` and -```config-server``` Docker containers. These containers have their own IP -addresses and hostnames (also ```node-admin``` and ```config-server```). - -### Deploying a Local Application - -To deploy an application, use ```scripts/app.sh```. Assuming you have checked -out ```vespa/basic-search-for-docker``` to ```~```, and packaged it with ```mvn -package```, you can deploy the application with: - -``` -scripts/app.sh deploy ~/vespa/basic-search-for-docker/target/application -``` - -You can undeploy it with - -``` -scripts/app.sh undeploy -``` diff --git a/node-admin/README_LINUX.md b/node-admin/README_LINUX.md index b369044eef5..6d250d1a71c 100644 --- a/node-admin/README_LINUX.md +++ b/node-admin/README_LINUX.md @@ -1,123 +1,66 @@ # Setting up Docker on a linux machine First, install Docker. With Fedora 21 you should follow -https://docs.docker.com/installation/fedora/, which describes how to install -Docker, start the Docker daemon, verify you can download images and run them, -and making your user run docker instances. +https://docs.docker.com/engine/installation/linux/fedora/, which describes how to install +Docker. -``` -sudo systemctl enable docker -``` - -## Set up yahoo user - -The Vespa docker containers will run images that need to access the host file -system as the user 'yahoo' with UID 1000, e.g. the Node Admin runs as this -user. If this UID is not already taken, you can create a yahoo user as follows: - -``` -sudo useradd -u 1000 -g 100 -s /dev/null yahoo -``` - -If the UID is already in use you should move the user to a new UID first. -Alternatively, it might be possible to reuse that user, but this is confusing -and may lead to errors later on (and has not been tested). In the following -example we move the 'landesk' user from UID 1000 to 1010, keeping its GID 1001. - -``` -sudo usermod -u 1010 landesk -sudo find / -user 1000 -exec chown -h 1010 {} \; -``` - -## Set up image storage (aka don't break your machine) +## Configuring Docker -Docker will by default download (huge) images to a directory under /var. On our -Fedora machines, /var is part of the root filesystem, which does not have a lot -of free space. Since docker runs as root, it is allowed to completely fill up +Docker will by default download (huge) images to a directory under `/var`. On our +Fedora machines, `/var` is part of the root filesystem, which does not have a lot +of free space. Since docker runs as `root`, it is allowed to completely fill up the filesystem, and it will happily do so. Fedora works very poorly with a full root filesystem. You won't even be able to log in and clean up the disk usage once it's happened. -So you'll want to store images somewhere else. An obvious choice is /home, +So you'll want to store images somewhere else. An obvious choice is `/home`, which typically has a lot more room. Make Docker use directories in the docker -user's home directory. Run the following script to do this: +user's home directory, set the `--graph` option by editing +`/etc/systemd/system/docker.service.d/docker.conf` to: ``` -scripts/setup-docker.sh home +[Service] +ExecStart= +ExecStart=/usr/bin/dockerd \ + --debug \ + --graph=/home/docker/data \ + --host=127.0.0.1:2376 \ + --host=unix:///var/run/docker.sock \ + --selinux-enabled \ + --storage-driver=devicemapper \ + --storage-opt=dm.basesize=20G ``` -## Set up TLS - -By default, the docker CLI communicates with the docker daemon via a unix -socket. This is fine in itself, but not suffficient for our use. Node Admin, -itself running in a container, will talk to the docker daemon to start -containers for vespa nodes. Node Admin uses a Java library for communication -with the docker daemon, and this library depends on JNI (native) code for unix -socket communication. We don't want that, so that dependency has been -excluded. Therefore, Node Admin uses TLS over IP to communicate with the docker -daemon. You must therefore set up docker with TLS. Mostly, you can follow the -instructions at https://docs.docker.com/articles/https/. - -The commands can be run with - +Finally, start docker: ``` -scripts/setup-docker.sh certs +sudo systemctl start docker ``` -Note the following: - - - You will be asked to generate a key, and will repeatedly be asked for it. - - Use your fully qualified domain name for Common Name. - -Now, you need to tell the docker daemon to use TLS. Edit the file ```/lib/systemd/system/docker.service``` and change -the ExecStart line so it includes the following arguments: -``` ---tlsverify --tlscacert=/etc/dockercert_daemon/ca_cert.pem --tlscert=/etc/dockercert_daemon/server_cert.pem --tlskey=/etc/dockercert_daemon/server_key.pem -H=0.0.0.0:2376 -``` +## Set up yahoo user -Then restart docker: -``` -sudo systemctl daemon-reload -sudo systemctl restart docker -``` +The Vespa docker containers will run images that need to access the host file +system as the user `yahoo` with UID 1000, e.g. the Node Admin runs as this +user. If this UID is not already taken, you can create a `yahoo` user as follows: -Now tell the docker CLI how to communicate with the docker daemon: ``` -export DOCKER_HOST=tcp://127.0.0.1:2376 -export DOCKER_TLS_VERIFY=1 -export DOCKER_CERT_PATH=/etc/dockercert_cli +sudo useradd -u 1000 -g 100 -s /dev/null yahoo ``` -(You might want to add this to your .bashrc file.) - -Now, test that the docker cli can talk to the docker daemon: -``` -docker version -docker run --rm hello-world -``` +If the UID is already in use you should move the user to a new UID first. +Alternatively, it might be possible to reuse that user, but this is confusing +and may lead to errors later on (and has not been tested). In the following +example we move the `landesk` user from UID 1000 to 1010, keeping its GID 1001. -These should run without errors. Finally, to run Node Admin locally, it needs access to the certificate/key files. ``` -export CONTAINER_CERT_PATH=/etc/dockercert_container +sudo usermod -u 1010 landesk +sudo find / -user 1000 -exec chown -h 1010 {} \; ``` -This environment variable will be used when starting the container, which is decribed in the platform-independent -README file. - -While docker can and should be run as your user, it's nice to make it possible -to run docker under root too. To enable this you must make sure sudo doesn't -strip off the environment variables, otherwise certain docker commands may -hang. Add a file /etc/sudoers.d/passthrough-docker-env with the content: - -``` -Defaults env_keep += "DOCKER_HOST DOCKER_TLS_VERIFY DOCKER_CERT_PATH CONTAINER_CERT_PATH" -``` -You are now done with the linux-specific setup work. ## Other -For more information on how to configure the docker daemon, see https://docs.docker.com/articles/systemd/. +For more information on how to configure the docker daemon, see https://docs.docker.com/engine/admin/systemd/. ## Upgrade of Docker diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/docker/RunVespaLocal.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/docker/RunVespaLocal.java index 58dd64f48bc..9862daa2ac5 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/docker/RunVespaLocal.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/docker/RunVespaLocal.java @@ -3,7 +3,6 @@ package com.yahoo.vespa.hosted.node.admin.docker; import com.yahoo.metrics.simple.MetricReceiver; import com.yahoo.net.HostName; -import com.yahoo.vespa.hosted.dockerapi.ContainerName; import com.yahoo.vespa.hosted.dockerapi.Docker; import com.yahoo.vespa.hosted.dockerapi.DockerImage; import com.yahoo.vespa.hosted.dockerapi.DockerTestUtils; @@ -14,47 +13,26 @@ import com.yahoo.vespa.hosted.node.admin.nodeadmin.NodeAdminStateUpdater; import com.yahoo.vespa.hosted.node.admin.provider.ComponentsProviderImpl; import com.yahoo.vespa.hosted.node.admin.util.Environment; import com.yahoo.vespa.hosted.node.admin.util.InetAddressResolver; -import com.yahoo.vespa.hosted.node.maintenance.Maintainer; import com.yahoo.vespa.hosted.provision.Node; -import org.junit.Before; import java.io.IOException; -import java.net.HttpURLConnection; +import java.net.InetAddress; +import java.net.MalformedURLException; import java.net.URL; +import java.net.UnknownHostException; import java.nio.file.Path; -import java.nio.file.Paths; -import java.time.Instant; +import java.time.Duration; import java.util.Collections; import java.util.Set; import java.util.concurrent.ExecutionException; import java.util.logging.Logger; import static org.junit.Assert.assertTrue; -import static org.junit.Assume.assumeTrue; -import static org.mockito.Mockito.any; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; /** * <pre> * Requires docker daemon, see {@link com.yahoo.vespa.hosted.dockerapi.DockerTestUtils} for more details. * - * To get started: - * 1. Add config-server and container nodes hostnames to /etc/hosts: - * $ sudo ./vespa/node-admin/scripts/etc-hosts.sh - * 2. Set environmental variables in shell or e.g. ~/.bashrc: - * VESPA_HOME="/home/y" - * VESPA_WEB_SERVICE_PORT="4080" - * - * Linux only: - * 1. Create /home/docker/container-storage with read/write permissions - * - * Example usage: - DockerImage vespaDockerBase = new DockerImage("docker-registry.ops.yahoo.com:4443/vespa/ci:6.52.35"); - Path pathToAppToDeploy = Paths.get("/home/valerijf/dev/basic-search/target/application.zip"); - RunVespaLocal runVespaLocal = new RunVespaLocal(vespaDockerBase, pathToAppToDeploy); - runVespaLocal.runVespaLocalTest(); - * * Issues: * 1. If you cannot make Docker Toolbox start, try starting Virtualbox and turn off the "default" machine * 2. If the above is not enough try "sudo ifconfig vboxnet0 down && sudo ifconfig vboxnet0 up" (see https://github.com/docker/kitematic/issues/1193) @@ -66,34 +44,20 @@ public class RunVespaLocal { private static final Environment environment = new Environment( Collections.singleton(LocalZoneUtils.CONFIG_SERVER_HOSTNAME), "prod", "vespa-local", HostName.getLocalhost(), new InetAddressResolver()); - private static final Maintainer maintainer = mock(Maintainer.class); + private NodeAdminStateUpdater nodeAdminStateUpdater = null; private final Docker docker; - private final DockerImage vespaBaseImage; - private final Path pathToAppToDeploy; - private final Logger logger = Logger.getLogger("RunVespaLocal"); - - RunVespaLocal(DockerImage vespaBaseImage, Path pathToAppToDeploy) { + RunVespaLocal() { this.docker = DockerTestUtils.getDocker(); - this.vespaBaseImage = vespaBaseImage; - this.pathToAppToDeploy = pathToAppToDeploy; } - void runVespaLocalTest() throws IOException, InterruptedException, ExecutionException { - DockerTestUtils.OS operatingSystem = DockerTestUtils.getSystemOS(); - if (operatingSystem == DockerTestUtils.OS.Mac_OS_X) { - when(maintainer.pathInHostFromPathInNode(any(), any())).thenReturn(Paths.get("/tmp/")); - } else { - when(maintainer.pathInHostFromPathInNode(any(), any())).thenCallRealMethod(); - } - when(maintainer.pathInNodeAdminToNodeCleanup(any())).thenReturn(Paths.get("/tmp")); - when(maintainer.pathInNodeAdminFromPathInNode(any(), any())).thenAnswer(invocation -> { - Object[] args = invocation.getArguments(); - return maintainer.pathInHostFromPathInNode((ContainerName) args[0], (String) args[1]); - }); - + /** + * Pulls the base image and builds the vespa-local image + * @param vespaBaseImage Vespa docker image to use as base for the image that the config-server and nodes will run + */ + void buildVespaLocalImage(DockerImage vespaBaseImage) throws ExecutionException, InterruptedException, IOException { if (!docker.imageIsDownloaded(vespaBaseImage)) { logger.info("Pulling " + vespaBaseImage.asString() + " (This may take a while)"); docker.pullImageAsync(vespaBaseImage).get(); @@ -101,49 +65,78 @@ public class RunVespaLocal { logger.info("Building " + LocalZoneUtils.VESPA_LOCAL_IMAGE.asString()); LocalZoneUtils.buildVespaLocalDockerImage(docker, vespaBaseImage); + } + /** + * Starts config server, provisions numNodesToProvision and puts them in ready state + */ + void startLocalZoneWithNodes(int numNodesToProvision) throws IOException, InterruptedException, ExecutionException { logger.info("Starting config-server"); - assertTrue("Could not start config server", LocalZoneUtils.startConfigServerIfNeeded(docker, environment)); + LocalZoneUtils.startConfigServerIfNeeded(docker, environment); + + logger.info("Waiting until config-server is ready to serve"); + URL configServerUrl = new URL("http://" + LocalZoneUtils.CONFIG_SERVER_HOSTNAME + + ":" + LocalZoneUtils.CONFIG_SERVER_WEB_SERVICE_PORT + "/state/v1/health"); + assertTrue("Could not start config server", LocalZoneUtils.isReachableURL(configServerUrl, Duration.ofSeconds(120))); logger.info("Provisioning nodes"); + Set<String> hostnames = LocalZoneUtils.provisionNodes(HostName.getLocalhost(), numNodesToProvision); + hostnames.forEach(hostname -> LocalZoneUtils.setState(Node.State.ready, hostname)); + } + + /** + * Start node-admin in IDE + * @param pathToContainerStorage Path to where the container data will be stored, the path must exist and must + * be writeable by user, normally /home/docker/container-storage + */ + void startNodeAdminInIDE(Path pathToContainerStorage) { + logger.info("Starting node-admin"); + nodeAdminStateUpdater = new ComponentsProviderImpl(docker, + new MetricReceiverWrapper(MetricReceiver.nullImplementation), + new StorageMaintainerMock(LocalZoneUtils.getMaintainer(pathToContainerStorage), new CallOrderVerifier()), + environment).getNodeAdminStateUpdater(); + } + + /** + * Starts node-admin inside a container + * @param pathToContainerStorage Path to where the container data will be stored, the path must exist and must + * be writeable by user, normally /home/docker/container-storage + */ + void startNodeAdminAsContainer(Path pathToContainerStorage) throws UnknownHostException { + String hostname = InetAddress.getByName("172.18.0.1").getHostName(); + logger.info("Provisioning host at " + hostname); + LocalZoneUtils.provisionHost(hostname); + + logger.info("Starting node-admin"); + LocalZoneUtils.startNodeAdminIfNeeded(docker, environment, pathToContainerStorage); try { - Set<String> hostnames = LocalZoneUtils.provisionNodes(HostName.getLocalhost(), 5); - for (String hostname : hostnames) { - try { - LocalZoneUtils.setState(Node.State.ready, hostname); - } catch (RuntimeException e) { - logger.warning(e.getMessage()); - } - } - } catch (RuntimeException e) { - logger.warning(e.getMessage()); + URL nodeUrl = new URL("http://localhost:" + System.getenv("VESPA_WEB_SERVICE_PORT") + "/"); + assertTrue(LocalZoneUtils.isReachableURL(nodeUrl, Duration.ofSeconds(120))); + logger.info("Ready"); + } catch (MalformedURLException e) { + e.printStackTrace(); } + } + /** + * Deploys an app and waits for the node to come up + * @param pathToAppToDeploy Path to .zip file of the application to deploy + */ + void deployApplication(Path pathToAppToDeploy) { logger.info("Deploying application"); LocalZoneUtils.deployApp(docker, pathToAppToDeploy); - logger.info("Starting node-admin"); - NodeAdminStateUpdater nodeAdminStateUpdater = new ComponentsProviderImpl(docker, - new MetricReceiverWrapper(MetricReceiver.nullImplementation), - new StorageMaintainerMock(maintainer, new CallOrderVerifier()), - environment).getNodeAdminStateUpdater(); - - logger.info("Ready"); // TODO: Automatically find correct node to send request to - URL url = new URL("http://cnode-1:" + System.getenv("VESPA_WEB_SERVICE_PORT") + "/"); - Instant start = Instant.now(); - boolean okResponse = false; - do { - try { - HttpURLConnection http = (HttpURLConnection) url.openConnection(); - if (http != null && http.getResponseCode() == 200) okResponse = true; - } catch (IOException e) { - Thread.sleep(100); - } - } while (! okResponse && Instant.now().isBefore(start.plusSeconds(120))); - assertTrue(okResponse); + try { + URL nodeUrl = new URL("http://cnode-1:" + System.getenv("VESPA_WEB_SERVICE_PORT") + "/"); + assertTrue(LocalZoneUtils.isReachableURL(nodeUrl, Duration.ofSeconds(120))); + logger.info("Ready"); + } catch (MalformedURLException e) { + e.printStackTrace(); + } + } + void undeployApplication() { LocalZoneUtils.deleteApplication(); - nodeAdminStateUpdater.deconstruct(); } } |