Merge pull request #1278 from yahoo/freva/update-node-admin-readme

Freva/update node admin readme

2 files changed, 191 insertions, 100 deletions

diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/docker/LocalZoneUtils.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/docker/LocalZoneUtils.java
index 0f86c9e753f..9d5726d7071 100644
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/docker/LocalZoneUtils.java
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/docker/LocalZoneUtils.java
@@ -1,6 +1,7 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.node.admin.docker;
 
+import com.yahoo.net.HostName;
 import com.yahoo.vespa.defaults.Defaults;
 import com.yahoo.vespa.hosted.dockerapi.Container;
 import com.yahoo.vespa.hosted.dockerapi.ContainerName;
@@ -10,6 +11,7 @@ import com.yahoo.vespa.hosted.dockerapi.DockerImpl;
 import com.yahoo.vespa.hosted.dockerapi.ProcessResult;
 import com.yahoo.vespa.hosted.node.admin.util.ConfigServerHttpRequestExecutor;
 import com.yahoo.vespa.hosted.node.admin.util.Environment;
+import com.yahoo.vespa.hosted.node.maintenance.Maintainer;
 import com.yahoo.vespa.hosted.provision.Node;
 
 import java.io.IOException;
@@ -20,22 +22,31 @@ import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.nio.file.StandardCopyOption;
+import java.time.Duration;
+import java.time.Instant;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
-import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
 import java.util.Set;
+import java.util.stream.Collectors;
+
+import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
 
 /**
  * @author freva
  */
 public class LocalZoneUtils {
+    public static final int CONFIG_SERVER_WEB_SERVICE_PORT = 4080;
     public static final String CONFIG_SERVER_HOSTNAME = "config-server";
     public static final ContainerName CONFIG_SERVER_CONTAINER_NAME = new ContainerName(CONFIG_SERVER_HOSTNAME);
-    public static final int CONFIG_SERVER_WEB_SERVICE_PORT = 4080;
+    public static final String NODE_ADMIN_HOSTNAME = HostName.getLocalhost();
+    public static final ContainerName NODE_ADMIN_CONTAINER_NAME = new ContainerName("node-admin");
     public static final DockerImage VESPA_LOCAL_IMAGE = new DockerImage("vespa-local:latest");
 
     private static final ConfigServerHttpRequestExecutor requestExecutor = ConfigServerHttpRequestExecutor.create(
@@ -45,10 +56,10 @@ public class LocalZoneUtils {
     private static final String APPLICATION_NAME = "default";
     private static final Path PROJECT_ROOT = Paths.get("").toAbsolutePath();
 
-    public static boolean startConfigServerIfNeeded(Docker docker, Environment environment) throws UnknownHostException {
+    public static void startConfigServerIfNeeded(Docker docker, Environment environment) throws UnknownHostException {
         Optional<Container> container = docker.getContainer(CONFIG_SERVER_HOSTNAME);
         if (container.isPresent()) {
-            if (container.get().isRunning) return true;
+            if (container.get().isRunning) return;
             else docker.deleteContainer(CONFIG_SERVER_CONTAINER_NAME);
         }
 
@@ -66,21 +77,52 @@ public class LocalZoneUtils {
                 .create();
 
         docker.startContainer(CONFIG_SERVER_CONTAINER_NAME);
+    }
 
-        int maxRetries = 2000;
-        for (int i = 1; i < maxRetries; i++) {
-            try {
-                if (i % 100 == 0) System.out.println("Check if config server is up, try " + i + " of " + maxRetries);
-
-                URL url = new URL("http://" + CONFIG_SERVER_HOSTNAME + ":" + CONFIG_SERVER_WEB_SERVICE_PORT +
-                        "/state/v1/health");
-                Thread.sleep(100);
-                HttpURLConnection http = (HttpURLConnection) url.openConnection();
-                if (http.getResponseCode() == 200) return true;
-            } catch (IOException | InterruptedException ignored) { }
+    public static void startNodeAdminIfNeeded(Docker docker, Environment environment, Path pathToContainerStorage) {
+        Optional<Container> container = docker.getContainer(NODE_ADMIN_HOSTNAME);
+        if (container.isPresent()) {
+            if (container.get().isRunning) return;
+            else docker.deleteContainer(NODE_ADMIN_CONTAINER_NAME);
         }
 
-        return false;
+        Docker.CreateContainerCommand createCmd = docker.createContainerCommand(VESPA_LOCAL_IMAGE,
+                NODE_ADMIN_CONTAINER_NAME, NODE_ADMIN_HOSTNAME)
+                .withNetworkMode("host")
+                .withVolume("/proc", "/host/proc")
+                .withVolume("/var/run/docker.sock", "/host/var/run/docker.sock")
+                .withVolume(pathToContainerStorage.toString(), "/host" + pathToContainerStorage.toString())
+                .withEnvironment("ENVIRONMENT", environment.getEnvironment())
+                .withEnvironment("REGION", environment.getRegion())
+                .withEnvironment("CONFIG_SERVER_ADDRESS", CONFIG_SERVER_HOSTNAME)
+                .withEnvironment("JPDA_PORT", "localhost:5555")
+                .withEnvironment("JPDA_SUSPEND", "n")
+                .withEnvironment("YJAVA_OPTS", "-Dvespa.freezedetector.disable=true")
+                .withEntrypoint("/usr/local/bin/start-node-admin.sh");
+
+        Arrays.asList(
+                    "/home/y/logs",
+                    "/home/y/var/cache",
+                    "/home/y/var/crash",
+                    "/home/y/var/db/jdisc",
+                    "/home/y/var/db/vespa",
+                    "/home/y/var/jdisc_container",
+                    "/home/y/var/jdisc_core",
+                    "/home/y/var/maven",
+                    "/home/y/var/run",
+                    "/home/y/var/scoreboards",
+                    "/home/y/var/service",
+                    "/home/y/var/share",
+                    "/home/y/var/spool",
+                    "/home/y/var/vespa",
+                    "/home/y/var/yca",
+                    "/home/y/var/ycore++",
+                    "/home/y/var/ymon",
+                    "/home/y/var/zookeeper")
+                .forEach(path -> createCmd.withVolume(pathToContainerStorage.resolve("node-admin" + path).toString(), path));
+
+        createCmd.create();
+        docker.startContainer(NODE_ADMIN_CONTAINER_NAME);
     }
 
     public static void buildVespaLocalDockerImage(Docker docker, DockerImage vespaBaseImage) throws IOException {
@@ -108,11 +150,26 @@ public class LocalZoneUtils {
         docker.buildImage(dockerfilePath.getParent().toFile(), VESPA_LOCAL_IMAGE);
     }
 
+    public static void provisionHost(String hostname) {
+        List<Map<String, String>> nodesToAdd = new ArrayList<>();
+        Map<String, String> provisionNodeRequest = new HashMap<>();
+        provisionNodeRequest.put("type", "host");
+        provisionNodeRequest.put("flavor", "docker");
+        provisionNodeRequest.put("hostname", hostname);
+        provisionNodeRequest.put("openStackId", "fake-" + hostname);
+        nodesToAdd.add(provisionNodeRequest);
+
+        try {
+            requestExecutor.post("/nodes/v2/node", CONFIG_SERVER_WEB_SERVICE_PORT, nodesToAdd, Map.class);
+        } catch (RuntimeException e) {
+            if (! e.getMessage().contains("A node with this name already exists")) throw e;
+        }
+    }
+
     /**
      * Adds numberOfNodes to node-repo and returns a set of node hostnames.
      */
     public static Set<String> provisionNodes(String parentHostname, int numberOfNodes) {
-        Set<String> hostnames = new HashSet<>();
         List<Map<String, String>> nodesToAdd = new ArrayList<>();
         for (int i = 1; i <= numberOfNodes; i++) {
             final String hostname = APP_HOSTNAME_PREFIX + i;
@@ -123,16 +180,23 @@ public class LocalZoneUtils {
             provisionNodeRequest.put("hostname", hostname);
             provisionNodeRequest.put("openStackId", "fake-" + hostname);
             nodesToAdd.add(provisionNodeRequest);
-            hostnames.add(hostname);
         }
 
-        requestExecutor.post("/nodes/v2/node", CONFIG_SERVER_WEB_SERVICE_PORT, nodesToAdd, Map.class);
-        return hostnames;
+        try {
+            requestExecutor.post("/nodes/v2/node", CONFIG_SERVER_WEB_SERVICE_PORT, nodesToAdd, Map.class);
+        } catch (RuntimeException e) {
+            if (! e.getMessage().contains("A node with this name already exists")) throw e;
+        }
+        return nodesToAdd.stream().map(i -> i.get("hostname")).collect(Collectors.toSet());
     }
 
     public static void setState(Node.State state, String hostname) {
-        requestExecutor.put("/nodes/v2/state/" + state + "/" + hostname,
-                CONFIG_SERVER_WEB_SERVICE_PORT, Optional.empty(), Map.class);
+        try {
+            requestExecutor.put("/nodes/v2/state/" + state + "/" + hostname,
+                    CONFIG_SERVER_WEB_SERVICE_PORT, Optional.empty(), Map.class);
+        } catch (RuntimeException e) {
+            if (! e.getMessage().contains("Not registered as provisioned, dirty, failed or parked")) throw e;
+        }
     }
 
     public static void deployApp(Docker docker, Path pathToApp) {
@@ -176,5 +240,39 @@ public class LocalZoneUtils {
         requestExecutor.delete("/application/v2/tenant/" + tenantName + "/application/" + appName,
                 CONFIG_SERVER_WEB_SERVICE_PORT, Map.class);
     }
+
+    public static boolean isReachableURL(URL url, Duration timeout) {
+        Instant start = Instant.now();
+        while (Instant.now().minus(timeout).isBefore(start)) {
+            try {
+                Thread.sleep(100);
+                HttpURLConnection http = (HttpURLConnection) url.openConnection();
+                if (http.getResponseCode() == 200) return true;
+            } catch (IOException | InterruptedException ignored) { }
+        }
+
+        return false;
+    }
+
+    public static Maintainer getMaintainer(Path pathToContainerStorage) {
+        Maintainer maintainer = mock(Maintainer.class);
+        when(maintainer.pathInHostFromPathInNode(any(), any())).thenAnswer(invocation -> {
+            Object[] args = invocation.getArguments();
+            return pathToContainerStorage
+                    .resolve(((ContainerName) args[0]).asString())
+                    .resolve((String) args[1]);
+        });
+        when(maintainer.pathInNodeAdminFromPathInNode(any(), any())).thenAnswer(invocation -> {
+            Object[] args = invocation.getArguments();
+            return maintainer.pathInHostFromPathInNode((ContainerName) args[0], (String) args[1]);
+        });
+        when(maintainer.pathInNodeAdminToNodeCleanup(any())).thenAnswer(invocation -> {
+            Object[] args = invocation.getArguments();
+            return pathToContainerStorage
+                    .resolve("cleanup_" + ((ContainerName) args[0]).asString() + "_" + System.currentTimeMillis());
+        });
+
+        return maintainer;
+    }
 }
 
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/docker/RunVespaLocal.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/docker/RunVespaLocal.java
index 58dd64f48bc..bc333adde22 100644
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/docker/RunVespaLocal.java
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/docker/RunVespaLocal.java
@@ -3,7 +3,6 @@ package com.yahoo.vespa.hosted.node.admin.docker;
 
 import com.yahoo.metrics.simple.MetricReceiver;
 import com.yahoo.net.HostName;
-import com.yahoo.vespa.hosted.dockerapi.ContainerName;
 import com.yahoo.vespa.hosted.dockerapi.Docker;
 import com.yahoo.vespa.hosted.dockerapi.DockerImage;
 import com.yahoo.vespa.hosted.dockerapi.DockerTestUtils;
@@ -14,47 +13,26 @@ import com.yahoo.vespa.hosted.node.admin.nodeadmin.NodeAdminStateUpdater;
 import com.yahoo.vespa.hosted.node.admin.provider.ComponentsProviderImpl;
 import com.yahoo.vespa.hosted.node.admin.util.Environment;
 import com.yahoo.vespa.hosted.node.admin.util.InetAddressResolver;
-import com.yahoo.vespa.hosted.node.maintenance.Maintainer;
 import com.yahoo.vespa.hosted.provision.Node;
-import org.junit.Before;
 
 import java.io.IOException;
-import java.net.HttpURLConnection;
+import java.net.InetAddress;
+import java.net.MalformedURLException;
 import java.net.URL;
+import java.net.UnknownHostException;
 import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.time.Instant;
+import java.time.Duration;
 import java.util.Collections;
 import java.util.Set;
 import java.util.concurrent.ExecutionException;
 import java.util.logging.Logger;
 
 import static org.junit.Assert.assertTrue;
-import static org.junit.Assume.assumeTrue;
-import static org.mockito.Mockito.any;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
 
 /**
  * <pre>
  * Requires docker daemon, see {@link com.yahoo.vespa.hosted.dockerapi.DockerTestUtils} for more details.
  *
- * To get started:
- *  1. Add config-server and container nodes hostnames to /etc/hosts:
- *      $ sudo ./vespa/node-admin/scripts/etc-hosts.sh
- *  2. Set environmental variables in shell or e.g. ~/.bashrc:
- *      VESPA_HOME="/home/y"
- *      VESPA_WEB_SERVICE_PORT="4080"
- *
- * Linux only:
- *  1. Create /home/docker/container-storage with read/write permissions
- *
- * Example usage:
-     DockerImage vespaDockerBase = new DockerImage("docker-registry.ops.yahoo.com:4443/vespa/ci:6.52.35");
-     Path pathToAppToDeploy = Paths.get("/home/valerijf/dev/basic-search/target/application.zip");
-     RunVespaLocal runVespaLocal = new RunVespaLocal(vespaDockerBase, pathToAppToDeploy);
-     runVespaLocal.runVespaLocalTest();
- *
  * Issues:
  *  1. If you cannot make Docker Toolbox start, try starting Virtualbox and turn off the "default" machine
  *  2. If the above is not enough try "sudo ifconfig vboxnet0 down && sudo ifconfig vboxnet0 up" (see https://github.com/docker/kitematic/issues/1193)
@@ -66,34 +44,20 @@ public class RunVespaLocal {
     private static final Environment environment = new Environment(
             Collections.singleton(LocalZoneUtils.CONFIG_SERVER_HOSTNAME), "prod", "vespa-local",
             HostName.getLocalhost(), new InetAddressResolver());
-    private static final Maintainer maintainer = mock(Maintainer.class);
 
+    private NodeAdminStateUpdater nodeAdminStateUpdater = null;
     private final Docker docker;
-    private final DockerImage vespaBaseImage;
-    private final Path pathToAppToDeploy;
-
     private final Logger logger = Logger.getLogger("RunVespaLocal");
 
-
-    RunVespaLocal(DockerImage vespaBaseImage, Path pathToAppToDeploy) {
+    RunVespaLocal() {
         this.docker = DockerTestUtils.getDocker();
-        this.vespaBaseImage = vespaBaseImage;
-        this.pathToAppToDeploy = pathToAppToDeploy;
     }
 
-    void runVespaLocalTest() throws IOException, InterruptedException, ExecutionException {
-        DockerTestUtils.OS operatingSystem = DockerTestUtils.getSystemOS();
-        if (operatingSystem == DockerTestUtils.OS.Mac_OS_X) {
-            when(maintainer.pathInHostFromPathInNode(any(), any())).thenReturn(Paths.get("/tmp/"));
-        } else {
-            when(maintainer.pathInHostFromPathInNode(any(), any())).thenCallRealMethod();
-        }
-        when(maintainer.pathInNodeAdminToNodeCleanup(any())).thenReturn(Paths.get("/tmp"));
-        when(maintainer.pathInNodeAdminFromPathInNode(any(), any())).thenAnswer(invocation -> {
-            Object[] args = invocation.getArguments();
-            return maintainer.pathInHostFromPathInNode((ContainerName) args[0], (String) args[1]);
-        });
-
+    /**
+     * Pulls the base image and builds the vespa-local image
+     * @param vespaBaseImage Vespa docker image to use as base for the image that the config-server and nodes will run
+     */
+    void buildVespaLocalImage(DockerImage vespaBaseImage) throws ExecutionException, InterruptedException, IOException {
         if (!docker.imageIsDownloaded(vespaBaseImage)) {
             logger.info("Pulling " + vespaBaseImage.asString() + " (This may take a while)");
             docker.pullImageAsync(vespaBaseImage).get();
@@ -101,49 +65,78 @@ public class RunVespaLocal {
 
         logger.info("Building " + LocalZoneUtils.VESPA_LOCAL_IMAGE.asString());
         LocalZoneUtils.buildVespaLocalDockerImage(docker, vespaBaseImage);
+    }
 
+    /**
+     * Starts config server, provisions numNodesToProvision and puts them in ready state
+     */
+    void startLocalZoneWithNodes(int numNodesToProvision) throws IOException, InterruptedException, ExecutionException {
         logger.info("Starting config-server");
-        assertTrue("Could not start config server", LocalZoneUtils.startConfigServerIfNeeded(docker, environment));
+        LocalZoneUtils.startConfigServerIfNeeded(docker, environment);
+
+        logger.info("Waiting until config-server is ready to serve");
+        URL configServerUrl = new URL("http://" + LocalZoneUtils.CONFIG_SERVER_HOSTNAME +
+                ":" + LocalZoneUtils.CONFIG_SERVER_WEB_SERVICE_PORT + "/state/v1/health");
+        assertTrue("Could not start config server", LocalZoneUtils.isReachableURL(configServerUrl, Duration.ofSeconds(120)));
 
         logger.info("Provisioning nodes");
+        Set<String> hostnames = LocalZoneUtils.provisionNodes(HostName.getLocalhost(), numNodesToProvision);
+        hostnames.forEach(hostname -> LocalZoneUtils.setState(Node.State.ready, hostname));
+    }
+
+    /**
+     * Start node-admin in IDE
+     * @param pathToContainerStorage Path to where the container data will be stored, the path must exist and must
+     *                               be writeable by user, normally /home/docker/container-storage
+     */
+    void startNodeAdminInIDE(Path pathToContainerStorage) {
+        logger.info("Starting node-admin");
+        nodeAdminStateUpdater = new ComponentsProviderImpl(docker,
+                new MetricReceiverWrapper(MetricReceiver.nullImplementation),
+                new StorageMaintainerMock(LocalZoneUtils.getMaintainer(pathToContainerStorage), new CallOrderVerifier()),
+                environment).getNodeAdminStateUpdater();
+    }
+
+    /**
+     * Starts node-admin inside a container
+     * @param pathToContainerStorage Path to where the container data will be stored, the path must exist and must
+     *                               be writeable by user, normally /home/docker/container-storage
+     */
+    void startNodeAdminAsContainer(Path pathToContainerStorage) throws UnknownHostException {
+        String hostname = InetAddress.getByName("172.18.0.1").getHostName();
+        logger.info("Provisioning host at " + hostname);
+        LocalZoneUtils.provisionHost(hostname);
+
+        logger.info("Starting node-admin");
+        LocalZoneUtils.startNodeAdminIfNeeded(docker, environment, pathToContainerStorage);
         try {
-            Set<String> hostnames = LocalZoneUtils.provisionNodes(HostName.getLocalhost(), 5);
-            for (String hostname : hostnames) {
-                try {
-                    LocalZoneUtils.setState(Node.State.ready, hostname);
-                } catch (RuntimeException e) {
-                    logger.warning(e.getMessage());
-                }
-            }
-        } catch (RuntimeException e) {
-            logger.warning(e.getMessage());
+            URL nodeUrl = new URL("http://localhost:" + System.getenv("VESPA_WEB_SERVICE_PORT") + "/");
+            assertTrue(LocalZoneUtils.isReachableURL(nodeUrl, Duration.ofSeconds(120)));
+            logger.info("Ready");
+        } catch (MalformedURLException e) {
+            e.printStackTrace();
         }
+    }
 
+    /**
+     * Deploys an app and waits for the node to come up
+     * @param pathToAppToDeploy Path to .zip file of the application to deploy
+     */
+    void deployApplication(Path pathToAppToDeploy) {
         logger.info("Deploying application");
         LocalZoneUtils.deployApp(docker, pathToAppToDeploy);
 
-        logger.info("Starting node-admin");
-        NodeAdminStateUpdater nodeAdminStateUpdater = new ComponentsProviderImpl(docker,
-                new MetricReceiverWrapper(MetricReceiver.nullImplementation),
-                new StorageMaintainerMock(maintainer, new CallOrderVerifier()),
-                environment).getNodeAdminStateUpdater();
-
-        logger.info("Ready");
         // TODO: Automatically find correct node to send request to
-        URL url = new URL("http://cnode-1:" + System.getenv("VESPA_WEB_SERVICE_PORT") + "/");
-        Instant start = Instant.now();
-        boolean okResponse = false;
-        do {
-            try {
-                HttpURLConnection http = (HttpURLConnection) url.openConnection();
-                if (http != null && http.getResponseCode() == 200) okResponse = true;
-            } catch (IOException e) {
-                Thread.sleep(100);
-            }
-        } while (! okResponse && Instant.now().isBefore(start.plusSeconds(120)));
-        assertTrue(okResponse);
+        try {
+            URL nodeUrl = new URL("http://cnode-1:" + System.getenv("VESPA_WEB_SERVICE_PORT") + "/");
+            assertTrue(LocalZoneUtils.isReachableURL(nodeUrl, Duration.ofSeconds(120)));
+            logger.info("Ready");
+        } catch (MalformedURLException e) {
+            e.printStackTrace();
+        }
+    }
 
+    void deleteApplication() {
         LocalZoneUtils.deleteApplication();
-        nodeAdminStateUpdater.deconstruct();
     }
 }