Add data that should be available in the container running a config server

4 files changed, 218 insertions, 1 deletions

diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/containerdata/ConfigServerContainerData.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/containerdata/ConfigServerContainerData.java
new file mode 100644
index 00000000000..53f2f70a74d
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/containerdata/ConfigServerContainerData.java
@@ -0,0 +1,62 @@
+package com.yahoo.vespa.hosted.node.admin.containerdata;
+
+import com.yahoo.vespa.hosted.dockerapi.ContainerName;
+import com.yahoo.vespa.hosted.node.admin.component.Environment;
+
+import java.nio.file.Path;
+import java.nio.file.Paths;
+
+public class ConfigServerContainerData {
+
+    public static final Path configServerAppDir = Paths.get("home/y/conf/configserver-app/");
+
+    private final Environment environment;
+    private final String configServerNodeHostName;
+
+    public ConfigServerContainerData(Environment environment, String configServerNodeHostName) {
+        this.environment = environment;
+        this.configServerNodeHostName = configServerNodeHostName;
+    }
+
+    public void create() {
+        ContainerData containerData = new ContainerData(environment.pathInHostFromPathInNode(
+                ContainerName.fromHostname(configServerNodeHostName),
+                ContainerData.containerDataPath));
+        containerData.addFile(getPath("configserver-config.xml"), createConfigServerConfigXml());
+        containerData.addFile(getPath("node-repository-config.xml"), createNodeRepoConfigXml());
+    }
+
+    private Path getPath(String fileName) {
+        return configServerAppDir.resolve(fileName);
+    }
+
+    private String createConfigServerConfigXml() {
+        return "<config name=\"cloud.config.configserver\">\n" +
+                "  <system>" + environment.getSystem() + "</system>\n" +
+                "  <environment>" + environment.getEnvironment() + "</environment>\n" +
+                "  <region>" + environment.getRegion() + "</region>\n" +
+                "  <hostedVespa>true</hostedVespa>\n" +
+                "  <defaultFlavor>t2.xlarge</defaultFlavor>\n" + // TODO: Avoid hardcoding
+                createZookeeperServers() +
+                "  <serverId>" + configServerNodeHostName + "</serverId>\n" +
+                "  <nodeAdminInContainer>false</nodeAdminInContainer>\n" +
+                "</config>\n";
+    }
+
+    private String createZookeeperServers() {
+        StringBuilder zookeeperServers = new StringBuilder("  <zookeeperserver>\n");
+        environment.getConfigServerHostNames().forEach(hostname -> zookeeperServers.append("  <item>\n" +
+                                                                                                   "    <hostname>" + hostname + "</hostname>\n" +
+                                                                                                   "  </item>\n"));
+        zookeeperServers.append("  </zookeeperserver>\n");
+        return zookeeperServers.toString();
+    }
+
+    // TODO: Avoid hardcoded Docker registry
+    private String createNodeRepoConfigXml() {
+        return "<config name=\"config.provisioning.node-repository\">\n" +
+                "    <dockerImage>658543512185.dkr.ecr.us-east-2.amazonaws.com:4443/vespa/aws</dockerImage>\n" +
+                "</config>\n";
+    }
+
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/containerdata/ContainerData.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/containerdata/ContainerData.java
new file mode 100644
index 00000000000..e5d8105ce01
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/containerdata/ContainerData.java
@@ -0,0 +1,90 @@
+package com.yahoo.vespa.hosted.node.admin.containerdata;
+
+import com.yahoo.io.IOUtils;
+import com.yahoo.log.LogLevel;
+import com.yahoo.text.Utf8;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.NoSuchFileException;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.Collections;
+import java.util.List;
+import java.util.logging.Logger;
+import java.util.stream.Collectors;
+
+/**
+ * This class can be used for adding files that should be accessible in a container's file system after
+ * the container has been started.
+ * <p>
+ * Files that are added will be copied into the destination path on the host.
+ * The entry point for a Docker image will take care of copying
+ * everything into it's right place (ATM only done for config server nodes)
+ * Note: Creating a new instance of this will cleanup all old data in the destination path
+ */
+public class ContainerData {
+
+    private static final Logger log = Logger.getLogger(ContainerData.class.getName());
+    public static final Path containerDataPath = Paths.get("/home/y/var/container-data");
+
+    private final Path destinationPathOnHost;
+
+    public ContainerData(Path destinationPathOnHost) {
+        this.destinationPathOnHost = destinationPathOnHost;
+        try {
+            cleanup(destinationPathOnHost);
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    public void addFile(Path pathInContainer, String data) {
+        Path path = destinationPathOnHost.resolve(pathInContainer);
+        if (!path.toFile().exists()) {
+            try {
+                Files.createDirectories(path.getParent());
+            } catch (IOException e) {
+                throw new RuntimeException(e);
+            }
+        }
+        IOUtils.writeFile(path.toFile(), Utf8.toBytes(data));
+    }
+
+    private void cleanup(Path path) throws IOException {
+        log.log(LogLevel.INFO, "Cleaning up " + path.toAbsolutePath());
+        recursiveDelete(path);
+    }
+
+
+    /*  The below is copied from FileHelper in node-maintainer. Use methods in that class
+        instead when we start depending on node-maintainer
+    */
+
+    /**
+     * Similar to rm -rf file:
+     * - It's not an error if file doesn't exist
+     * - If file is a directory, it and all content is removed
+     * - For symlinks: Only the symlink is removed, not what the symlink points to
+     */
+    private static void recursiveDelete(Path basePath) throws IOException {
+        if (Files.isDirectory(basePath)) {
+            for (Path path : listContentsOfDirectory(basePath)) {
+                recursiveDelete(path);
+            }
+        }
+
+        Files.deleteIfExists(basePath);
+    }
+
+    private static List<Path> listContentsOfDirectory(Path basePath) {
+        try {
+            return Files.list(basePath).collect(Collectors.toList());
+        } catch (NoSuchFileException ignored) {
+            return Collections.emptyList();
+        } catch (IOException e) {
+            throw new RuntimeException("Failed to list contents of directory " + basePath.toAbsolutePath(), e);
+        }
+    }
+
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java
index fcbe4e15213..1d3016d51ef 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java
@@ -3,6 +3,7 @@ package com.yahoo.vespa.hosted.node.admin.nodeagent;
 
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.yahoo.concurrent.ThreadFactoryFactory;
+import com.yahoo.config.provision.NodeType;
 import com.yahoo.vespa.hosted.dockerapi.Container;
 import com.yahoo.vespa.hosted.dockerapi.ContainerName;
 import com.yahoo.vespa.hosted.dockerapi.ContainerResources;
@@ -17,6 +18,7 @@ import com.yahoo.vespa.hosted.dockerapi.metrics.MetricReceiverWrapper;
 import com.yahoo.vespa.hosted.node.admin.ContainerNodeSpec;
 import com.yahoo.vespa.hosted.node.admin.docker.DockerOperations;
 import com.yahoo.vespa.hosted.node.admin.maintenance.StorageMaintainer;
+import com.yahoo.vespa.hosted.node.admin.containerdata.ConfigServerContainerData;
 import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeRepository;
 import com.yahoo.vespa.hosted.node.admin.configserver.orchestrator.Orchestrator;
 import com.yahoo.vespa.hosted.node.admin.configserver.orchestrator.OrchestratorException;
@@ -256,6 +258,7 @@ public class NodeAgentImpl implements NodeAgent {
     }
 
     private void startContainer(ContainerNodeSpec nodeSpec) {
+        createContainerData(nodeSpec);
         dockerOperations.createContainer(containerName, nodeSpec);
         dockerOperations.startContainer(containerName, nodeSpec);
         aclMaintainer.run();
@@ -675,4 +678,11 @@ public class NodeAgentImpl implements NodeAgent {
         logger.info("Ask Orchestrator for permission to suspend node " + hostname);
         orchestrator.suspend(hostname);
     }
+
+    private void createContainerData(ContainerNodeSpec nodeSpec) {
+        if (nodeSpec.nodeType.equals(NodeType.config.name())) {
+            logger.info("Creating files needed by config server");
+            new ConfigServerContainerData(environment, nodeSpec.hostname).create();
+        }
+    }
 }
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java
index 0b9564fad8c..b942ef3ea0c 100644
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java
@@ -12,6 +12,8 @@ import com.yahoo.vespa.hosted.dockerapi.Docker;
 import com.yahoo.vespa.hosted.dockerapi.DockerImage;
 import com.yahoo.vespa.hosted.dockerapi.metrics.MetricReceiverWrapper;
 import com.yahoo.vespa.hosted.node.admin.ContainerNodeSpec;
+import com.yahoo.vespa.hosted.node.admin.containerdata.ConfigServerContainerData;
+import com.yahoo.vespa.hosted.node.admin.containerdata.ContainerData;
 import com.yahoo.vespa.hosted.node.admin.docker.DockerOperations;
 import com.yahoo.vespa.hosted.node.admin.maintenance.StorageMaintainer;
 import com.yahoo.vespa.hosted.node.admin.maintenance.acl.AclMaintainer;
@@ -55,7 +57,7 @@ import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
 
 /**
- * @author bakksjo
+ * @author Øyvind Bakksjø
  */
 public class NodeAgentImplTest {
     private static final Duration NODE_AGENT_SCAN_INTERVAL = Duration.ofSeconds(30);
@@ -623,6 +625,59 @@ public class NodeAgentImplTest {
         assertEquals(Collections.emptySet(), actualMetrics);
     }
 
+    @Test
+    public void testRunningConfigServer() throws IOException {
+        final long rebootGeneration = 0;
+        final ContainerNodeSpec nodeSpec = nodeSpecBuilder
+                .nodeType("config")
+                .wantedDockerImage(dockerImage)
+                .nodeState(Node.State.active)
+                .wantedVespaVersion(vespaVersion)
+                .build();
+
+        System.out.println(nodeSpec);
+
+        NodeAgentImpl nodeAgent = makeNodeAgent(null, false);
+
+        when(nodeRepository.getContainerNodeSpec(hostName)).thenReturn(Optional.of(nodeSpec));
+        Path tempDirectory = Files.createTempDirectory("foo");
+        when(pathResolver.getApplicationStoragePathForHost()).thenReturn(tempDirectory);
+        when(dockerOperations.pullImageAsyncIfNeeded(eq(dockerImage))).thenReturn(false);
+        when(storageMaintainer.getDiskUsageFor(eq(containerName))).thenReturn(Optional.of(201326592000L));
+
+        nodeAgent.converge();
+
+        verify(dockerOperations, never()).removeContainer(any(), any());
+        verify(orchestrator, never()).suspend(any(String.class));
+
+        final InOrder inOrder = inOrder(dockerOperations, orchestrator, nodeRepository, aclMaintainer);
+        inOrder.verify(dockerOperations, times(1)).pullImageAsyncIfNeeded(eq(dockerImage));
+        inOrder.verify(dockerOperations, times(1)).createContainer(eq(containerName), eq(nodeSpec));
+        inOrder.verify(dockerOperations, times(1)).startContainer(eq(containerName), eq(nodeSpec));
+        inOrder.verify(aclMaintainer, times(1)).run();
+        inOrder.verify(dockerOperations, times(1)).resumeNode(eq(containerName));
+        inOrder.verify(nodeRepository).updateNodeAttributes(
+                hostName, new NodeAttributes()
+                        .withRebootGeneration(rebootGeneration)
+                        .withDockerImage(dockerImage)
+                        .withVespaVersion(vespaVersion));
+        inOrder.verify(orchestrator).resume(hostName);
+
+        // Files written in createContainerData()
+        assertFileExists(containerName, tempDirectory, "node-repository-config.xml");
+        assertFileExists(containerName, tempDirectory, "configserver-config.xml");
+    }
+
+    private void assertFileExists(ContainerName containerName, Path tempDirectory, String filename) {
+        File file = tempDirectory
+                .resolve(containerName.asString())
+                .resolve(Paths.get("/").relativize(ContainerData.containerDataPath))
+                .resolve(ConfigServerContainerData.configServerAppDir)
+                .resolve(filename)
+                .toAbsolutePath()
+                .toFile();
+        assertTrue("File " + file + " does not exist", file.exists());
+    }
 
     private NodeAgentImpl makeNodeAgent(DockerImage dockerImage, boolean isRunning) {
         Optional<Container> container = dockerImage != null ?