Add container environment resolver

5 files changed, 91 insertions, 47 deletions

diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/ContainerEnvironmentResolver.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/ContainerEnvironmentResolver.java
new file mode 100644
index 00000000000..a9d9e888e40
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/ContainerEnvironmentResolver.java
@@ -0,0 +1,43 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.component;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.yahoo.vespa.hosted.node.admin.ContainerNodeSpec;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * For creating environment variables that should be available inside a Docker container.
+ * An environment variable CONTAINER_ENVIRONMENT_SETTINGS with the settings here will be
+ * available inside the container. Serializing happens in ContainerEnvironmentSetting in this interface.
+ *
+ * @author hmusum
+ */
+public interface ContainerEnvironmentResolver {
+
+    String createSettings(Environment environment, ContainerNodeSpec nodeSpec);
+
+    class ContainerEnvironmentSettings {
+
+        @JsonProperty(value = "environmentVariables")
+        private final Map<String, Object> variables = new HashMap<>();
+
+        public ContainerEnvironmentSettings set(String argument, Object value) {
+            variables.put(argument, value);
+            return this;
+        }
+
+        public String build() {
+            ObjectMapper objectMapper = new ObjectMapper();
+            try {
+                return objectMapper.writeValueAsString(this);
+            } catch (JsonProcessingException e) {
+                throw new IllegalArgumentException("Could not write " + this, e);
+            }
+        }
+    }
+
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/DefaultContainerEnvironmentResolver.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/DefaultContainerEnvironmentResolver.java
new file mode 100644
index 00000000000..1eff6b5805f
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/DefaultContainerEnvironmentResolver.java
@@ -0,0 +1,19 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.component;
+
+import com.yahoo.vespa.hosted.node.admin.ContainerNodeSpec;
+
+/**
+ * @author hmusum
+ */
+public class DefaultContainerEnvironmentResolver implements ContainerEnvironmentResolver {
+
+    public String createSettings(Environment environment, ContainerNodeSpec nodeSpec) {
+        return new ContainerEnvironmentSettings()
+                .set("configServerAddresses", environment.getConfigServerHostNames())
+                .set("nodeType", nodeSpec.nodeType)
+                .set("cloud", environment.getCloud())
+                .build();
+    }
+
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/Environment.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/Environment.java
index 4015129f695..2176b4bf004 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/Environment.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/Environment.java
@@ -61,7 +61,7 @@ public class Environment {
     private final NodeType nodeType;
     private final String defaultFlavor;
     private final String cloud;
-    private final String jvmArgs;
+    private final ContainerEnvironmentResolver containerEnvironmentResolver;
 
     static {
         filenameFormatter.setTimeZone(TimeZone.getTimeZone("UTC"));
@@ -79,21 +79,23 @@ public class Environment {
              Optional.of(getEnvironmentVariable(COREDUMP_FEED_ENDPOINT)),
              NodeType.host,
              "d-2-8-50",
-             getEnvironmentVariable(CLOUD));
+             getEnvironmentVariable(CLOUD),
+             new DefaultContainerEnvironmentResolver());
     }
 
     private Environment(ConfigServerConfig configServerConfig,
-                       String environment,
-                       String region,
-                       String system,
-                       String parentHostHostname,
-                       InetAddressResolver inetAddressResolver,
-                       PathResolver pathResolver,
-                       List<String> logstashNodes,
-                       Optional<String> coreDumpFeedEndpoint,
-                       NodeType nodeType,
-                       String defaultFlavor,
-                       String cloud) {
+                        String environment,
+                        String region,
+                        String system,
+                        String parentHostHostname,
+                        InetAddressResolver inetAddressResolver,
+                        PathResolver pathResolver,
+                        List<String> logstashNodes,
+                        Optional<String> coreDumpFeedEndpoint,
+                        NodeType nodeType,
+                        String defaultFlavor,
+                        String cloud,
+                        ContainerEnvironmentResolver containerEnvironmentResolver) {
         Objects.requireNonNull(configServerConfig, "configServerConfig cannot be null");
         Objects.requireNonNull(environment, "environment cannot be null");
         Objects.requireNonNull(region, "region cannot be null");
@@ -129,7 +131,7 @@ public class Environment {
         this.nodeType = nodeType;
         this.defaultFlavor = defaultFlavor;
         this.cloud = cloud;
-        this.jvmArgs = configServerConfig.jvmargs();
+        this.containerEnvironmentResolver = containerEnvironmentResolver;
     }
 
     public List<String> getConfigServerHostNames() { return configServerHostNames; }
@@ -274,7 +276,9 @@ public class Environment {
 
     public String getCloud() { return cloud; }
 
-    public String getJvmArgs() { return jvmArgs; }
+    public ContainerEnvironmentResolver getContainerEnvironmentResolver() {
+        return containerEnvironmentResolver;
+    }
 
     public static class Builder {
         private ConfigServerConfig configServerConfig;
@@ -289,6 +293,7 @@ public class Environment {
         private NodeType nodeType = NodeType.tenant;
         private String defaultFlavor;
         private String cloud;
+        private ContainerEnvironmentResolver containerEnvironmentResolver;
 
         public Builder configServerConfig(ConfigServerConfig configServerConfig) {
             this.configServerConfig = configServerConfig;
@@ -325,6 +330,11 @@ public class Environment {
             return this;
         }
 
+        public Builder containerEnvironmentResolver(ContainerEnvironmentResolver containerEnvironmentResolver) {
+            this.containerEnvironmentResolver = containerEnvironmentResolver;
+            return this;
+        }
+
         public Builder logstashNodes(List<String> hosts) {
             this.logstashNodes = hosts;
             return this;
@@ -362,7 +372,8 @@ public class Environment {
                                    coredumpFeedEndpoint,
                                    nodeType,
                                    defaultFlavor,
-                                   cloud);
+                                   cloud,
+                                   Optional.ofNullable(containerEnvironmentResolver).orElseGet(DefaultContainerEnvironmentResolver::new));
         }
     }
 }
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java
index d435ac0cecb..79a10ce09af 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java
@@ -1,9 +1,6 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.node.admin.docker;
 
-import com.fasterxml.jackson.annotation.JsonProperty;
-import com.fasterxml.jackson.core.JsonProcessingException;
-import com.fasterxml.jackson.databind.ObjectMapper;
 import com.yahoo.collections.Pair;
 import com.yahoo.config.provision.NodeType;
 import com.yahoo.system.ProcessExecuter;
@@ -85,7 +82,8 @@ public class DockerOperationsImpl implements DockerOperations {
                     nodeSpec.hostname)
                     .withManagedBy(MANAGER_NAME)
                     .withEnvironment("CONFIG_SERVER_ADDRESS", configServers) // TODO: Remove when all images support CONTAINER_ENVIRONMENT_SETTINGS
-                    .withEnvironment("CONTAINER_ENVIRONMENT_SETTINGS", createContainerEnvironmentSettings(environment, nodeSpec))
+                    .withEnvironment("CONTAINER_ENVIRONMENT_SETTINGS",
+                                     environment.getContainerEnvironmentResolver().createSettings(environment, nodeSpec))
                     .withUlimit("nofile", 262_144, 262_144)
                     .withUlimit("nproc", 32_768, 409_600)
                     .withUlimit("core", -1, -1)
@@ -336,31 +334,6 @@ public class DockerOperationsImpl implements DockerOperations {
         docker.deleteUnusedDockerImages();
     }
 
-    private String createContainerEnvironmentSettings(Environment environment, ContainerNodeSpec nodeSpec) {
-        ObjectMapper objectMapper = new ObjectMapper();
-
-        ContainerEnvironmentSettings settings = new ContainerEnvironmentSettings();
-        settings.set("configServerAddresses", environment.getConfigServerHostNames());
-        settings.set("nodeType", nodeSpec.nodeType);
-        settings.set("cloud", environment.getCloud());
-        settings.set("jvmArgs", environment.getJvmArgs());
-
-        try {
-            return objectMapper.writeValueAsString(settings);
-        } catch (JsonProcessingException e) {
-            throw new IllegalArgumentException("Could not write " + settings, e);
-        }
-    }
-
-    private class ContainerEnvironmentSettings {
-        @JsonProperty(value="environmentVariables")
-        private final Map<String, Object> variables = new HashMap<>();
-
-        void set(String argument, Object value) {
-            variables.put(argument, value);
-        }
-    }
-
     /**
      * Returns map of directories to mount and whether they should be writable by everyone
      */
diff --git a/node-admin/src/main/resources/configdefinitions/config-server.def b/node-admin/src/main/resources/configdefinitions/config-server.def
index 8ef8fe14941..0de79160277 100644
--- a/node-admin/src/main/resources/configdefinitions/config-server.def
+++ b/node-admin/src/main/resources/configdefinitions/config-server.def
@@ -18,5 +18,3 @@ keyStoreConfig.password      string                      default=""
 trustStoreConfig.path        string                      default=""
 trustStoreConfig.type        enum { JKS, PEM, PKCS12 }   default=JKS
 trustStoreConfig.password    string                      default=""
-
-jvmargs                      string                      default=""