Move IPAddresses to public repo and assign IPv4 address to container if available

10 files changed, 304 insertions, 82 deletions

diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/DockerAdminComponent.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/DockerAdminComponent.java
index 89f52a39fbb..a93d77d07bd 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/DockerAdminComponent.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/DockerAdminComponent.java
@@ -18,6 +18,7 @@ import com.yahoo.vespa.hosted.node.admin.nodeadmin.NodeAdminStateUpdaterImpl;
 import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgent;
 import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentImpl;
 import com.yahoo.vespa.hosted.node.admin.provider.NodeAdminStateUpdater;
+import com.yahoo.vespa.hosted.node.admin.task.util.network.IPAddressesImpl;
 
 import java.time.Clock;
 import java.time.Duration;
@@ -93,7 +94,8 @@ public class DockerAdminComponent implements AdminComponent {
         DockerOperations dockerOperations = new DockerOperationsImpl(
                 docker,
                 environment.get(),
-                processExecuter);
+                processExecuter,
+                new IPAddressesImpl());
 
         StorageMaintainer storageMaintainer = new StorageMaintainer(
                 dockerOperations,
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java
index b9b0855fa30..8496d968730 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java
@@ -17,9 +17,11 @@ import com.yahoo.vespa.hosted.dockerapi.ProcessResult;
 import com.yahoo.vespa.hosted.node.admin.ContainerNodeSpec;
 import com.yahoo.vespa.hosted.node.admin.component.Environment;
 import com.yahoo.vespa.hosted.node.admin.maintenance.acl.iptables.NATCommand;
+import com.yahoo.vespa.hosted.node.admin.task.util.network.IPAddresses;
 import com.yahoo.vespa.hosted.node.admin.util.PrefixLogger;
 
 import java.io.IOException;
+import java.net.Inet4Address;
 import java.net.Inet6Address;
 import java.net.InetAddress;
 import java.net.URI;
@@ -42,7 +44,8 @@ public class DockerOperationsImpl implements DockerOperations {
 
     private static final String MANAGER_NAME = "node-admin";
 
-    private static final String LOCAL_IPV6_PREFIX = "fd00::";
+    private static final String IPV6_NPT_PREFIX = "fd00::";
+    private static final String IPV4_NPT_PREFIX = "172.17.0.0";
     private static final String DOCKER_CUSTOM_BRIDGE_NETWORK_NAME = "vespa-bridge";
     
     private final Docker docker;
@@ -50,11 +53,13 @@ public class DockerOperationsImpl implements DockerOperations {
     private final ProcessExecuter processExecuter;
     private final String nodeProgram;
     private Map<Path, Boolean> directoriesToMount;
+    private final IPAddresses retriever;
 
-    public DockerOperationsImpl(Docker docker, Environment environment, ProcessExecuter processExecuter) {
+    public DockerOperationsImpl(Docker docker, Environment environment, ProcessExecuter processExecuter, IPAddresses retriever) {
         this.docker = docker;
         this.environment = environment;
         this.processExecuter = processExecuter;
+        this.retriever = retriever;
 
         this.nodeProgram = environment.pathInNodeUnderVespaHome("bin/vespa-nodectl").toString();
         this.directoriesToMount = getDirectoriesToMount(environment);
@@ -90,10 +95,21 @@ public class DockerOperationsImpl implements DockerOperations {
                 command.withNetworkMode(DockerImpl.DOCKER_CUSTOM_MACVLAN_NETWORK_NAME);
                 command.withVolume("/etc/hosts", "/etc/hosts"); // TODO This is probably not necessary - review later
             } else {
-                command.withIpAddress(NetworkPrefixTranslator.translate(
-                        nodeInetAddress,
-                        InetAddress.getByName(LOCAL_IPV6_PREFIX),
-                        64));
+                // IPv6 - Assume always valid
+                Inet6Address ipV6Address = this.retriever.getIPv6Address(nodeSpec.hostname).orElseThrow(
+                        () -> new RuntimeException("Unable to find a valid IPv6 address. Missing an AAAA DNS entry?"));
+                InetAddress ipV6Prefix = InetAddress.getByName(IPV6_NPT_PREFIX);
+                InetAddress ipV6Local = IPAddresses.prefixTranslate(ipV6Address, ipV6Prefix, 64);
+                command.withIpAddress(ipV6Local);
+
+                // IPv4 - Only present for some containers
+                Optional<Inet4Address> ipV4Address = this.retriever.getIPv4Address(nodeSpec.hostname);
+                if (ipV4Address.isPresent()) {
+                    InetAddress ipV4Prefix = InetAddress.getByName(IPV4_NPT_PREFIX);
+                    InetAddress ipV4Local = IPAddresses.prefixTranslate(ipV4Address.get(), ipV4Prefix, 16);
+                    command.withIpAddress(ipV4Local);
+                }
+
                 command.withNetworkMode(DOCKER_CUSTOM_BRIDGE_NETWORK_NAME);
             }
 
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/NetworkPrefixTranslator.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/NetworkPrefixTranslator.java
deleted file mode 100644
index a52dedb90e5..00000000000
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/NetworkPrefixTranslator.java
+++ /dev/null
@@ -1,38 +0,0 @@
-// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-
-/**
- * @author smorgrav
- */
-package com.yahoo.vespa.hosted.node.admin.docker;
-
-import java.net.Inet6Address;
-import java.net.InetAddress;
-import java.net.UnknownHostException;
-
-class NetworkPrefixTranslator {
-
-    /**
-     * For NPTed networks we want to find the private address from a public.
-     *
-     * @param address    The original address to translate
-     * @param prefix     The prefix address
-     * @param subnetSize in bits - e.g a /64 subnet equals 64 bits
-     * @return The translated address
-     */
-    static Inet6Address translate(InetAddress address, InetAddress prefix, int subnetSize) {
-
-        byte[] originalAddress = address.getAddress();
-        byte[] prefixAddress = prefix.getAddress();
-        byte[] translatedAddress = new byte[16];
-
-        for (int i = 0; i < 16; i++) {
-            translatedAddress[i] = i < subnetSize / 8 ? prefixAddress[i] : originalAddress[i];
-        }
-
-        try {
-            return (Inet6Address) InetAddress.getByAddress(address.getHostName(), translatedAddress);
-        } catch (UnknownHostException e) {
-            throw new RuntimeException(e);
-        }
-    }
-}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/network/IPAddresses.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/network/IPAddresses.java
new file mode 100644
index 00000000000..a49c4e1fbf6
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/network/IPAddresses.java
@@ -0,0 +1,130 @@
+package com.yahoo.vespa.hosted.node.admin.task.util.network;
+
+import com.google.common.net.InetAddresses;
+
+import java.net.Inet4Address;
+import java.net.Inet6Address;
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+import java.util.List;
+import java.util.Optional;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+/**
+ * IP addresses - IP utilities to retrieve and manipulate addresses for docker host and docker containers in a
+ * multi-home environment.
+ *
+ * The assumption is that DNS is the source of truth for which address are assigned to the host and which
+ * that belongs to the containers. Only one address should be assigned to each.
+ *
+ * The behavior with respect to site-local addresses are distinct for IPv4 and IPv6. For IPv4 we choose
+ * the site-local address (assume the public is a NAT address not assigned to the host interface (the typical aws setup)).
+ *
+ * For IPv6 we disregard any site-local addresses (these are normally not in DNS anyway).
+ *
+ * This class also provides some utilities for prefix translation.
+ *
+ * @author smorgrav
+ */
+public interface IPAddresses {
+
+    InetAddress[] getAddresses(String hostname);
+
+    default List<String> getAddresses(String hostname, IPVersion ipVersion) {
+        return Stream.of(getAddresses(hostname))
+                .filter(inetAddress -> isOfType(inetAddress, ipVersion))
+                .map(InetAddresses::toAddrString)
+                .collect(Collectors.toList());
+    }
+
+    /**
+     * Get the IPv6 address for the host if any.
+     *
+     * @throws RuntimeException if multiple addresses are found
+     * @return An optional string representation of the IP address (RFC 5952 compact format)
+     */
+    default Optional<Inet6Address> getIPv6Address(String hostname) {
+        List<InetAddress> ipv6addresses = Stream.of(getAddresses(hostname))
+                .filter(inetAddress -> inetAddress instanceof Inet6Address)
+                .filter(inetAddress -> !inetAddress.isLinkLocalAddress())
+                .filter(inetAddress -> !inetAddress.isSiteLocalAddress())
+                .collect(Collectors.toList());
+
+        if (ipv6addresses.isEmpty()) {
+            return Optional.empty();
+        }
+
+        if (ipv6addresses.size() > 1) {
+            String addresses =
+                    ipv6addresses.stream().map(InetAddresses::toAddrString).collect(Collectors.joining(","));
+            throw new RuntimeException(
+                    String.format(
+                            "Multiple IPv6 addresses found: %s. Perhaps a missing DNS entry or multiple AAAA records in DNS?",
+                            addresses));
+        }
+
+        return Optional.of((Inet6Address)ipv6addresses.get(0));
+    }
+
+    /**
+     * Get the IPv4 address for the host if any.
+     *
+     * @throws RuntimeException if multiple site-local addresses are found
+     * @return An optional string representation of the IP address
+     */
+    default Optional<Inet4Address> getIPv4Address(String hostname) {
+        InetAddress[] allAddresses = getAddresses(hostname);
+
+        List<InetAddress> ipv4Addresses = Stream.of(allAddresses)
+                .filter(inetAddress -> inetAddress instanceof Inet4Address)
+                .collect(Collectors.toList());
+
+        if (ipv4Addresses.size() == 1) return Optional.of((Inet4Address)ipv4Addresses.get(0));
+
+        if (ipv4Addresses.isEmpty()) {
+            Optional.empty();
+        }
+
+        List<InetAddress> siteLocalIPv4Addresses = Stream.of(allAddresses)
+                .filter(inetAddress -> inetAddress instanceof Inet4Address)
+                .filter(InetAddress::isSiteLocalAddress)
+                .collect(Collectors.toList());
+
+        if (siteLocalIPv4Addresses.size() == 1) return Optional.of((Inet4Address)siteLocalIPv4Addresses.get(0));
+
+        String addresses =
+                ipv4Addresses.stream().map(InetAddresses::toAddrString).collect(Collectors.joining(","));
+        throw new RuntimeException(
+                String.format(
+                        "Multiple IPv4 addresses found: %s. Perhaps a missing DNS entry or multiple A records in DNS?",
+                        addresses));
+    }
+
+    static boolean isOfType(InetAddress address, IPVersion ipVersion) {
+        if (ipVersion.equals(IPVersion.IPv4) && address instanceof Inet4Address) return true;
+        if (ipVersion.equals(IPVersion.IPv6) && address instanceof Inet6Address) return true;
+        return false;
+    }
+
+    /**
+     * For NPTed networks we want to find the private address from a public.
+     *
+     * @param address    The original address to translate
+     * @param prefix     The prefix address
+     * @param subnetSizeInBytes in bits - e.g a /64 subnet equals 8 bytes
+     * @return The translated address
+     */
+    static InetAddress prefixTranslate(InetAddress address, InetAddress prefix, int subnetSizeInBytes) {
+        return prefixTranslate(address.getAddress(), prefix.getAddress(), subnetSizeInBytes);
+    }
+
+    static InetAddress prefixTranslate(byte[] address, byte[] prefix, int nofBytes) {
+        System.arraycopy(prefix, 0, address, 0, nofBytes);
+        try {
+            return InetAddress.getByAddress(address);
+        } catch (UnknownHostException e) {
+            throw new RuntimeException(e);
+        }
+    }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/network/IPAddressesImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/network/IPAddressesImpl.java
new file mode 100644
index 00000000000..9ae7181abe9
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/network/IPAddressesImpl.java
@@ -0,0 +1,19 @@
+package com.yahoo.vespa.hosted.node.admin.task.util.network;
+
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+
+/**
+ * @author smorgrav
+ */
+public class IPAddressesImpl implements IPAddresses {
+
+    @Override
+    public InetAddress[] getAddresses(String hostname) {
+        try {
+            return InetAddress.getAllByName(hostname);
+        } catch (UnknownHostException e) {
+            throw new RuntimeException(e);
+        }
+    }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/network/IPVersion.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/network/IPVersion.java
new file mode 100644
index 00000000000..f03f1a57f81
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/network/IPVersion.java
@@ -0,0 +1,22 @@
+package com.yahoo.vespa.hosted.node.admin.task.util.network;
+
+/**
+ * Strong type IPv4 and IPv6 with the respective iptables executable.
+ *
+ * @author smorgrav
+ */
+enum IPVersion {
+
+    IPv6("ip6tables"),
+    IPv4("iptables");
+
+    IPVersion(String exec) {
+        this.exec = exec;
+    }
+
+    private String exec;
+
+    String exec() {
+        return exec;
+    }
+}
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImplTest.java
index e2db9743412..a07bd4e0e91 100644
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImplTest.java
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImplTest.java
@@ -9,6 +9,7 @@ import com.yahoo.vespa.hosted.dockerapi.Docker;
 import com.yahoo.vespa.hosted.dockerapi.DockerImage;
 import com.yahoo.vespa.hosted.dockerapi.ProcessResult;
 import com.yahoo.vespa.hosted.node.admin.component.Environment;
+import com.yahoo.vespa.hosted.node.admin.task.util.network.IPAddressesMock;
 import org.junit.Test;
 import org.mockito.InOrder;
 
@@ -34,7 +35,9 @@ public class DockerOperationsImplTest {
             .build();
     private final Docker docker = mock(Docker.class);
     private final ProcessExecuter processExecuter = mock(ProcessExecuter.class);
-    private final DockerOperationsImpl dockerOperations = new DockerOperationsImpl(docker, environment, processExecuter);
+    private final IPAddressesMock addressesMock = new IPAddressesMock();
+    private final DockerOperationsImpl dockerOperations
+            = new DockerOperationsImpl(docker, environment, processExecuter, addressesMock);
 
     @Test
     public void processResultFromNodeProgramWhenSuccess() throws Exception {
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/docker/NetworkPrefixTranslatorTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/docker/NetworkPrefixTranslatorTest.java
deleted file mode 100644
index 96afe685a61..00000000000
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/docker/NetworkPrefixTranslatorTest.java
+++ /dev/null
@@ -1,36 +0,0 @@
-// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-
-/**
- * @author smorgrav
- */
-package com.yahoo.vespa.hosted.node.admin.docker;
-
-import org.junit.Assert;
-import org.junit.Test;
-
-import java.net.Inet6Address;
-import java.net.InetAddress;
-import java.net.UnknownHostException;
-
-public class NetworkPrefixTranslatorTest {
-
-    @Test
-    public void translator_with_valid_parameters() throws UnknownHostException {
-
-        // Test simplest possible address
-        Inet6Address original = (Inet6Address)InetAddress.getByName("2001:db8::1");
-        Inet6Address prefix = (Inet6Address)InetAddress.getByName("fd00::");
-        Inet6Address translated = NetworkPrefixTranslator.translate(original, prefix, 64);
-        Assert.assertEquals("fd00:0:0:0:0:0:0:1", translated.getHostAddress());
-
-
-        // Test an actual aws address we use
-        original = (Inet6Address)InetAddress.getByName("2600:1f16:f34:5300:ccc6:1703:b7c2:369d");
-        translated = NetworkPrefixTranslator.translate(original, prefix, 64);
-        Assert.assertEquals("fd00:0:0:0:ccc6:1703:b7c2:369d", translated.getHostAddress());
-
-        // Test different subnet size
-        translated = NetworkPrefixTranslator.translate(original, prefix, 48);
-        Assert.assertEquals("fd00:0:0:5300:ccc6:1703:b7c2:369d", translated.getHostAddress());
-    }
-}
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/network/IPAddressesMock.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/network/IPAddressesMock.java
new file mode 100644
index 00000000000..c799ee5eaca
--- /dev/null
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/network/IPAddressesMock.java
@@ -0,0 +1,33 @@
+package com.yahoo.vespa.hosted.node.admin.task.util.network;
+
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * @author smorgrav
+ */
+public class IPAddressesMock implements IPAddresses {
+
+    Map<String, List<InetAddress>> otherAddresses = new HashMap<>();
+
+    IPAddressesMock addAddress(String hostname, String ip) {
+        List<InetAddress> addresses = otherAddresses.getOrDefault(hostname, new ArrayList<>());
+        try {
+            addresses.add(InetAddress.getByName(ip));
+        } catch (UnknownHostException e) {
+            e.printStackTrace();
+        }
+        otherAddresses.put(hostname, addresses);
+        return this;
+    }
+
+    @Override
+    public InetAddress[] getAddresses(String hostname) {
+        List<InetAddress> addresses = otherAddresses.get(hostname);
+        return addresses.toArray(new InetAddress[addresses.size()]);
+    }
+}
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/network/IPAddressesTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/network/IPAddressesTest.java
new file mode 100644
index 00000000000..cac05bcf96c
--- /dev/null
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/network/IPAddressesTest.java
@@ -0,0 +1,71 @@
+package com.yahoo.vespa.hosted.node.admin.task.util.network;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.net.Inet6Address;
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+
+/**
+ * @author smorgrav
+ */
+public class IPAddressesTest {
+
+    private final IPAddressesMock mock = new IPAddressesMock();
+
+    @Test
+    public void choose_sitelocal_ipv4_over_public() {
+        mock.addAddress("localhost", "38.3.4.2")
+                .addAddress("localhost", "10.0.2.2")
+                .addAddress("localhost", "fe80::1")
+                .addAddress("localhost", "2001::1");
+
+        Assert.assertEquals("10.0.2.2", mock.getIPv4Address("localhost"));
+    }
+
+    @Test
+    public void choose_ipv6_public_over_local() {
+        mock.addAddress("localhost", "38.3.4.2")
+                .addAddress("localhost", "10.0.2.2")
+                .addAddress("localhost", "fe80::1")
+                .addAddress("localhost", "2001::1");
+
+        Assert.assertEquals("2001::1", mock.getIPv6Address("localhost"));
+    }
+
+    @Test(expected = RuntimeException.class)
+    public void throws_when_multiple_ipv6_addresses() {
+        mock.addAddress("localhost", "2001::1")
+                .addAddress("localhost", "2001::2");
+        mock.getIPv6Address("localhost");
+    }
+
+    @Test(expected = RuntimeException.class)
+    public void throws_when_multiple_private_ipv4_addresses() {
+        mock.addAddress("localhost", "38.3.4.2")
+                .addAddress("localhost", "10.0.2.2")
+                .addAddress("localhost", "10.0.2.3");
+        mock.getIPv4Address("localhost");
+    }
+
+    @Test
+    public void translator_with_valid_parameters() throws UnknownHostException {
+
+        // Test simplest possible address
+        Inet6Address original = (Inet6Address) InetAddress.getByName("2001:db8::1");
+        Inet6Address prefix = (Inet6Address) InetAddress.getByName("fd00::");
+        InetAddress translated = IPAddresses.prefixTranslate(original, prefix, 64);
+        Assert.assertEquals("fd00:0:0:0:0:0:0:1", translated.getHostAddress());
+
+
+        // Test an actual aws address we use
+        original = (Inet6Address) InetAddress.getByName("2600:1f16:f34:5300:ccc6:1703:b7c2:369d");
+        translated = IPAddresses.prefixTranslate(original, prefix, 64);
+        Assert.assertEquals("fd00:0:0:0:ccc6:1703:b7c2:369d", translated.getHostAddress());
+
+        // Test different subnet size
+        translated = IPAddresses.prefixTranslate(original, prefix, 48);
+        Assert.assertEquals("fd00:0:0:5300:ccc6:1703:b7c2:369d", translated.getHostAddress());
+    }
+}