Upgrade docker-java to 3.1.2

6 files changed, 128 insertions, 63 deletions

diff --git a/docker-api/pom.xml b/docker-api/pom.xml
index 4c1cf41bb12..1cb03f1819d 100644
--- a/docker-api/pom.xml
+++ b/docker-api/pom.xml
@@ -36,7 +36,7 @@
         <dependency>
             <groupId>com.github.docker-java</groupId>
             <artifactId>docker-java</artifactId>
-            <version>3.0.13</version>
+            <version>3.1.2</version>
             <scope>compile</scope>
             <exclusions>
                 <exclusion>
diff --git a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/ContainerStats.java b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/ContainerStats.java
index 738a65bc08b..d33ddadb52c 100644
--- a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/ContainerStats.java
+++ b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/ContainerStats.java
@@ -1,9 +1,18 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.dockerapi;
 
-import java.util.Collections;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.github.dockerjava.api.model.CpuStatsConfig;
+import com.github.dockerjava.api.model.MemoryStatsConfig;
+import com.github.dockerjava.api.model.StatisticNetworksConfig;
+import com.github.dockerjava.api.model.Statistics;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
 import java.util.Map;
 import java.util.Optional;
+import java.util.TreeMap;
+import java.util.stream.Collectors;
 
 /**
  * Wrapper class for {@link com.github.dockerjava.api.model.Statistics} to prevent leaking from docker-java library.
@@ -11,33 +20,103 @@ import java.util.Optional;
  * @author freva
  */
 public class ContainerStats {
-    private final Map<String, Object> networks;
-    private final Map<String, Object> cpuStats;
-    private final Map<String, Object> memoryStats;
-    private final Map<String, Object> blkioStats;
+    private final Map<String, NetworkStats> networkStatsByInterface;
+    private final MemoryStats memoryStats;
+    private final CpuStats cpuStats;
 
-    public ContainerStats(Map<String, Object> networks, Map<String, Object> cpuStats,
-                          Map<String, Object> memoryStats, Map<String, Object> blkioStats) {
+    ContainerStats(Statistics statistics) {
         // Network stats are null when container uses host network
-        this.networks = Optional.ofNullable(networks).orElse(Collections.emptyMap());
-        this.cpuStats = cpuStats;
-        this.memoryStats = memoryStats;
-        this.blkioStats = blkioStats;
+        this.networkStatsByInterface = Optional.ofNullable(statistics.getNetworks()).orElseGet(Map::of)
+                .entrySet().stream()
+                .collect(Collectors.toMap(
+                        Map.Entry::getKey,
+                        e -> new NetworkStats(e.getValue()),
+                        (u, v) -> { throw new IllegalStateException(); },
+                        TreeMap::new));
+        this.memoryStats = new MemoryStats(statistics.getMemoryStats());
+        this.cpuStats = new CpuStats(statistics.getCpuStats());
+    }
+
+    public Map<String, NetworkStats> getNetworks() {
+        return networkStatsByInterface;
     }
 
-    public Map<String, Object> getNetworks() {
-        return networks;
+    public MemoryStats getMemoryStats() {
+        return memoryStats;
     }
 
-    public Map<String, Object> getCpuStats() {
+    public CpuStats getCpuStats() {
         return cpuStats;
     }
 
-    public Map<String, Object> getMemoryStats() {
-        return memoryStats;
+    public static class NetworkStats {
+        private final long rxBytes;
+        private final long rxDropped;
+        private final long rxErrors;
+        private final long txBytes;
+        private final long txDropped;
+        private final long txErrors;
+
+        private NetworkStats(StatisticNetworksConfig statisticNetworksConfig) {
+            this.rxBytes = statisticNetworksConfig.getRxBytes();
+            this.rxDropped = statisticNetworksConfig.getRxDropped();
+            this.rxErrors = statisticNetworksConfig.getRxErrors();
+            this.txBytes = statisticNetworksConfig.getTxBytes();
+            this.txDropped = statisticNetworksConfig.getTxDropped();
+            this.txErrors = statisticNetworksConfig.getTxErrors();
+        }
+
+        public long getRxBytes() { return this.rxBytes; }
+        public long getRxDropped() { return this.rxDropped; }
+        public long getRxErrors() { return this.rxErrors; }
+        public long getTxBytes() { return this.txBytes; }
+        public long getTxDropped() { return this.txDropped; }
+        public long getTxErrors() { return this.txErrors; }
+    }
+
+    public class MemoryStats {
+        private final long cache;
+        private final long usage;
+        private final long limit;
+
+        private MemoryStats(MemoryStatsConfig memoryStats) {
+            this.cache = memoryStats.getStats().getCache();
+            this.usage = memoryStats.getUsage();
+            this.limit = memoryStats.getLimit();
+        }
+
+        public long getCache() { return this.cache; }
+        public long getUsage() { return this.usage; }
+        public long getLimit() { return this.limit; }
+    }
+
+    public class CpuStats {
+        private final int onlineCpus;
+        private final long systemCpuUsage;
+        private final long totalUsage;
+        private final long usageInKernelMode;
+
+        public CpuStats(CpuStatsConfig cpuStats) {
+            // Added in 1.27
+            this.onlineCpus = cpuStats.getCpuUsage().getPercpuUsage().size();
+            this.systemCpuUsage = cpuStats.getSystemCpuUsage();
+            this.totalUsage = cpuStats.getCpuUsage().getTotalUsage();
+            this.usageInKernelMode = cpuStats.getCpuUsage().getUsageInKernelmode();
+        }
+
+        public int getOnlineCpus() { return this.onlineCpus; }
+        public long getSystemCpuUsage() { return this.systemCpuUsage; }
+        public long getTotalUsage() { return totalUsage; }
+        public long getUsageInKernelMode() { return usageInKernelMode; }
     }
 
-    public Map<String, Object> getBlkioStats() {
-        return blkioStats;
+    // For testing only, create ContainerStats from JSON returned by docker daemon stats API
+    public static ContainerStats fromJson(String json) {
+        try {
+            Statistics statistics = new ObjectMapper().readValue(json, Statistics.class);
+            return new ContainerStats(statistics);
+        } catch (IOException e) {
+            throw new UncheckedIOException(e);
+        }
     }
 }
diff --git a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/CreateContainerCommandImpl.java b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/CreateContainerCommandImpl.java
index db4dc303ab9..2e5cfab36cc 100644
--- a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/CreateContainerCommandImpl.java
+++ b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/CreateContainerCommandImpl.java
@@ -167,34 +167,34 @@ class CreateContainerCommandImpl implements Docker.CreateContainerCommand {
         List<Bind> volumeBinds = volumeBindSpecs.stream().map(Bind::parse).collect(Collectors.toList());
 
         final HostConfig hostConfig = new HostConfig()
-                .withSecurityOpts(new ArrayList<>(securityOpts));
+                .withSecurityOpts(new ArrayList<>(securityOpts))
+                .withBinds(volumeBinds)
+                .withUlimits(ulimits)
+                .withCapAdd(addCapabilities.toArray(new Capability[0]))
+                .withCapDrop(dropCapabilities.toArray(new Capability[0]))
+                .withPrivileged(privileged);
 
         containerResources.ifPresent(cr -> hostConfig
                 .withCpuShares(cr.cpuShares())
                 .withMemory(cr.memoryBytes())
                 // MemorySwap is the total amount of memory and swap, if MemorySwap == Memory, then container has no access swap
                 .withMemorySwap(cr.memoryBytes())
-                .withCpuPeriod(cr.cpuQuota() > 0 ? cr.cpuPeriod() : null)
-                .withCpuQuota(cr.cpuQuota() > 0 ? cr.cpuQuota() : null));
+                .withCpuPeriod(cr.cpuQuota() > 0 ? (long) cr.cpuPeriod() : null)
+                .withCpuQuota(cr.cpuQuota() > 0 ? (long) cr.cpuQuota() : null));
 
         final CreateContainerCmd containerCmd = docker
                 .createContainerCmd(dockerImage.asString())
-                .withHostConfig(hostConfig) // MUST BE FIRST (some of the later setters are simply proxied to HostConfig)
+                .withHostConfig(hostConfig)
                 .withName(containerName.asString())
                 .withLabels(labels)
-                .withEnv(environmentAssignments)
-                .withBinds(volumeBinds)
-                .withUlimits(ulimits)
-                .withCapAdd(new ArrayList<>(addCapabilities))
-                .withCapDrop(new ArrayList<>(dropCapabilities))
-                .withPrivileged(privileged);
+                .withEnv(environmentAssignments);
 
         networkMode
                 .filter(mode -> ! mode.toLowerCase().equals("host"))
                 .ifPresent(mode -> containerCmd.withMacAddress(generateMACAddress(hostName, ipv4Address, ipv6Address)));
 
         hostName.ifPresent(containerCmd::withHostName);
-        networkMode.ifPresent(containerCmd::withNetworkMode);
+        networkMode.ifPresent(hostConfig::withNetworkMode);
         ipv4Address.ifPresent(containerCmd::withIpv4Address);
         ipv6Address.ifPresent(containerCmd::withIpv6Address);
         entrypoint.ifPresent(containerCmd::withEntrypoint);
diff --git a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/DockerImpl.java b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/DockerImpl.java
index e57f61ce5f4..a8e08a19d3d 100644
--- a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/DockerImpl.java
+++ b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/DockerImpl.java
@@ -183,8 +183,7 @@ public class DockerImpl implements Docker {
             DockerStatsCallback statsCallback = dockerClient.statsCmd(containerName.asString()).exec(new DockerStatsCallback());
             statsCallback.awaitCompletion(5, TimeUnit.SECONDS);
 
-            return statsCallback.stats.map(stats -> new ContainerStats(
-                    stats.getNetworks(), stats.getCpuStats(), stats.getMemoryStats(), stats.getBlkioStats()));
+            return statsCallback.stats.map(ContainerStats::new);
         } catch (NotFoundException ignored) {
             return Optional.empty();
         } catch (RuntimeException | InterruptedException e) {
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java
index 57e504a6ffd..96db16bf1c1 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java
@@ -33,7 +33,6 @@ import com.yahoo.vespa.hosted.node.admin.util.SecretAgentCheckConfig;
 
 import java.util.ArrayList;
 import java.util.List;
-import java.util.Map;
 import java.util.Objects;
 import java.util.Optional;
 import java.util.concurrent.atomic.AtomicBoolean;
@@ -534,13 +533,13 @@ public class NodeAgentImpl implements NodeAgent {
 
         ContainerStats stats = containerStats.get();
         final String APP = MetricReceiverWrapper.APPLICATION_NODE;
-        final int totalNumCpuCores = ((List<Number>) ((Map) stats.getCpuStats().get("cpu_usage")).get("percpu_usage")).size();
-        final long cpuContainerKernelTime = ((Number) ((Map) stats.getCpuStats().get("cpu_usage")).get("usage_in_kernelmode")).longValue();
-        final long cpuContainerTotalTime = ((Number) ((Map) stats.getCpuStats().get("cpu_usage")).get("total_usage")).longValue();
-        final long cpuSystemTotalTime = ((Number) stats.getCpuStats().get("system_cpu_usage")).longValue();
-        final long memoryTotalBytes = ((Number) stats.getMemoryStats().get("limit")).longValue();
-        final long memoryTotalBytesUsage = ((Number) stats.getMemoryStats().get("usage")).longValue();
-        final long memoryTotalBytesCache = ((Number) ((Map) stats.getMemoryStats().get("stats")).get("cache")).longValue();
+        final int totalNumCpuCores = stats.getCpuStats().getOnlineCpus();
+        final long cpuContainerKernelTime = stats.getCpuStats().getUsageInKernelMode();
+        final long cpuContainerTotalTime = stats.getCpuStats().getTotalUsage();
+        final long cpuSystemTotalTime = stats.getCpuStats().getSystemCpuUsage();
+        final long memoryTotalBytes = stats.getMemoryStats().getLimit();
+        final long memoryTotalBytesUsage = stats.getMemoryStats().getUsage();
+        final long memoryTotalBytesCache = stats.getMemoryStats().getCache();
         final long diskTotalBytes = (long) (node.getMinDiskAvailableGb() * BYTES_IN_GB);
         final Optional<Long> diskTotalBytesUsed = storageMaintainer.getDiskUsageFor(context);
 
@@ -573,14 +572,13 @@ public class NodeAgentImpl implements NodeAgent {
 
         stats.getNetworks().forEach((interfaceName, interfaceStats) -> {
             Dimensions netDims = dimensionsBuilder.add("interface", interfaceName).build();
-            Map<String, Number> infStats = (Map<String, Number>) interfaceStats;
             DimensionMetrics networkMetrics = new DimensionMetrics.Builder(APP, netDims)
-                    .withMetric("net.in.bytes", infStats.get("rx_bytes").longValue())
-                    .withMetric("net.in.errors", infStats.get("rx_errors").longValue())
-                    .withMetric("net.in.dropped", infStats.get("rx_dropped").longValue())
-                    .withMetric("net.out.bytes", infStats.get("tx_bytes").longValue())
-                    .withMetric("net.out.errors", infStats.get("tx_errors").longValue())
-                    .withMetric("net.out.dropped", infStats.get("tx_dropped").longValue())
+                    .withMetric("net.in.bytes", interfaceStats.getRxBytes())
+                    .withMetric("net.in.errors", interfaceStats.getRxErrors())
+                    .withMetric("net.in.dropped", interfaceStats.getRxDropped())
+                    .withMetric("net.out.bytes", interfaceStats.getTxBytes())
+                    .withMetric("net.out.errors", interfaceStats.getTxErrors())
+                    .withMetric("net.out.dropped", interfaceStats.getTxDropped())
                     .build();
             metrics.add(networkMetrics);
         });
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java
index dcbbaf792a8..620c59969d6 100644
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java
@@ -1,10 +1,10 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.node.admin.nodeagent;
 
-import com.fasterxml.jackson.databind.ObjectMapper;
 import com.yahoo.component.Version;
 import com.yahoo.config.provision.DockerImage;
 import com.yahoo.config.provision.NodeType;
+import com.yahoo.io.IOUtils;
 import com.yahoo.metrics.simple.MetricReceiver;
 import com.yahoo.vespa.flags.Flags;
 import com.yahoo.vespa.flags.InMemoryFlagSource;
@@ -30,7 +30,7 @@ import com.yahoo.vespa.hosted.node.admin.nodeadmin.ConvergenceException;
 import org.junit.Test;
 import org.mockito.InOrder;
 
-import java.net.URL;
+import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
@@ -82,8 +82,6 @@ public class NodeAgentImplTest {
     private final MetricReceiverWrapper metricReceiver = new MetricReceiverWrapper(MetricReceiver.nullImplementation);
     private final AclMaintainer aclMaintainer = mock(AclMaintainer.class);
     private final HealthChecker healthChecker = mock(HealthChecker.class);
-    private final ContainerStats emptyContainerStats = new ContainerStats(Collections.emptyMap(),
-            Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap());
     private final CredentialsMaintainer credentialsMaintainer = mock(CredentialsMaintainer.class);
     private final InMemoryFlagSource flagSource = new InMemoryFlagSource();
 
@@ -645,18 +643,10 @@ public class NodeAgentImplTest {
     @Test
     @SuppressWarnings("unchecked")
     public void testGetRelevantMetrics() throws Exception {
-        final ObjectMapper objectMapper = new ObjectMapper();
         ClassLoader classLoader = getClass().getClassLoader();
-        URL statsFile = classLoader.getResource("docker.stats.json");
-        Map<String, Map<String, Object>> dockerStats = objectMapper.readValue(statsFile, Map.class);
-
-        Map<String, Object> networks = dockerStats.get("networks");
-        Map<String, Object> precpu_stats = dockerStats.get("precpu_stats");
-        Map<String, Object> cpu_stats = dockerStats.get("cpu_stats");
-        Map<String, Object> memory_stats = dockerStats.get("memory_stats");
-        Map<String, Object> blkio_stats = dockerStats.get("blkio_stats");
-        ContainerStats stats1 = new ContainerStats(networks, precpu_stats, memory_stats, blkio_stats);
-        ContainerStats stats2 = new ContainerStats(networks, cpu_stats, memory_stats, blkio_stats);
+        String json = IOUtils.readAll(classLoader.getResourceAsStream("docker.stats.json"), StandardCharsets.UTF_8);
+        ContainerStats stats2 = ContainerStats.fromJson(json);
+        ContainerStats stats1 = ContainerStats.fromJson(json.replace("\"cpu_stats\"", "\"cpu_stats2\"").replace("\"precpu_stats\"", "\"cpu_stats\""));
 
         NodeOwner owner = new NodeOwner("tester", "testapp", "testinstance");
         NodeMembership membership = new NodeMembership("clustType", "clustId", "grp", 3, false);
@@ -759,7 +749,6 @@ public class NodeAgentImplTest {
     private NodeAgentImpl makeNodeAgent(DockerImage dockerImage, boolean isRunning) {
         mockGetContainer(dockerImage, isRunning);
 
-        when(dockerOperations.getContainerStats(any())).thenReturn(Optional.of(emptyContainerStats));
         doNothing().when(storageMaintainer).writeMetricsConfig(any());
 
         return new NodeAgentImpl(contextSupplier, nodeRepository, orchestrator, dockerOperations,