diff options
6 files changed, 128 insertions, 63 deletions
diff --git a/docker-api/pom.xml b/docker-api/pom.xml index 4c1cf41bb12..1cb03f1819d 100644 --- a/docker-api/pom.xml +++ b/docker-api/pom.xml @@ -36,7 +36,7 @@ <dependency> <groupId>com.github.docker-java</groupId> <artifactId>docker-java</artifactId> - <version>3.0.13</version> + <version>3.1.2</version> <scope>compile</scope> <exclusions> <exclusion> diff --git a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/ContainerStats.java b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/ContainerStats.java index 738a65bc08b..d33ddadb52c 100644 --- a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/ContainerStats.java +++ b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/ContainerStats.java @@ -1,9 +1,18 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.dockerapi; -import java.util.Collections; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.github.dockerjava.api.model.CpuStatsConfig; +import com.github.dockerjava.api.model.MemoryStatsConfig; +import com.github.dockerjava.api.model.StatisticNetworksConfig; +import com.github.dockerjava.api.model.Statistics; + +import java.io.IOException; +import java.io.UncheckedIOException; import java.util.Map; import java.util.Optional; +import java.util.TreeMap; +import java.util.stream.Collectors; /** * Wrapper class for {@link com.github.dockerjava.api.model.Statistics} to prevent leaking from docker-java library. @@ -11,33 +20,103 @@ import java.util.Optional; * @author freva */ public class ContainerStats { - private final Map<String, Object> networks; - private final Map<String, Object> cpuStats; - private final Map<String, Object> memoryStats; - private final Map<String, Object> blkioStats; + private final Map<String, NetworkStats> networkStatsByInterface; + private final MemoryStats memoryStats; + private final CpuStats cpuStats; - public ContainerStats(Map<String, Object> networks, Map<String, Object> cpuStats, - Map<String, Object> memoryStats, Map<String, Object> blkioStats) { + ContainerStats(Statistics statistics) { // Network stats are null when container uses host network - this.networks = Optional.ofNullable(networks).orElse(Collections.emptyMap()); - this.cpuStats = cpuStats; - this.memoryStats = memoryStats; - this.blkioStats = blkioStats; + this.networkStatsByInterface = Optional.ofNullable(statistics.getNetworks()).orElseGet(Map::of) + .entrySet().stream() + .collect(Collectors.toMap( + Map.Entry::getKey, + e -> new NetworkStats(e.getValue()), + (u, v) -> { throw new IllegalStateException(); }, + TreeMap::new)); + this.memoryStats = new MemoryStats(statistics.getMemoryStats()); + this.cpuStats = new CpuStats(statistics.getCpuStats()); + } + + public Map<String, NetworkStats> getNetworks() { + return networkStatsByInterface; } - public Map<String, Object> getNetworks() { - return networks; + public MemoryStats getMemoryStats() { + return memoryStats; } - public Map<String, Object> getCpuStats() { + public CpuStats getCpuStats() { return cpuStats; } - public Map<String, Object> getMemoryStats() { - return memoryStats; + public static class NetworkStats { + private final long rxBytes; + private final long rxDropped; + private final long rxErrors; + private final long txBytes; + private final long txDropped; + private final long txErrors; + + private NetworkStats(StatisticNetworksConfig statisticNetworksConfig) { + this.rxBytes = statisticNetworksConfig.getRxBytes(); + this.rxDropped = statisticNetworksConfig.getRxDropped(); + this.rxErrors = statisticNetworksConfig.getRxErrors(); + this.txBytes = statisticNetworksConfig.getTxBytes(); + this.txDropped = statisticNetworksConfig.getTxDropped(); + this.txErrors = statisticNetworksConfig.getTxErrors(); + } + + public long getRxBytes() { return this.rxBytes; } + public long getRxDropped() { return this.rxDropped; } + public long getRxErrors() { return this.rxErrors; } + public long getTxBytes() { return this.txBytes; } + public long getTxDropped() { return this.txDropped; } + public long getTxErrors() { return this.txErrors; } + } + + public class MemoryStats { + private final long cache; + private final long usage; + private final long limit; + + private MemoryStats(MemoryStatsConfig memoryStats) { + this.cache = memoryStats.getStats().getCache(); + this.usage = memoryStats.getUsage(); + this.limit = memoryStats.getLimit(); + } + + public long getCache() { return this.cache; } + public long getUsage() { return this.usage; } + public long getLimit() { return this.limit; } + } + + public class CpuStats { + private final int onlineCpus; + private final long systemCpuUsage; + private final long totalUsage; + private final long usageInKernelMode; + + public CpuStats(CpuStatsConfig cpuStats) { + // Added in 1.27 + this.onlineCpus = cpuStats.getCpuUsage().getPercpuUsage().size(); + this.systemCpuUsage = cpuStats.getSystemCpuUsage(); + this.totalUsage = cpuStats.getCpuUsage().getTotalUsage(); + this.usageInKernelMode = cpuStats.getCpuUsage().getUsageInKernelmode(); + } + + public int getOnlineCpus() { return this.onlineCpus; } + public long getSystemCpuUsage() { return this.systemCpuUsage; } + public long getTotalUsage() { return totalUsage; } + public long getUsageInKernelMode() { return usageInKernelMode; } } - public Map<String, Object> getBlkioStats() { - return blkioStats; + // For testing only, create ContainerStats from JSON returned by docker daemon stats API + public static ContainerStats fromJson(String json) { + try { + Statistics statistics = new ObjectMapper().readValue(json, Statistics.class); + return new ContainerStats(statistics); + } catch (IOException e) { + throw new UncheckedIOException(e); + } } } diff --git a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/CreateContainerCommandImpl.java b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/CreateContainerCommandImpl.java index db4dc303ab9..2e5cfab36cc 100644 --- a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/CreateContainerCommandImpl.java +++ b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/CreateContainerCommandImpl.java @@ -167,34 +167,34 @@ class CreateContainerCommandImpl implements Docker.CreateContainerCommand { List<Bind> volumeBinds = volumeBindSpecs.stream().map(Bind::parse).collect(Collectors.toList()); final HostConfig hostConfig = new HostConfig() - .withSecurityOpts(new ArrayList<>(securityOpts)); + .withSecurityOpts(new ArrayList<>(securityOpts)) + .withBinds(volumeBinds) + .withUlimits(ulimits) + .withCapAdd(addCapabilities.toArray(new Capability[0])) + .withCapDrop(dropCapabilities.toArray(new Capability[0])) + .withPrivileged(privileged); containerResources.ifPresent(cr -> hostConfig .withCpuShares(cr.cpuShares()) .withMemory(cr.memoryBytes()) // MemorySwap is the total amount of memory and swap, if MemorySwap == Memory, then container has no access swap .withMemorySwap(cr.memoryBytes()) - .withCpuPeriod(cr.cpuQuota() > 0 ? cr.cpuPeriod() : null) - .withCpuQuota(cr.cpuQuota() > 0 ? cr.cpuQuota() : null)); + .withCpuPeriod(cr.cpuQuota() > 0 ? (long) cr.cpuPeriod() : null) + .withCpuQuota(cr.cpuQuota() > 0 ? (long) cr.cpuQuota() : null)); final CreateContainerCmd containerCmd = docker .createContainerCmd(dockerImage.asString()) - .withHostConfig(hostConfig) // MUST BE FIRST (some of the later setters are simply proxied to HostConfig) + .withHostConfig(hostConfig) .withName(containerName.asString()) .withLabels(labels) - .withEnv(environmentAssignments) - .withBinds(volumeBinds) - .withUlimits(ulimits) - .withCapAdd(new ArrayList<>(addCapabilities)) - .withCapDrop(new ArrayList<>(dropCapabilities)) - .withPrivileged(privileged); + .withEnv(environmentAssignments); networkMode .filter(mode -> ! mode.toLowerCase().equals("host")) .ifPresent(mode -> containerCmd.withMacAddress(generateMACAddress(hostName, ipv4Address, ipv6Address))); hostName.ifPresent(containerCmd::withHostName); - networkMode.ifPresent(containerCmd::withNetworkMode); + networkMode.ifPresent(hostConfig::withNetworkMode); ipv4Address.ifPresent(containerCmd::withIpv4Address); ipv6Address.ifPresent(containerCmd::withIpv6Address); entrypoint.ifPresent(containerCmd::withEntrypoint); diff --git a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/DockerImpl.java b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/DockerImpl.java index e57f61ce5f4..a8e08a19d3d 100644 --- a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/DockerImpl.java +++ b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/DockerImpl.java @@ -183,8 +183,7 @@ public class DockerImpl implements Docker { DockerStatsCallback statsCallback = dockerClient.statsCmd(containerName.asString()).exec(new DockerStatsCallback()); statsCallback.awaitCompletion(5, TimeUnit.SECONDS); - return statsCallback.stats.map(stats -> new ContainerStats( - stats.getNetworks(), stats.getCpuStats(), stats.getMemoryStats(), stats.getBlkioStats())); + return statsCallback.stats.map(ContainerStats::new); } catch (NotFoundException ignored) { return Optional.empty(); } catch (RuntimeException | InterruptedException e) { diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java index 57e504a6ffd..96db16bf1c1 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java @@ -33,7 +33,6 @@ import com.yahoo.vespa.hosted.node.admin.util.SecretAgentCheckConfig; import java.util.ArrayList; import java.util.List; -import java.util.Map; import java.util.Objects; import java.util.Optional; import java.util.concurrent.atomic.AtomicBoolean; @@ -534,13 +533,13 @@ public class NodeAgentImpl implements NodeAgent { ContainerStats stats = containerStats.get(); final String APP = MetricReceiverWrapper.APPLICATION_NODE; - final int totalNumCpuCores = ((List<Number>) ((Map) stats.getCpuStats().get("cpu_usage")).get("percpu_usage")).size(); - final long cpuContainerKernelTime = ((Number) ((Map) stats.getCpuStats().get("cpu_usage")).get("usage_in_kernelmode")).longValue(); - final long cpuContainerTotalTime = ((Number) ((Map) stats.getCpuStats().get("cpu_usage")).get("total_usage")).longValue(); - final long cpuSystemTotalTime = ((Number) stats.getCpuStats().get("system_cpu_usage")).longValue(); - final long memoryTotalBytes = ((Number) stats.getMemoryStats().get("limit")).longValue(); - final long memoryTotalBytesUsage = ((Number) stats.getMemoryStats().get("usage")).longValue(); - final long memoryTotalBytesCache = ((Number) ((Map) stats.getMemoryStats().get("stats")).get("cache")).longValue(); + final int totalNumCpuCores = stats.getCpuStats().getOnlineCpus(); + final long cpuContainerKernelTime = stats.getCpuStats().getUsageInKernelMode(); + final long cpuContainerTotalTime = stats.getCpuStats().getTotalUsage(); + final long cpuSystemTotalTime = stats.getCpuStats().getSystemCpuUsage(); + final long memoryTotalBytes = stats.getMemoryStats().getLimit(); + final long memoryTotalBytesUsage = stats.getMemoryStats().getUsage(); + final long memoryTotalBytesCache = stats.getMemoryStats().getCache(); final long diskTotalBytes = (long) (node.getMinDiskAvailableGb() * BYTES_IN_GB); final Optional<Long> diskTotalBytesUsed = storageMaintainer.getDiskUsageFor(context); @@ -573,14 +572,13 @@ public class NodeAgentImpl implements NodeAgent { stats.getNetworks().forEach((interfaceName, interfaceStats) -> { Dimensions netDims = dimensionsBuilder.add("interface", interfaceName).build(); - Map<String, Number> infStats = (Map<String, Number>) interfaceStats; DimensionMetrics networkMetrics = new DimensionMetrics.Builder(APP, netDims) - .withMetric("net.in.bytes", infStats.get("rx_bytes").longValue()) - .withMetric("net.in.errors", infStats.get("rx_errors").longValue()) - .withMetric("net.in.dropped", infStats.get("rx_dropped").longValue()) - .withMetric("net.out.bytes", infStats.get("tx_bytes").longValue()) - .withMetric("net.out.errors", infStats.get("tx_errors").longValue()) - .withMetric("net.out.dropped", infStats.get("tx_dropped").longValue()) + .withMetric("net.in.bytes", interfaceStats.getRxBytes()) + .withMetric("net.in.errors", interfaceStats.getRxErrors()) + .withMetric("net.in.dropped", interfaceStats.getRxDropped()) + .withMetric("net.out.bytes", interfaceStats.getTxBytes()) + .withMetric("net.out.errors", interfaceStats.getTxErrors()) + .withMetric("net.out.dropped", interfaceStats.getTxDropped()) .build(); metrics.add(networkMetrics); }); diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java index dcbbaf792a8..620c59969d6 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java @@ -1,10 +1,10 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.node.admin.nodeagent; -import com.fasterxml.jackson.databind.ObjectMapper; import com.yahoo.component.Version; import com.yahoo.config.provision.DockerImage; import com.yahoo.config.provision.NodeType; +import com.yahoo.io.IOUtils; import com.yahoo.metrics.simple.MetricReceiver; import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.flags.InMemoryFlagSource; @@ -30,7 +30,7 @@ import com.yahoo.vespa.hosted.node.admin.nodeadmin.ConvergenceException; import org.junit.Test; import org.mockito.InOrder; -import java.net.URL; +import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; @@ -82,8 +82,6 @@ public class NodeAgentImplTest { private final MetricReceiverWrapper metricReceiver = new MetricReceiverWrapper(MetricReceiver.nullImplementation); private final AclMaintainer aclMaintainer = mock(AclMaintainer.class); private final HealthChecker healthChecker = mock(HealthChecker.class); - private final ContainerStats emptyContainerStats = new ContainerStats(Collections.emptyMap(), - Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap()); private final CredentialsMaintainer credentialsMaintainer = mock(CredentialsMaintainer.class); private final InMemoryFlagSource flagSource = new InMemoryFlagSource(); @@ -645,18 +643,10 @@ public class NodeAgentImplTest { @Test @SuppressWarnings("unchecked") public void testGetRelevantMetrics() throws Exception { - final ObjectMapper objectMapper = new ObjectMapper(); ClassLoader classLoader = getClass().getClassLoader(); - URL statsFile = classLoader.getResource("docker.stats.json"); - Map<String, Map<String, Object>> dockerStats = objectMapper.readValue(statsFile, Map.class); - - Map<String, Object> networks = dockerStats.get("networks"); - Map<String, Object> precpu_stats = dockerStats.get("precpu_stats"); - Map<String, Object> cpu_stats = dockerStats.get("cpu_stats"); - Map<String, Object> memory_stats = dockerStats.get("memory_stats"); - Map<String, Object> blkio_stats = dockerStats.get("blkio_stats"); - ContainerStats stats1 = new ContainerStats(networks, precpu_stats, memory_stats, blkio_stats); - ContainerStats stats2 = new ContainerStats(networks, cpu_stats, memory_stats, blkio_stats); + String json = IOUtils.readAll(classLoader.getResourceAsStream("docker.stats.json"), StandardCharsets.UTF_8); + ContainerStats stats2 = ContainerStats.fromJson(json); + ContainerStats stats1 = ContainerStats.fromJson(json.replace("\"cpu_stats\"", "\"cpu_stats2\"").replace("\"precpu_stats\"", "\"cpu_stats\"")); NodeOwner owner = new NodeOwner("tester", "testapp", "testinstance"); NodeMembership membership = new NodeMembership("clustType", "clustId", "grp", 3, false); @@ -759,7 +749,6 @@ public class NodeAgentImplTest { private NodeAgentImpl makeNodeAgent(DockerImage dockerImage, boolean isRunning) { mockGetContainer(dockerImage, isRunning); - when(dockerOperations.getContainerStats(any())).thenReturn(Optional.of(emptyContainerStats)); doNothing().when(storageMaintainer).writeMetricsConfig(any()); return new NodeAgentImpl(contextSupplier, nodeRepository, orchestrator, dockerOperations, |