diff options
Diffstat (limited to 'node-admin')
12 files changed, 75 insertions, 43 deletions
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/ContainerNodeSpec.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/ContainerNodeSpec.java index c5e1da5cd85..b52604e8ca1 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/ContainerNodeSpec.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/ContainerNodeSpec.java @@ -27,9 +27,9 @@ public class ContainerNodeSpec { public final Optional<Long> currentRestartGeneration; public final Optional<Long> wantedRebootGeneration; public final Optional<Long> currentRebootGeneration; - public final Optional<Double> minCpuCores; - public final Optional<Double> minMainMemoryAvailableGb; - public final Optional<Double> minDiskAvailableGb; + public final Double minCpuCores; + public final Double minMainMemoryAvailableGb; + public final Double minDiskAvailableGb; public ContainerNodeSpec( final String hostname, @@ -47,13 +47,16 @@ public class ContainerNodeSpec { final Optional<Long> currentRestartGeneration, final Optional<Long> wantedRebootGeneration, final Optional<Long> currentRebootGeneration, - final Optional<Double> minCpuCores, - final Optional<Double> minMainMemoryAvailableGb, - final Optional<Double> minDiskAvailableGb) { + final Double minCpuCores, + final Double minMainMemoryAvailableGb, + final Double minDiskAvailableGb) { Objects.requireNonNull(hostname); Objects.requireNonNull(nodeState); Objects.requireNonNull(nodeType); Objects.requireNonNull(nodeFlavor); + Objects.requireNonNull(minCpuCores); + Objects.requireNonNull(minMainMemoryAvailableGb); + Objects.requireNonNull(minDiskAvailableGb); this.hostname = hostname; this.wantedDockerImage = wantedDockerImage; @@ -258,9 +261,9 @@ public class ContainerNodeSpec { private Optional<Long> currentRestartGeneration = Optional.empty(); private Optional<Long> wantedRebootGeneration = Optional.empty(); private Optional<Long> currentRebootGeneration = Optional.empty(); - private Optional<Double> minCpuCores = Optional.of(1d); - private Optional<Double> minMainMemoryAvailableGb = Optional.of(1d); - private Optional<Double> minDiskAvailableGb = Optional.of(1d); + private Double minCpuCores; + private Double minMainMemoryAvailableGb; + private Double minDiskAvailableGb; public Builder() {} @@ -270,6 +273,9 @@ public class ContainerNodeSpec { nodeType(nodeSpec.nodeType); nodeFlavor(nodeSpec.nodeFlavor); nodeCanonicalFlavor(nodeSpec.nodeCanonicalFlavor); + minCpuCores(nodeSpec.minCpuCores); + minMainMemoryAvailableGb(nodeSpec.minMainMemoryAvailableGb); + minDiskAvailableGb(nodeSpec.minDiskAvailableGb); nodeSpec.wantedDockerImage.ifPresent(this::wantedDockerImage); nodeSpec.currentDockerImage.ifPresent(this::currentDockerImage); @@ -281,9 +287,6 @@ public class ContainerNodeSpec { nodeSpec.currentRestartGeneration.ifPresent(this::currentRestartGeneration); nodeSpec.wantedRebootGeneration.ifPresent(this::wantedRebootGeneration); nodeSpec.currentRebootGeneration.ifPresent(this::currentRebootGeneration); - nodeSpec.minCpuCores.ifPresent(this::minCpuCores); - nodeSpec.minMainMemoryAvailableGb.ifPresent(this::minMainMemoryAvailableGb); - nodeSpec.minDiskAvailableGb.ifPresent(this::minDiskAvailableGb); } public Builder hostname(String hostname) { @@ -361,17 +364,17 @@ public class ContainerNodeSpec { } public Builder minCpuCores(double minCpuCores) { - this.minCpuCores = Optional.of(minCpuCores); + this.minCpuCores = minCpuCores; return this; } public Builder minMainMemoryAvailableGb(double minMainMemoryAvailableGb) { - this.minMainMemoryAvailableGb = Optional.of(minMainMemoryAvailableGb); + this.minMainMemoryAvailableGb = minMainMemoryAvailableGb; return this; } public Builder minDiskAvailableGb(double minDiskAvailableGb) { - this.minDiskAvailableGb = Optional.of(minDiskAvailableGb); + this.minDiskAvailableGb = minDiskAvailableGb; return this; } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java index 99a7c5abfa0..46787f11bb1 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java @@ -127,22 +127,15 @@ public class DockerOperationsImpl implements DockerOperations { } // TODO: Enforce disk constraints - // TODO: Consider if CPU shares or quota should be set. For now we are just assuming they are - // nicely controlled by docker. - if (nodeSpec.minMainMemoryAvailableGb.isPresent()) { - long minMainMemoryAvailableMb = (long) (nodeSpec.minMainMemoryAvailableGb.get() * 1024); - if (minMainMemoryAvailableMb > 0) { - command.withMemoryInMb(minMainMemoryAvailableMb); - // VESPA_TOTAL_MEMORY_MB is used to make any jdisc container think the machine - // only has this much physical memory (overrides total memory reported by `free -m`). - command.withEnvironment("VESPA_TOTAL_MEMORY_MB", Long.toString(minMainMemoryAvailableMb)); - - // TODO: Remove once the lowest version in prod is 6.95 - command.withEnvironment("TOTAL_MEMORY_MB", Long.toString(minMainMemoryAvailableMb)); - } + long minMainMemoryAvailableMb = (long) (nodeSpec.minMainMemoryAvailableGb * 1024); + if (minMainMemoryAvailableMb > 0) { + command.withMemoryInMb(minMainMemoryAvailableMb); + // VESPA_TOTAL_MEMORY_MB is used to make any jdisc container think the machine + // only has this much physical memory (overrides total memory reported by `free -m`). + command.withEnvironment("VESPA_TOTAL_MEMORY_MB", Long.toString(minMainMemoryAvailableMb)); } - nodeSpec.minCpuCores.ifPresent(cpuShares -> command.withCpuShares((int) Math.round(10 * cpuShares))); + command.withCpuShares((int) Math.round(10 * nodeSpec.minCpuCores)); logger.info("Starting new container with args: " + command); command.create(); diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java index 46990cf3630..c64297956fd 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java @@ -526,7 +526,7 @@ public class NodeAgentImpl implements NodeAgent { final long memoryTotalBytes = ((Number) stats.getMemoryStats().get("limit")).longValue(); final long memoryTotalBytesUsage = ((Number) stats.getMemoryStats().get("usage")).longValue(); final long memoryTotalBytesCache = ((Number) ((Map) stats.getMemoryStats().get("stats")).get("cache")).longValue(); - final Optional<Long> diskTotalBytes = nodeSpec.minDiskAvailableGb.map(size -> (long) (size * bytesInGB)); + final long diskTotalBytes = (long) (nodeSpec.minDiskAvailableGb * bytesInGB); final Optional<Long> diskTotalBytesUsed = storageMaintainer.flatMap(maintainer -> maintainer .getDiskUsageFor(containerName)); @@ -535,20 +535,19 @@ public class NodeAgentImpl implements NodeAgent { // CPU usage by a container as percentage of total CPU allocated to it is given by dividing the // cpuPercentageOfHost with the ratio of container minCpuCores by total number of CPU cores. double cpuPercentageOfHost = lastCpuMetric.getCpuUsagePercentage(cpuContainerTotalTime, cpuSystemTotalTime); - Optional<Double> cpuPercentageOfAllocated = nodeSpec.minCpuCores.map(containerNumCpuCores -> - totalNumCpuCores * cpuPercentageOfHost / containerNumCpuCores); + double cpuPercentageOfAllocated = totalNumCpuCores * cpuPercentageOfHost / nodeSpec.minCpuCores; long memoryTotalBytesUsed = memoryTotalBytesUsage - memoryTotalBytesCache; double memoryPercentUsed = 100.0 * memoryTotalBytesUsed / memoryTotalBytes; - Optional<Double> diskPercentUsed = diskTotalBytes.flatMap(total -> diskTotalBytesUsed.map(used -> 100.0 * used / total)); + Optional<Double> diskPercentUsed = diskTotalBytesUsed.map(used -> 100.0 * used / diskTotalBytes); List<DimensionMetrics> metrics = new ArrayList<>(); DimensionMetrics.Builder systemMetricsBuilder = new DimensionMetrics.Builder(APP, dimensions) .withMetric("mem.limit", memoryTotalBytes) .withMetric("mem.used", memoryTotalBytesUsed) - .withMetric("mem.util", memoryPercentUsed); + .withMetric("mem.util", memoryPercentUsed) + .withMetric("cpu.util", cpuPercentageOfAllocated) + .withMetric("disk.limit", diskTotalBytes); - cpuPercentageOfAllocated.ifPresent(cpuUtil -> systemMetricsBuilder.withMetric("cpu.util", cpuUtil)); - diskTotalBytes.ifPresent(diskLimit -> systemMetricsBuilder.withMetric("disk.limit", diskLimit)); diskTotalBytesUsed.ifPresent(diskUsed -> systemMetricsBuilder.withMetric("disk.used", diskUsed)); diskPercentUsed.ifPresent(diskUtil -> systemMetricsBuilder.withMetric("disk.util", diskUtil)); metrics.add(systemMetricsBuilder.build()); diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/noderepository/NodeRepositoryImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/noderepository/NodeRepositoryImpl.java index a83b7dd61b6..7d73d05ca36 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/noderepository/NodeRepositoryImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/noderepository/NodeRepositoryImpl.java @@ -136,9 +136,9 @@ public class NodeRepositoryImpl implements NodeRepository { Optional.ofNullable(node.currentRestartGeneration), Optional.ofNullable(node.wantedRebootGeneration), Optional.ofNullable(node.currentRestartGeneration), - Optional.ofNullable(node.minCpuCores), - Optional.ofNullable(node.minMainMemoryAvailableGb), - Optional.ofNullable(node.minDiskAvailableGb)); + node.minCpuCores, + node.minMainMemoryAvailableGb, + node.minDiskAvailableGb); } @Override diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerFailTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerFailTest.java index 4db5e4d972d..60a8da80f2b 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerFailTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerFailTest.java @@ -24,6 +24,9 @@ public class DockerFailTest { .nodeFlavor("docker") .wantedRestartGeneration(1L) .currentRestartGeneration(1L) + .minCpuCores(1) + .minMainMemoryAvailableGb(1) + .minDiskAvailableGb(1) .build(); dockerTester.addContainerNodeSpec(containerNodeSpec); diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/MultiDockerTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/MultiDockerTest.java index 542d01655ef..a6bc8a9b4e2 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/MultiDockerTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/MultiDockerTest.java @@ -25,6 +25,9 @@ public class MultiDockerTest { dockerTester.addContainerNodeSpec( new ContainerNodeSpec.Builder(containerNodeSpec2) .nodeState(Node.State.dirty) + .minCpuCores(1) + .minMainMemoryAvailableGb(1) + .minDiskAvailableGb(1) .build()); // Wait until it is marked ready @@ -71,6 +74,9 @@ public class MultiDockerTest { .nodeFlavor("docker") .wantedRestartGeneration(1L) .currentRestartGeneration(1L) + .minCpuCores(1) + .minMainMemoryAvailableGb(1) + .minDiskAvailableGb(1) .build(); tester.addContainerNodeSpec(containerNodeSpec); diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/NodeStateTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/NodeStateTest.java index 71d82215d7d..26194ace7cf 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/NodeStateTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/NodeStateTest.java @@ -23,6 +23,9 @@ public class NodeStateTest { .nodeFlavor("docker") .wantedRestartGeneration(1L) .currentRestartGeneration(1L) + .minCpuCores(1) + .minMainMemoryAvailableGb(1) + .minDiskAvailableGb(1) .build(); private void setup(DockerTester tester) throws InterruptedException { @@ -46,6 +49,9 @@ public class NodeStateTest { // Change node state to dirty dockerTester.addContainerNodeSpec(new ContainerNodeSpec.Builder(initialContainerNodeSpec) .nodeState(Node.State.dirty) + .minCpuCores(1) + .minMainMemoryAvailableGb(1) + .minDiskAvailableGb(1) .build()); // Wait until it is marked ready @@ -73,6 +79,9 @@ public class NodeStateTest { dockerTester.addContainerNodeSpec(new ContainerNodeSpec.Builder(initialContainerNodeSpec) .wantedDockerImage(newDockerImage) .nodeState(Node.State.inactive) + .minCpuCores(1) + .minMainMemoryAvailableGb(1) + .minDiskAvailableGb(1) .build()); dockerTester.callOrderVerifier.assertInOrderWithAssertMessage( @@ -85,6 +94,9 @@ public class NodeStateTest { dockerTester.addContainerNodeSpec(new ContainerNodeSpec.Builder(initialContainerNodeSpec) .wantedDockerImage(newDockerImage) .nodeState(Node.State.active) + .minCpuCores(1) + .minMainMemoryAvailableGb(1) + .minDiskAvailableGb(1) .build()); // Check that the container is started again after the delete call diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/RestartTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/RestartTest.java index de38d7e589d..6922aa6cfe2 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/RestartTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/RestartTest.java @@ -54,6 +54,9 @@ public class RestartTest { .nodeFlavor("docker") .wantedRestartGeneration(wantedRestartGeneration) .currentRestartGeneration(currentRestartGeneration) + .minCpuCores(1) + .minMainMemoryAvailableGb(1) + .minDiskAvailableGb(1) .build(); } } diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/logging/FilebeatConfigProviderTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/logging/FilebeatConfigProviderTest.java index 8d779ba9e1f..399579a7ce7 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/logging/FilebeatConfigProviderTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/logging/FilebeatConfigProviderTest.java @@ -58,6 +58,9 @@ public class FilebeatConfigProviderTest { .nodeState(Node.State.active) .nodeType("type") .hostname("hostname") + .minCpuCores(1) + .minMainMemoryAvailableGb(1) + .minDiskAvailableGb(1) .build(); Optional<String> config = filebeatConfigProvider.getConfig(nodeSpec); assertFalse(config.isPresent()); @@ -113,6 +116,9 @@ public class FilebeatConfigProviderTest { .nodeState(Node.State.active) .nodeType("type") .hostname("hostname") + .minCpuCores(1) + .minMainMemoryAvailableGb(1) + .minDiskAvailableGb(1) .build(); } diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainerTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainerTest.java index 0324ff285d1..95c240b171a 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainerTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainerTest.java @@ -61,7 +61,11 @@ public class StorageMaintainerTest { .hostname(hostname) .nodeState(Node.State.ready) .nodeType("tenants") - .nodeFlavor("docker").build(); + .nodeFlavor("docker") + .minCpuCores(1) + .minMainMemoryAvailableGb(1) + .minDiskAvailableGb(1) + .build(); try { when(processExecuter.exec(any(String[].class))).thenReturn(new Pair<>(0, "")); diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdaterTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdaterTest.java index 28ed462d73b..712000a8cb7 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdaterTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdaterTest.java @@ -55,6 +55,9 @@ public class NodeAdminStateUpdaterTest { .nodeState(i % 3 == 0 ? Node.State.active : Node.State.ready) .nodeType("tenant") .nodeFlavor("docker") + .minCpuCores(1) + .minMainMemoryAvailableGb(1) + .minDiskAvailableGb(1) .build()); } List<String> activeHostnames = Arrays.asList( diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/noderepository/NodeRepositoryImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/noderepository/NodeRepositoryImplTest.java index 3f2ccc156f2..4650e9bf317 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/noderepository/NodeRepositoryImplTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/noderepository/NodeRepositoryImplTest.java @@ -94,9 +94,9 @@ public class NodeRepositoryImplTest { assertThat(nodeSpec.nodeState, is(Node.State.active)); assertThat(nodeSpec.wantedRestartGeneration.get(), is(0L)); assertThat(nodeSpec.currentRestartGeneration.get(), is(0L)); - assertThat(nodeSpec.minCpuCores.get(), is(0.2)); - assertThat(nodeSpec.minMainMemoryAvailableGb.get(), is(0.5)); - assertThat(nodeSpec.minDiskAvailableGb.get(), is(100.0)); + assertThat(nodeSpec.minCpuCores, is(0.2)); + assertThat(nodeSpec.minMainMemoryAvailableGb, is(0.5)); + assertThat(nodeSpec.minDiskAvailableGb, is(100.0)); } @Test |