diff options
author | Jon Marius Venstad <jonmv@users.noreply.github.com> | 2022-02-23 12:47:19 +0100 |
---|---|---|
committer | GitHub <noreply@github.com> | 2022-02-23 12:47:19 +0100 |
commit | 4c1babd6cd22cab305d779999f9f462ad3fa0427 (patch) | |
tree | 44ec3eed5d6d0519137eac8d7b9dffdaf7dbcab6 /node-repository | |
parent | 14719d7d2b44f84f55518d34c3852f652ab66d6c (diff) | |
parent | 6583a1c11ce9b221a803a22c3e1d32057984a0fb (diff) |
Merge pull request #21331 from vespa-engine/bratseth/out-of-capacity
Replace 'out of capacity' by 'node allocation failure'
Diffstat (limited to 'node-repository')
9 files changed, 54 insertions, 53 deletions
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainer.java index e6476cd7373..6eaee7b33de 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainer.java @@ -8,7 +8,7 @@ import com.yahoo.config.provision.ClusterMembership; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; -import com.yahoo.config.provision.OutOfCapacityException; +import com.yahoo.config.provision.NodeAllocationException; import com.yahoo.jdisc.Metric; import com.yahoo.lang.MutableInteger; import com.yahoo.transaction.Mutex; @@ -121,11 +121,11 @@ public class DynamicProvisioningMaintainer extends NodeRepositoryMaintainer { List<Node> excessHosts; try { excessHosts = provision(nodes); - } catch (OutOfCapacityException | IllegalStateException e) { - log.log(Level.WARNING, "Failed to provision preprovision capacity and/or find excess hosts: " + e.getMessage()); + } catch (NodeAllocationException | IllegalStateException e) { + log.log(Level.WARNING, "Failed to allocate preprovisioned capacity and/or find excess hosts: " + e.getMessage()); return; // avoid removing excess hosts } catch (RuntimeException e) { - log.log(Level.WARNING, "Failed to provision preprovision capacity and/or find excess hosts", e); + log.log(Level.WARNING, "Failed to allocate preprovisioned capacity and/or find excess hosts", e); return; // avoid removing excess hosts } @@ -213,7 +213,7 @@ public class DynamicProvisioningMaintainer extends NodeRepositoryMaintainer { /** * @return the nodes in {@code nodeList} plus all hosts provisioned, plus all preprovision capacity * nodes that were allocated. - * @throws OutOfCapacityException if there were problems provisioning hosts, and in case message + * @throws NodeAllocationException if there were problems provisioning hosts, and in case message * should be sufficient (avoid no stack trace) * @throws IllegalStateException if there was an algorithmic problem, and in case message * should be sufficient (avoid no stack trace). @@ -251,8 +251,8 @@ public class DynamicProvisioningMaintainer extends NodeRepositoryMaintainer { .collect(Collectors.toList()); nodeRepository().nodes().addNodes(hosts, Agent.DynamicProvisioningMaintainer); return hosts; - } catch (OutOfCapacityException | IllegalArgumentException | IllegalStateException e) { - throw new OutOfCapacityException("Failed to provision " + count + " " + nodeResources + ": " + e.getMessage()); + } catch (NodeAllocationException | IllegalArgumentException | IllegalStateException e) { + throw new NodeAllocationException("Failed to provision " + count + " " + nodeResources + ": " + e.getMessage()); } catch (RuntimeException e) { throw new RuntimeException("Failed to provision " + count + " " + nodeResources + ", will retry in " + interval(), e); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java index ae65f367684..9148a2165a5 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java @@ -5,7 +5,7 @@ import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.NodeType; -import com.yahoo.config.provision.OutOfCapacityException; +import com.yahoo.config.provision.NodeAllocationException; import com.yahoo.transaction.Mutex; import com.yahoo.vespa.hosted.provision.LockedNodeList; import com.yahoo.vespa.hosted.provision.Node; @@ -125,8 +125,9 @@ public class GroupPreparer { } if (! allocation.fulfilled() && requestedNodes.canFail()) - throw new OutOfCapacityException((cluster.group().isPresent() ? "Out of capacity on " + cluster.group().get() :"") + - allocation.outOfCapacityDetails()); + throw new NodeAllocationException((cluster.group().isPresent() ? "Node allocation failure on " + + cluster.group().get() : "") + + allocation.allocationFailureDetails()); // Carry out and return allocation nodeRepository.nodes().reserve(allocation.reservableNodes()); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java index 6c22a26d88a..7836ca3265e 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java @@ -441,7 +441,7 @@ class NodeAllocation { .collect(Collectors.toList()); } - public String outOfCapacityDetails() { + public String allocationFailureDetails() { List<String> reasons = new ArrayList<>(); if (rejectedDueToExclusivity > 0) reasons.add("host exclusivity constraints"); @@ -453,7 +453,7 @@ class NodeAllocation { reasons.add("insufficient real resources on hosts"); if (reasons.isEmpty()) return ""; - return ": Not enough nodes available due to " + String.join(", ", reasons); + return ": Not enough suitable nodes available due to " + String.join(", ", reasons); } private static Integer parseIndex(String hostname) { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Preparer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Preparer.java index c98ff31ecb6..838d6751d09 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Preparer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Preparer.java @@ -4,7 +4,7 @@ package com.yahoo.vespa.hosted.provision.provisioning; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterMembership; import com.yahoo.config.provision.ClusterSpec; -import com.yahoo.config.provision.OutOfCapacityException; +import com.yahoo.config.provision.NodeAllocationException; import com.yahoo.vespa.hosted.provision.LockedNodeList; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeList; @@ -40,10 +40,10 @@ class Preparer { prepareLoadBalancer(application, cluster, requestedNodes); return nodes; } - catch (OutOfCapacityException e) { - throw new OutOfCapacityException("Could not satisfy " + requestedNodes + - ( wantedGroups > 1 ? " (in " + wantedGroups + " groups)" : "") + - " in " + application + " " + cluster + ": " + e.getMessage()); + catch (NodeAllocationException e) { + throw new NodeAllocationException("Could not satisfy " + requestedNodes + + ( wantedGroups > 1 ? " (in " + wantedGroups + " groups)" : "") + + " in " + application + " " + cluster + ": " + e.getMessage()); } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockHostProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockHostProvisioner.java index 49fc3c2b7b2..0a125c5fb95 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockHostProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockHostProvisioner.java @@ -7,7 +7,7 @@ import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; -import com.yahoo.config.provision.OutOfCapacityException; +import com.yahoo.config.provision.NodeAllocationException; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.node.Address; import com.yahoo.vespa.hosted.provision.node.IP; @@ -59,7 +59,7 @@ public class MockHostProvisioner implements HostProvisioner { Optional<ClusterSpec.Type> clusterType) { Flavor hostFlavor = this.hostFlavor.orElseGet(() -> flavors.stream().filter(f -> compatible(f, resources)) .findFirst() - .orElseThrow(() -> new OutOfCapacityException("No host flavor matches " + resources))); + .orElseThrow(() -> new NodeAllocationException("No host flavor matches " + resources))); List<ProvisionedHost> hosts = new ArrayList<>(); for (int index : provisionIndices) { String hostHostname = hostType == NodeType.host ? "hostname" + index : hostType.name() + index; diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicAllocationTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicAllocationTest.java index 583ccdda656..2cc687e51d3 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicAllocationTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicAllocationTest.java @@ -11,7 +11,7 @@ import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.HostSpec; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; -import com.yahoo.config.provision.OutOfCapacityException; +import com.yahoo.config.provision.NodeAllocationException; import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.Zone; @@ -220,7 +220,7 @@ public class DynamicAllocationTest { assertEquals(2, hostsWithChildren.size()); } - @Test(expected = OutOfCapacityException.class) + @Test(expected = NodeAllocationException.class) public void multiple_groups_are_on_separate_parent_hosts() { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).flavorsConfig(flavorsConfig()).build(); tester.makeReadyNodes(5, "host-small", NodeType.host, 32); @@ -257,7 +257,7 @@ public class DynamicAllocationTest { try { hosts = tester.prepare(application1, clusterSpec, 4, 1, flavor); fail("Was able to deploy with 4 nodes, should not be able to use spare capacity"); - } catch (OutOfCapacityException ignored) { } + } catch (NodeAllocationException ignored) { } tester.fail(hosts.get(0)); hosts = tester.prepare(application1, clusterSpec, 3, 1, flavor); @@ -283,7 +283,7 @@ public class DynamicAllocationTest { try { tester.prepare(application1, clusterSpec, 4, 1, flavor); fail("Should not be able to deploy 4 nodes on 4 hosts because 1 is suspended"); - } catch (OutOfCapacityException ignored) { } + } catch (NodeAllocationException ignored) { } // Resume the host, the deployment goes through tester.orchestrator().resume(randomHost); @@ -319,7 +319,7 @@ public class DynamicAllocationTest { tester.activate(application1, Set.copyOf(hosts)); } - @Test(expected = OutOfCapacityException.class) + @Test(expected = NodeAllocationException.class) public void allocation_should_fail_when_host_is_not_in_allocatable_state() { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).flavorsConfig(flavorsConfig()).build(); tester.makeProvisionedNodes(3, "host-small", NodeType.host, 32).forEach(node -> @@ -377,7 +377,7 @@ public class DynamicAllocationTest { tester.activate(application, hosts); } - private void provisionFastAndSlowThenDeploy(NodeResources.DiskSpeed requestDiskSpeed, boolean expectOutOfCapacity) { + private void provisionFastAndSlowThenDeploy(NodeResources.DiskSpeed requestDiskSpeed, boolean expectNodeAllocationFailure) { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).flavorsConfig(flavorsConfig()).build(); tester.makeReadyNodes(2, new Flavor(new NodeResources(1, 8, 120, 1, NodeResources.DiskSpeed.fast)), NodeType.host, 10, true); tester.makeReadyNodes(2, new Flavor(new NodeResources(1, 8, 120, 1, NodeResources.DiskSpeed.slow)), NodeType.host, 10, true); @@ -389,12 +389,12 @@ public class DynamicAllocationTest { try { List<HostSpec> hosts = tester.prepare(application, cluster, 4, 1, resources); - if (expectOutOfCapacity) fail("Expected out of capacity"); + if (expectNodeAllocationFailure) fail("Expected node allocation fail"); assertEquals(4, hosts.size()); tester.activate(application, hosts); } - catch (OutOfCapacityException e) { - if ( ! expectOutOfCapacity) throw e; + catch (NodeAllocationException e) { + if ( ! expectNodeAllocationFailure) throw e; } } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/InPlaceResizeProvisionTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/InPlaceResizeProvisionTest.java index 70f6d3abf6f..a03811778d2 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/InPlaceResizeProvisionTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/InPlaceResizeProvisionTest.java @@ -7,7 +7,7 @@ import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.HostSpec; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; -import com.yahoo.config.provision.OutOfCapacityException; +import com.yahoo.config.provision.NodeAllocationException; import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.Zone; import com.yahoo.vespa.flags.InMemoryFlagSource; @@ -133,8 +133,8 @@ public class InPlaceResizeProvisionTest { // Attempt to increase resources of the other app try { new PrepareHelper(tester, app).prepare(container1, 4, 1, largeResources); - fail("Expected to fail due to out of capacity"); - } catch (OutOfCapacityException ignored) { } + fail("Expected to fail node allocation"); + } catch (NodeAllocationException ignored) { } // Add 2 more parent host, now we should be able to do the same deployment that failed earlier // 2 of the nodes will be increased in-place and 2 will be allocated to the new hosts. @@ -211,7 +211,7 @@ public class InPlaceResizeProvisionTest { assertSizeAndResources(listCluster(content1).not().retired(), 8, halvedResources); } - @Test(expected = OutOfCapacityException.class) + @Test(expected = NodeAllocationException.class) public void cannot_inplace_decrease_resources_while_increasing_cluster_size() { addParentHosts(6, mediumResources.with(fast).with(local)); @@ -221,7 +221,7 @@ public class InPlaceResizeProvisionTest { new PrepareHelper(tester, app).prepare(content1, 6, 1, smallResources); } - @Test(expected = OutOfCapacityException.class) + @Test(expected = NodeAllocationException.class) public void cannot_inplace_change_resources_while_decreasing_cluster_size() { addParentHosts(4, largeResources.with(fast).with(local)); @@ -231,7 +231,7 @@ public class InPlaceResizeProvisionTest { new PrepareHelper(tester, app).prepare(container1, 2, 1, smallResources); } - @Test(expected = OutOfCapacityException.class) + @Test(expected = NodeAllocationException.class) public void cannot_inplace_change_resources_while_changing_topology() { addParentHosts(4, largeResources.with(fast).with(local)); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java index c9b384b95a6..d0eb55469b6 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java @@ -14,7 +14,7 @@ import com.yahoo.config.provision.HostFilter; import com.yahoo.config.provision.HostSpec; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; -import com.yahoo.config.provision.OutOfCapacityException; +import com.yahoo.config.provision.NodeAllocationException; import com.yahoo.config.provision.ParentHostUnavailableException; import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.SystemName; @@ -333,9 +333,9 @@ public class ProvisioningTest { // redeploy a too large application try { SystemState state2 = prepare(application1, 3, 0, 3, 0, defaultResources, tester); - fail("Expected out of capacity exception"); + fail("Expected node allocation exception"); } - catch (OutOfCapacityException expected) { + catch (NodeAllocationException expected) { } // deploy first state again @@ -626,7 +626,7 @@ public class ProvisioningTest { prepare(application, 2, 2, 3, 3, defaultResources, tester); fail("Expected exception"); } - catch (OutOfCapacityException e) { + catch (NodeAllocationException e) { assertTrue(e.getMessage().startsWith("Could not satisfy request")); } } @@ -653,7 +653,7 @@ public class ProvisioningTest { try { prepare(application, 2, 0, 2, 0, defaultResources, tester); fail("Expected exception"); - } catch (OutOfCapacityException e) { + } catch (NodeAllocationException e) { assertTrue(e.getMessage().startsWith("Could not satisfy request")); } } @@ -826,11 +826,11 @@ public class ProvisioningTest { ApplicationId application = ProvisioningTester.applicationId(); tester.makeReadyHosts(2, defaultResources).activateTenantHosts(); - // Deploy fails with out of capacity + // Node allocation fails try { prepare(application, 2, 0, 2, 0, defaultResources, tester); fail("Expected exception"); - } catch (OutOfCapacityException ignored) {} + } catch (NodeAllocationException ignored) {} assertEquals("Reserved a subset of required nodes", 2, tester.getNodes(application, Node.State.reserved).size()); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/VirtualNodeProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/VirtualNodeProvisioningTest.java index 9a056541c20..1255cb20cd2 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/VirtualNodeProvisioningTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/VirtualNodeProvisioningTest.java @@ -14,7 +14,7 @@ import com.yahoo.config.provision.HostSpec; import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; -import com.yahoo.config.provision.OutOfCapacityException; +import com.yahoo.config.provision.NodeAllocationException; import com.yahoo.config.provision.ProvisionLock; import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.SystemName; @@ -155,7 +155,7 @@ public class VirtualNodeProvisioningTest { assertEquals(2, nodes.stream().filter(n -> n.allocation().get().membership().retired()).count()); } - @Test(expected = OutOfCapacityException.class) + @Test(expected = NodeAllocationException.class) public void fail_when_too_few_distinct_parent_hosts() { ProvisioningTester tester = new ProvisioningTester.Builder().build(); tester.makeReadyChildren(2, resources1, "parentHost1"); @@ -192,10 +192,10 @@ public class VirtualNodeProvisioningTest { tester.makeReadyChildren(1, resources1, "parentHost1"); tester.makeReadyChildren(2, resources1, "parentHost2"); - OutOfCapacityException expectedException = null; + NodeAllocationException expectedException = null; try { tester.prepare(applicationId, contentClusterSpec, contentNodeCount, groups, resources1); - } catch (OutOfCapacityException e) { + } catch (NodeAllocationException e) { expectedException = e; } assertNotNull(expectedException); @@ -279,7 +279,7 @@ public class VirtualNodeProvisioningTest { ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("myContent")).vespaVersion(wantedVespaVersion).build(), nodeCount, 1, resources2); fail("Expected the allocation to fail due to parent hosts not being active yet"); - } catch (OutOfCapacityException expected) { } + } catch (NodeAllocationException expected) { } // Activate the hosts, thereby allocating the parents tester.activateTenantHosts(); @@ -321,7 +321,7 @@ public class VirtualNodeProvisioningTest { 5, 1, resources); fail("Expected exception"); } - catch (OutOfCapacityException e) { + catch (NodeAllocationException e) { // Success: Not enough nonreserved hosts left } @@ -349,8 +349,8 @@ public class VirtualNodeProvisioningTest { tester.prepare(applicationId, ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("myContent")).vespaVersion(wantedVespaVersion).build(), 6, 1, resources); - fail("Expected to fail due to out of capacity"); - } catch (OutOfCapacityException ignored) { } + fail("Expected to fail node allocation"); + } catch (NodeAllocationException ignored) { } // Same cluster, but content type is now 'content' List<HostSpec> nodes = tester.prepare(applicationId, @@ -447,8 +447,8 @@ public class VirtualNodeProvisioningTest { "Could not satisfy request for 3 nodes with " + "[vcpu: 1.0, memory: 4.0 Gb, disk 100.0 Gb, bandwidth: 1.0 Gbps] " + "in tenant2.app2 container cluster 'myContainer' 6.39: " + - "Out of capacity on group 0: " + - "Not enough nodes available due to host exclusivity constraints", + "Node allocation failure on group 0: " + + "Not enough suitable nodes available due to host exclusivity constraints", e.getMessage()); } @@ -470,11 +470,11 @@ public class VirtualNodeProvisioningTest { 2, 1, resources2.with(NodeResources.StorageType.remote)); } - catch (OutOfCapacityException e) { + catch (NodeAllocationException e) { assertEquals("Could not satisfy request for 2 nodes with " + "[vcpu: 1.0, memory: 4.0 Gb, disk 100.0 Gb, bandwidth: 1.0 Gbps, storage type: remote] " + "in tenant.app1 content cluster 'myContent'" + - " 6.42: Out of capacity on group 0", + " 6.42: Node allocation failure on group 0", e.getMessage()); } } |