diff options
6 files changed, 372 insertions, 16 deletions
diff --git a/node-repository/pom.xml b/node-repository/pom.xml index 6c7dbf402f8..164c5dd4f7c 100644 --- a/node-repository/pom.xml +++ b/node-repository/pom.xml @@ -60,26 +60,10 @@ <scope>provided</scope> </dependency> <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>testutil</artifactId> - <version>${project.version}</version> - <scope>test</scope> - </dependency> - <dependency> <groupId>com.google.guava</groupId> <artifactId>guava</artifactId> </dependency> <dependency> - <groupId>junit</groupId> - <artifactId>junit</artifactId> - <scope>test</scope> - </dependency> - <dependency> - <groupId>org.apache.curator</groupId> - <artifactId>curator-test</artifactId> - <scope>test</scope> - </dependency> - <dependency> <groupId>com.fasterxml.jackson.datatype</groupId> <artifactId>jackson-datatype-joda</artifactId> <version>${jackson2.version}</version> @@ -105,6 +89,28 @@ <version>${project.version}</version> <scope>provided</scope> </dependency> + + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>testutil</artifactId> + <version>${project.version}</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>junit</groupId> + <artifactId>junit</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.apache.curator</groupId> + <artifactId>curator-test</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.mockito</groupId> + <artifactId>mockito-core</artifactId> + <scope>test</scope> + </dependency> </dependencies> <build> <plugins> diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRetirer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRetirer.java new file mode 100644 index 00000000000..80199ccacae --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRetirer.java @@ -0,0 +1,78 @@ +package com.yahoo.vespa.hosted.provision.maintenance; + +import com.yahoo.config.provision.Flavor; +import com.yahoo.transaction.Mutex; +import com.yahoo.vespa.hosted.provision.Node; +import com.yahoo.vespa.hosted.provision.NodeRepository; +import com.yahoo.vespa.hosted.provision.maintenance.retire.RetirementPolicy; +import com.yahoo.vespa.hosted.provision.node.Agent; + +import java.time.Duration; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +/** + * @author freva + */ +public class NodeRetirer extends Maintainer { + private final RetirementPolicy retirementPolicy; + + public NodeRetirer(NodeRepository nodeRepository, Duration interval, RetirementPolicy retirementPolicy) { + super(nodeRepository, interval); + this.retirementPolicy = retirementPolicy; + } + + @Override + protected void maintain() { + + } + + boolean retireUnallocated() { + try (Mutex lock = nodeRepository().lockUnallocated()) { + List<Node> allNodes = nodeRepository().getNodes(); + Map<Flavor, Long> numSpareNodesByFlavor = getNumberSpareReadyNodesByFlavor(allNodes); + + long numFlavorsWithUnsuccessfullyRetiredNodes = allNodes.stream() + .filter(node -> node.state() == Node.State.ready) + .filter(retirementPolicy::shouldRetire) + .collect(Collectors.groupingBy( + Node::flavor, + Collectors.toSet())) + .entrySet().stream() + .filter(entry -> { + long numSpareReadyNodesForCurrentFlavor = numSpareNodesByFlavor.get(entry.getKey()); + entry.getValue().stream() + .limit(numSpareReadyNodesForCurrentFlavor) + .forEach(node -> nodeRepository().park(node.hostname(), Agent.system)); + + return numSpareReadyNodesForCurrentFlavor < entry.getValue().size(); + }).count(); + + return numFlavorsWithUnsuccessfullyRetiredNodes == 0; + } + } + + Map<Flavor, Long> getNumberSpareReadyNodesByFlavor(List<Node> allNodes) { + Map<Flavor, Long> numActiveNodesByFlavor = allNodes.stream() + .filter(node -> node.state() == Node.State.active) + .collect(Collectors.groupingBy(Node::flavor, Collectors.counting())); + + return allNodes.stream() + .filter(node -> node.state() == Node.State.ready) + .collect(Collectors.groupingBy(Node::flavor, Collectors.counting())) + .entrySet().stream() + .collect(Collectors.toMap( + Map.Entry::getKey, + entry -> { + long numActiveNodesByCurrentFlavor = numActiveNodesByFlavor.getOrDefault(entry.getKey(), 0L); + long numNodesToToSpare = (long) Math.max(2, 0.1 * numActiveNodesByCurrentFlavor); + return Math.max(0L, entry.getValue() - numNodesToToSpare); + })); + } + + @Override + public String toString() { + return "Node retirer"; + } +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/retire/RetireIPv4OnlyNodes.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/retire/RetireIPv4OnlyNodes.java new file mode 100644 index 00000000000..b57965db70b --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/retire/RetireIPv4OnlyNodes.java @@ -0,0 +1,19 @@ +package com.yahoo.vespa.hosted.provision.maintenance.retire; + +import com.google.common.net.InetAddresses; +import com.yahoo.vespa.hosted.provision.Node; + +import java.net.Inet4Address; + +/** + * @author freva + */ +public class RetireIPv4OnlyNodes implements RetirementPolicy { + + @Override + public boolean shouldRetire(Node node) { + return node.ipAddresses().stream() + .map(InetAddresses::forString) + .allMatch(address -> address instanceof Inet4Address); + } +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/retire/RetirementPolicy.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/retire/RetirementPolicy.java new file mode 100644 index 00000000000..c68b76c2167 --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/retire/RetirementPolicy.java @@ -0,0 +1,10 @@ +package com.yahoo.vespa.hosted.provision.maintenance.retire; + +import com.yahoo.vespa.hosted.provision.Node; + +/** + * @author freva + */ +public interface RetirementPolicy { + boolean shouldRetire(Node node); +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRetirerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRetirerTest.java new file mode 100644 index 00000000000..daadd184ee8 --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRetirerTest.java @@ -0,0 +1,186 @@ +package com.yahoo.vespa.hosted.provision.maintenance; + +import com.yahoo.component.Version; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ClusterMembership; +import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.Flavor; +import com.yahoo.config.provision.NodeType; +import com.yahoo.vespa.hosted.provision.Node; +import com.yahoo.vespa.hosted.provision.NodeRepository; +import com.yahoo.vespa.hosted.provision.maintenance.retire.RetirementPolicy; +import com.yahoo.vespa.hosted.provision.node.Agent; +import com.yahoo.vespa.hosted.provision.node.Allocation; +import com.yahoo.vespa.hosted.provision.node.Generation; +import com.yahoo.vespa.hosted.provision.node.History; +import com.yahoo.vespa.hosted.provision.node.Status; +import com.yahoo.vespa.hosted.provision.testutils.FlavorConfigBuilder; +import org.junit.Before; +import org.junit.Test; + +import java.time.Duration; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.UUID; +import java.util.stream.Collectors; + +import static org.junit.Assert.assertEquals; +import static org.mockito.Matchers.anyString; +import static org.mockito.Matchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * @author freva + */ +public class NodeRetirerTest { + private final List<Flavor> flavors = makeFlavors(5); + private final Map<String, Node> nodesByHostname = new HashMap<>(); + private final NodeRepository nodeRepository = mock(NodeRepository.class); + + @Test + public void testRetireUnallocatedNodes() { + addNodesByFlavor(Node.State.ready, 5, 3, 77, 47); + deployApp("vespa", "calendar", 0, 3); + deployApp("vespa", "notes", 2, 12); + deployApp("sports", "results", 2, 7); + deployApp("search", "images", 3, 6); + + RetirementPolicy policy = node -> node.ipAddresses().equals(Collections.singleton("::1")); + NodeRetirer retirer = new NodeRetirer(nodeRepository, Duration.ofDays(1), policy); + Map<Flavor, Long> expected = expectedCountsByFlavor(0, 1, 56, 39); + Map<Flavor, Long> actual = retirer.getNumberSpareReadyNodesByFlavor(nodeRepository.getNodes()); + assertEquals(expected, actual); + + retirer.retireUnallocated(); + Map<Flavor, Long> parkedCountsByFlavor = nodesByHostname.values().stream() + .filter(node -> node.state() == Node.State.parked) + .collect(Collectors.groupingBy(Node::flavor, Collectors.counting())); + expected.remove(flavors.get(0)); // Flavor-0 has 0 ready nodes, so just remove it to easily compare maps + assertEquals(expected, parkedCountsByFlavor); + + expected = expectedCountsByFlavor(0, 0, 0, 0); + actual = retirer.getNumberSpareReadyNodesByFlavor(nodeRepository.getNodes()); + assertEquals(expected, actual); + + // Lets remove all the currently parked nodes and reprovision them different IP address + nodesByHostname.entrySet().stream().filter(entry -> entry.getValue().state() == Node.State.parked).forEach(entry -> + updateNode(entry.getKey(), entry.getValue().flavor(), Node.State.ready, Optional.empty(), Collections.singleton("::2"))); + + expected = expectedCountsByFlavor(0, 1, 56, 39); + actual = retirer.getNumberSpareReadyNodesByFlavor(nodeRepository.getNodes()); + assertEquals(expected, actual); + + retirer.retireUnallocated(); + parkedCountsByFlavor = nodesByHostname.values().stream() + .filter(node -> node.state() == Node.State.parked) + .collect(Collectors.groupingBy(Node::flavor, Collectors.counting())); + expected = expectedCountsByFlavor(0, 1, 2, 2); + expected.remove(flavors.get(0)); // Flavor-0 has 0 ready nodes, so just remove it to easily compare maps + assertEquals(expected, parkedCountsByFlavor); + } + + @Test + public void testGetNumberSpareNodesWithNoActiveNodes() { + addNodesByFlavor(Node.State.ready, 5, 3, 77); + + NodeRetirer retirer = new NodeRetirer(nodeRepository, Duration.ofDays(1), node -> false); + Map<Flavor, Long> expected = expectedCountsByFlavor(3, 1, 75); + Map<Flavor, Long> actual = retirer.getNumberSpareReadyNodesByFlavor(nodeRepository.getNodes()); + assertEquals(expected, actual); + } + + @Test + public void testGetNumberSpareNodesWithActiveNodes() { + addNodesByFlavor(Node.State.ready, 5, 3, 77, 47); + addNodesByFlavor(Node.State.active, 0, 10, 2, 230, 137); + + NodeRetirer retirer = new NodeRetirer(nodeRepository, Duration.ofDays(1), node -> false); + Map<Flavor, Long> expected = expectedCountsByFlavor(3, 1, 75, 24); + Map<Flavor, Long> actual = retirer.getNumberSpareReadyNodesByFlavor(nodeRepository.getNodes()); + assertEquals(expected, actual); + } + + + @Before + public void setup() { + when(nodeRepository.getNodes()).thenAnswer(invoc -> new ArrayList<>(nodesByHostname.values())); + when(nodeRepository.lockUnallocated()).thenReturn(null); + when(nodeRepository.park(anyString(), eq(Agent.system))).then(invocation -> { + Object[] args = invocation.getArguments(); + String hostname = (String) args[0]; + Node nodeToPark = Optional.ofNullable(nodesByHostname.get(hostname)) + .orElseThrow(() -> new RuntimeException("Could not find node with hostname " + hostname)); + if (nodeToPark.state() == Node.State.parked) throw new RuntimeException("Node already parked!"); + + updateNode(nodeToPark.hostname(), nodeToPark.flavor(), Node.State.parked, nodeToPark.allocation(), nodeToPark.ipAddresses()); + return null; + }); + } + + private Map<Flavor, Long> expectedCountsByFlavor(int... nums) { + Map<Flavor, Long> countsByFlavor = new HashMap<>(); + for (int i = 0; i < nums.length; i++) { + Flavor flavor = flavors.get(i); + countsByFlavor.put(flavor, (long) nums[i]); + } + return countsByFlavor; + } + + private void addNodesByFlavor(Node.State state, int... nums) { + for (int i = 0; i < nums.length; i++) { + Flavor flavor = flavors.get(i); + for (int j = 0; j < nums[i]; j++) { + int id = nodesByHostname.size(); + updateNode("host-" + id + ".yahoo.com", flavor, state, Optional.empty(), Collections.singleton("::1")); + } + } + } + + private void deployApp(String tenantName, String applicationName, int flavorId, int numNodes) { + Flavor flavor = flavors.get(flavorId); + List<Node> freeNodes = nodeRepository.getNodes().stream() + .filter(node -> node.state() == Node.State.ready) + .filter(node -> node.flavor() == flavor) + .collect(Collectors.toList()); + + if (freeNodes.size() < numNodes) throw new IllegalArgumentException( + "Not enough nodes to deploy " + applicationName + " on " + flavor + " needed " + numNodes + " but has " + freeNodes.size()); + + ApplicationId applicationId = ApplicationId.from(tenantName, applicationName, "default"); + ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("test"), Version.fromString("6.99")); + Allocation allocation = new Allocation(applicationId, ClusterMembership.from(cluster, 0), new Generation(0, 0), false); + freeNodes.stream().limit(numNodes).forEach(node -> { + updateNode(node.hostname(), node.flavor(), Node.State.active, Optional.of(allocation), node.ipAddresses()); + }); + } + + private List<Flavor> makeFlavors(int numFlavors) { + FlavorConfigBuilder flavorConfigBuilder = new FlavorConfigBuilder(); + for (int i = 0; i < numFlavors; i++) { + flavorConfigBuilder.addFlavor("flavor-" + i, 1. /* cpu*/, 3. /* mem GB*/, 2. /*disk GB*/, Flavor.Type.BARE_METAL); + } + return flavorConfigBuilder.build().flavor().stream().map(Flavor::new).collect(Collectors.toList()); + } + + private void updateNode(String hostname, Flavor flavor, Node.State state, Optional<Allocation> allocation, Set<String> ipAddresses) { + Node node = new Node( + UUID.randomUUID().toString(), + ipAddresses, + hostname, + Optional.empty(), + flavor, + Status.initial(), + state, + allocation, + History.empty(), + NodeType.tenant); + + nodesByHostname.put(hostname, node); + } +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/retire/RetireIPv4OnlyNodesTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/retire/RetireIPv4OnlyNodesTest.java new file mode 100644 index 00000000000..e1e6fdf4e3e --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/retire/RetireIPv4OnlyNodesTest.java @@ -0,0 +1,57 @@ +package com.yahoo.vespa.hosted.provision.maintenance.retire; + +import com.yahoo.config.provision.NodeType; +import com.yahoo.vespa.hosted.provision.Node; +import com.yahoo.vespa.hosted.provision.maintenance.NodeFailTester; +import org.junit.Test; + +import java.util.Arrays; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +/** + * @author freva + */ +public class RetireIPv4OnlyNodesTest { + private final RetireIPv4OnlyNodes policy = new RetireIPv4OnlyNodes(); + + @Test + public void testSingleIPv4Address() { + Node node = createNodeWithAddresses("127.0.0.1"); + assertTrue(policy.shouldRetire(node)); + } + + @Test + public void testSingleIPv6Address() { + Node node = createNodeWithAddresses("::1"); + assertFalse(policy.shouldRetire(node)); + } + + @Test + public void testMultipleIPv4Address() { + Node node = createNodeWithAddresses("127.0.0.1", "10.0.0.1", "192.168.0.1"); + assertTrue(policy.shouldRetire(node)); + } + + @Test + public void testMultipleIPv6Address() { + Node node = createNodeWithAddresses("::1", "::2", "1234:5678:90ab::cdef"); + assertFalse(policy.shouldRetire(node)); + } + + @Test + public void testCombinationAddress() { + Node node = createNodeWithAddresses("127.0.0.1", "::1", "10.0.0.1", "::2"); + assertFalse(policy.shouldRetire(node)); + } + + private Node createNodeWithAddresses(String... addresses) { + Set<String> ipAddresses = Arrays.stream(addresses).collect(Collectors.toSet()); + return Node.create("openstackid", ipAddresses, "hostname", Optional.empty(), + NodeFailTester.nodeFlavors.getFlavorOrThrow("default"), NodeType.tenant); + } +} |