1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
|
// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.hosted.provision.provisioning;
import com.yahoo.config.provision.ApplicationId;
import com.yahoo.config.provision.ClusterSpec;
import com.yahoo.config.provision.OutOfCapacityException;
import com.yahoo.lang.MutableInteger;
import com.yahoo.transaction.Mutex;
import com.yahoo.vespa.flags.BooleanFlag;
import com.yahoo.vespa.flags.FetchVector;
import com.yahoo.vespa.flags.FlagSource;
import com.yahoo.vespa.flags.Flags;
import com.yahoo.vespa.flags.ListFlag;
import com.yahoo.vespa.flags.custom.PreprovisionCapacity;
import com.yahoo.vespa.hosted.provision.LockedNodeList;
import com.yahoo.vespa.hosted.provision.Node;
import com.yahoo.vespa.hosted.provision.NodeRepository;
import com.yahoo.vespa.hosted.provision.node.Agent;
import java.util.List;
import java.util.Optional;
import java.util.stream.Collectors;
/**
* Performs preparation of node activation changes for a single host group in an application.
*
* @author bratseth
*/
public class GroupPreparer {
private final NodeRepository nodeRepository;
private final Optional<HostProvisioner> hostProvisioner;
private final HostResourcesCalculator hostResourcesCalculator;
private final BooleanFlag dynamicProvisioningEnabledFlag;
private final ListFlag<PreprovisionCapacity> preprovisionCapacityFlag;
public GroupPreparer(NodeRepository nodeRepository, Optional<HostProvisioner> hostProvisioner,
HostResourcesCalculator hostResourcesCalculator, FlagSource flagSource) {
this.nodeRepository = nodeRepository;
this.hostProvisioner = hostProvisioner;
this.hostResourcesCalculator = hostResourcesCalculator;
this.dynamicProvisioningEnabledFlag = Flags.ENABLE_DYNAMIC_PROVISIONING.bindTo(flagSource);
this.preprovisionCapacityFlag = Flags.PREPROVISION_CAPACITY.bindTo(flagSource);
}
/**
* Ensure sufficient nodes are reserved or active for the given application, group and cluster
*
* @param application the application we are allocating to
* @param cluster the cluster and group we are allocating to
* @param requestedNodes a specification of the requested nodes
* @param surplusActiveNodes currently active nodes which are available to be assigned to this group.
* This method will remove from this list if it finds it needs additional nodes
* @param highestIndex the current highest node index among all active nodes in this cluster.
* This method will increase this number when it allocates new nodes to the cluster.
* @param spareCount The number of spare docker hosts we want when dynamically allocate docker containers
* @return the list of nodes this cluster group will have allocated if activated
*/
// Note: This operation may make persisted changes to the set of reserved and inactive nodes,
// but it may not change the set of active nodes, as the active nodes must stay in sync with the
// active config model which is changed on activate
public List<Node> prepare(ApplicationId application, ClusterSpec cluster, NodeSpec requestedNodes,
List<Node> surplusActiveNodes, MutableInteger highestIndex, int spareCount, int wantedGroups) {
boolean dynamicProvisioningEnabled = hostProvisioner.isPresent() && dynamicProvisioningEnabledFlag
.with(FetchVector.Dimension.APPLICATION_ID, application.serializedForm())
.value();
boolean allocateFully = dynamicProvisioningEnabled && preprovisionCapacityFlag.value().isEmpty();
try (Mutex lock = nodeRepository.lock(application)) {
// Lock ready pool to ensure that the same nodes are not simultaneously allocated by others
try (Mutex allocationLock = nodeRepository.lockUnallocated()) {
// Create a prioritized set of nodes
LockedNodeList nodeList = nodeRepository.list(allocationLock);
NodePrioritizer prioritizer = new NodePrioritizer(nodeList, application, cluster, requestedNodes,
spareCount, wantedGroups, nodeRepository.nameResolver(),
hostResourcesCalculator, allocateFully);
prioritizer.addApplicationNodes();
prioritizer.addSurplusNodes(surplusActiveNodes);
prioritizer.addReadyNodes();
prioritizer.addNewDockerNodes(nodeRepository::canAllocateTenantNodeTo);
// Allocate from the prioritized list
NodeAllocation allocation = new NodeAllocation(nodeList, application, cluster, requestedNodes,
highestIndex, nodeRepository.getAvailableFlavors(),
nodeRepository.zone(), nodeRepository.clock());
allocation.offer(prioritizer.prioritize());
if (dynamicProvisioningEnabled) {
List<ProvisionedHost> provisionedHosts = allocation.getFulfilledDockerDeficit()
.map(deficit -> hostProvisioner.get().provisionHosts(nodeRepository.database().getProvisionIndexes(deficit.getCount()),
deficit.getFlavor(),
application))
.orElseGet(List::of);
// At this point we have started provisioning of the hosts, the first priority is to make sure that
// the returned hosts are added to the node-repo so that they are tracked by the provision maintainers
List<Node> hosts = provisionedHosts.stream()
.map(ProvisionedHost::generateHost)
.collect(Collectors.toList());
nodeRepository.addNodes(hosts, Agent.application);
// Offer the nodes on the newly provisioned hosts, this should be enough to cover the deficit
List<PrioritizableNode> nodes = provisionedHosts.stream()
.map(provisionedHost -> new PrioritizableNode.Builder(provisionedHost.generateNode())
.parent(provisionedHost.generateHost())
.newNode(true)
.build())
.collect(Collectors.toList());
allocation.offer(nodes);
}
if (! allocation.fulfilled() && requestedNodes.canFail())
throw new OutOfCapacityException("Could not satisfy " + requestedNodes + " for " + cluster +
" in " + application.toShortString() +
outOfCapacityDetails(allocation));
// Carry out and return allocation
nodeRepository.reserve(allocation.reservableNodes());
nodeRepository.addDockerNodes(new LockedNodeList(allocation.newNodes(), allocationLock));
surplusActiveNodes.removeAll(allocation.surplusNodes());
return allocation.finalNodes(surplusActiveNodes);
}
}
}
private static String outOfCapacityDetails(NodeAllocation allocation) {
if (allocation.wouldBeFulfilledWithoutExclusivity())
return ": Not enough nodes available due to host exclusivity constraints.";
else if (allocation.wouldBeFulfilledWithClashingParentHost())
return ": Not enough nodes available on separate physical hosts.";
else if (allocation.wouldBeFulfilledWithRetiredNodes())
return ": Not enough nodes available due to retirement.";
else
return ".";
}
}
|
