1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
|
// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.model.content;
import com.yahoo.vespa.model.builder.xml.dom.ModelElement;
import com.yahoo.vespa.model.content.cluster.DomResourceLimitsBuilder;
import java.util.Optional;
import java.util.function.Consumer;
/**
* Class tracking the feed block resource limits for a content cluster.
*
* This includes the limits used by the cluster controller and the content nodes (proton).
*
* @author geirst
*/
public class ClusterResourceLimits {
private final ResourceLimits clusterControllerLimits;
private final ResourceLimits contentNodeLimits;
private ClusterResourceLimits(Builder builder) {
clusterControllerLimits = builder.ctrlBuilder.build();
contentNodeLimits = builder.nodeBuilder.build();
}
public ResourceLimits getClusterControllerLimits() {
return clusterControllerLimits;
}
public ResourceLimits getContentNodeLimits() {
return contentNodeLimits;
}
public static class Builder {
private final boolean hostedVespa;
private final double resourceLimitDisk;
private final double resourceLimitMemory;
private ResourceLimits.Builder ctrlBuilder = new ResourceLimits.Builder();
private ResourceLimits.Builder nodeBuilder = new ResourceLimits.Builder();
public Builder(boolean hostedVespa,
double resourceLimitDisk,
double resourceLimitMemory) {
this.hostedVespa = hostedVespa;
this.resourceLimitDisk = resourceLimitDisk;
this.resourceLimitMemory = resourceLimitMemory;
verifyLimits(resourceLimitDisk, resourceLimitMemory);
}
public ClusterResourceLimits build(ModelElement clusterElem) {
ctrlBuilder = createBuilder(clusterElem.childByPath("tuning"));
nodeBuilder = createBuilder(clusterElem.childByPath("engine.proton"));
deriveLimits();
return new ClusterResourceLimits(this);
}
private ResourceLimits.Builder createBuilder(ModelElement element) {
return element == null
? new ResourceLimits.Builder()
: DomResourceLimitsBuilder.createBuilder(element, hostedVespa);
}
public void setClusterControllerBuilder(ResourceLimits.Builder builder) {
ctrlBuilder = builder;
}
public void setContentNodeBuilder(ResourceLimits.Builder builder) {
nodeBuilder = builder;
}
public ClusterResourceLimits build() {
deriveLimits();
return new ClusterResourceLimits(this);
}
private void deriveLimits() {
// This also ensures that content nodes limits are derived according to the formula in calcContentNodeLimit().
considerSettingDefaultClusterControllerLimit(ctrlBuilder.getDiskLimit(),
nodeBuilder.getDiskLimit(),
ctrlBuilder::setDiskLimit,
resourceLimitDisk);
considerSettingDefaultClusterControllerLimit(ctrlBuilder.getMemoryLimit(),
nodeBuilder.getMemoryLimit(),
ctrlBuilder::setMemoryLimit,
resourceLimitMemory);
deriveClusterControllerLimit(ctrlBuilder.getDiskLimit(), nodeBuilder.getDiskLimit(), ctrlBuilder::setDiskLimit);
deriveClusterControllerLimit(ctrlBuilder.getMemoryLimit(), nodeBuilder.getMemoryLimit(), ctrlBuilder::setMemoryLimit);
deriveContentNodeLimit(nodeBuilder.getDiskLimit(), ctrlBuilder.getDiskLimit(), 0.6, nodeBuilder::setDiskLimit);
deriveContentNodeLimit(nodeBuilder.getMemoryLimit(), ctrlBuilder.getMemoryLimit(), 0.5, nodeBuilder::setMemoryLimit);
}
private void considerSettingDefaultClusterControllerLimit(Optional<Double> clusterControllerLimit,
Optional<Double> contentNodeLimit,
Consumer<Double> setter,
double resourceLimit) {
// TODO: remove this when feed block in distributor is default enabled.
if (clusterControllerLimit.isEmpty() && contentNodeLimit.isEmpty()) {
setter.accept(resourceLimit);
}
}
private void deriveClusterControllerLimit(Optional<Double> clusterControllerLimit,
Optional<Double> contentNodeLimit,
Consumer<Double> setter) {
if (clusterControllerLimit.isEmpty()) {
contentNodeLimit.ifPresent(limit ->
// TODO: emit warning when feed block in distributor is default enabled.
setter.accept(Double.max(0.0, (limit - 0.01))));
}
}
private void deriveContentNodeLimit(Optional<Double> contentNodeLimit,
Optional<Double> clusterControllerLimit,
double scaleFactor,
Consumer<Double> setter) {
if (contentNodeLimit.isEmpty()) {
clusterControllerLimit.ifPresent(limit ->
setter.accept(calcContentNodeLimit(limit, scaleFactor)));
}
}
private double calcContentNodeLimit(double clusterControllerLimit, double scaleFactor) {
return clusterControllerLimit + ((1.0 - clusterControllerLimit) * scaleFactor);
}
private void verifyLimits(double resourceLimitDisk, double resourceLimitMemory) {
verifyLimitInRange(resourceLimitDisk, "disk");
verifyLimitInRange(resourceLimitMemory, "memory");
}
private void verifyLimitInRange(double limit, String type) {
String message = "Resource limit for " + type + " is set to illegal value " + limit +
", but must be in the range [0.0, 1.0]";
if (limit < 0.0)
throw new IllegalArgumentException(message);
if (limit > 1.0)
throw new IllegalArgumentException(message);
}
}
}
|
