1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
|
// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.hosted.provision.os;
import com.google.common.cache.Cache;
import com.google.common.cache.CacheBuilder;
import com.yahoo.component.Version;
import com.yahoo.config.provision.CloudAccount;
import com.yahoo.config.provision.NodeType;
import com.yahoo.vespa.flags.IntFlag;
import com.yahoo.vespa.flags.PermanentFlags;
import com.yahoo.vespa.hosted.provision.Node;
import com.yahoo.vespa.hosted.provision.NodeList;
import com.yahoo.vespa.hosted.provision.NodeRepository;
import com.yahoo.vespa.hosted.provision.provisioning.HostProvisioner;
import com.yahoo.yolean.Exceptions;
import java.time.Duration;
import java.time.Instant;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
import java.util.logging.Level;
import java.util.logging.Logger;
/**
* Interface for an OS upgrader.
*
* @author mpolden
*/
public abstract class OsUpgrader {
private final Logger LOG = Logger.getLogger(OsUpgrader.class.getName());
private final IntFlag maxActiveUpgrades;
private final Optional<HostProvisioner> hostProvisioner;
// Supported versions is queried for each host to upgrade, so we cache the results for a while to avoid excessive
// API calls to the host provisioner
private final Cache<CloudAccount, Set<Version>> supportedVersions = CacheBuilder.newBuilder()
.expireAfterWrite(10, TimeUnit.MINUTES)
.build();
final NodeRepository nodeRepository;
public OsUpgrader(NodeRepository nodeRepository, Optional<HostProvisioner> hostProvisioner) {
this.nodeRepository = Objects.requireNonNull(nodeRepository);
this.maxActiveUpgrades = PermanentFlags.MAX_OS_UPGRADES.bindTo(nodeRepository.flagSource());
this.hostProvisioner = Objects.requireNonNull(hostProvisioner);
}
/** Trigger upgrade to given target */
abstract void upgradeTo(OsVersionTarget target);
/** Disable OS upgrade for all nodes of given type */
abstract void disableUpgrade(NodeType type);
/** Returns the number of upgrade slots available for given target */
final int upgradeSlots(OsVersionTarget target, NodeList candidates) {
if (!candidates.stream().allMatch(node -> node.type() == target.nodeType())) {
throw new IllegalArgumentException("All node types must type of OS version target " + target.nodeType());
}
int max = target.nodeType() == NodeType.host ? maxActiveUpgrades.value() : 1;
int upgrading = candidates.changingOsVersionTo(target.version()).size();
return Math.max(0, max - upgrading);
}
/** Returns whether node can upgrade to version at given instant */
final boolean canUpgradeTo(Version version, Instant instant, Node node) {
Set<Version> versions = supportedVersions(node, version);
boolean versionAvailable = versions.contains(version);
if (!versionAvailable) {
LOG.log(Level.WARNING, "Want to upgrade host " + node.hostname() + " to OS version " +
version.toFullString() + ", but this version does not exist in " +
node.cloudAccount() + ". Found " + versions.stream().sorted().toList());
}
return versionAvailable &&
(node.status().osVersion().downgrading() || // Fast-track downgrades
node.history().age(instant).compareTo(gracePeriod()) > 0);
}
private Set<Version> supportedVersions(Node host, Version requestedVersion) {
if (hostProvisioner.isEmpty()) {
return Set.of(requestedVersion);
}
try {
return supportedVersions.get(host.cloudAccount(),
() -> hostProvisioner.get().osVersions(host, requestedVersion.getMajor()));
} catch (ExecutionException e) {
LOG.log(Level.WARNING, "Failed to list supported OS versions in " + host.cloudAccount() + ": " + Exceptions.toMessageString(e));
return Set.of();
}
}
/** The duration this leaves new nodes alone before scheduling any upgrade */
private Duration gracePeriod() {
return nodeRepository.zone().system().isCd() ? Duration.ofHours(4) : Duration.ofDays(1);
}
}
|