1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
|
// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.hosted.routing.nginx;
import com.yahoo.collections.Pair;
import com.yahoo.config.provision.zone.RoutingMethod;
import com.yahoo.jdisc.Metric;
import com.yahoo.system.ProcessExecuter;
import com.yahoo.vespa.hosted.routing.Router;
import com.yahoo.vespa.hosted.routing.RoutingTable;
import com.yahoo.vespa.hosted.routing.status.RoutingStatus;
import com.yahoo.yolean.Exceptions;
import com.yahoo.yolean.concurrent.Sleeper;
import java.io.IOException;
import java.io.UncheckedIOException;
import java.nio.file.FileSystem;
import java.nio.file.Files;
import java.nio.file.NoSuchFileException;
import java.nio.file.Path;
import java.nio.file.StandardCopyOption;
import java.time.Clock;
import java.time.Duration;
import java.time.Instant;
import java.time.LocalDateTime;
import java.time.ZoneOffset;
import java.util.Objects;
import java.util.Optional;
import java.util.logging.Level;
import java.util.logging.Logger;
/**
* This loads a {@link RoutingTable} into a running Nginx process.
*
* @author mpolden
*/
public class Nginx implements Router {
private static final Logger LOG = Logger.getLogger(Nginx.class.getName());
private static final int EXEC_ATTEMPTS = 5;
static final String GENERATED_UPSTREAMS_METRIC = "upstreams_generated";
static final String CONFIG_RELOADS_METRIC = "upstreams_nginx_reloads";
static final String OK_CONFIG_RELOADS_METRIC = "upstreams_nginx_reloads_succeeded";
private final FileSystem fileSystem;
private final ProcessExecuter processExecuter;
private final Sleeper sleeper;
private final Clock clock;
private final RoutingStatus routingStatus;
private final Metric metric;
private final Object monitor = new Object();
public Nginx(FileSystem fileSystem, ProcessExecuter processExecuter, Sleeper sleeper, Clock clock, RoutingStatus routingStatus, Metric metric) {
this.fileSystem = Objects.requireNonNull(fileSystem);
this.processExecuter = Objects.requireNonNull(processExecuter);
this.sleeper = Objects.requireNonNull(sleeper);
this.clock = Objects.requireNonNull(clock);
this.routingStatus = Objects.requireNonNull(routingStatus);
this.metric = Objects.requireNonNull(metric);
}
@Override
public void load(RoutingTable table) {
synchronized (monitor) {
try {
table = table.routingMethod(RoutingMethod.sharedLayer4); // This router only supports layer 4 endpoints
testConfig(table);
loadConfig(table.asMap().size());
gcConfig();
} catch (IOException e) {
throw new UncheckedIOException(e);
}
}
}
/** Write given routing table to a temporary config file and test it */
private void testConfig(RoutingTable table) throws IOException {
String config = NginxConfig.from(table, routingStatus);
Files.createDirectories(NginxPath.root.in(fileSystem));
atomicWriteString(NginxPath.temporaryConfig.in(fileSystem), config);
// This retries config testing because it can fail due to external factors, such as hostnames not resolving in
// DNS. Retrying can be removed if we switch to having only IP addresses in config
retryingExec("/usr/bin/sudo /opt/vespa/bin/vespa-verify-nginx");
}
/** Load tested config into Nginx */
private void loadConfig(int upstreamCount) throws IOException {
Path configPath = NginxPath.config.in(fileSystem);
Path tempConfigPath = NginxPath.temporaryConfig.in(fileSystem);
try {
String currentConfig = Files.readString(configPath);
String newConfig = Files.readString(tempConfigPath);
if (currentConfig.equals(newConfig)) {
Files.deleteIfExists(tempConfigPath);
return;
}
Path rotatedConfig = NginxPath.config.rotatedIn(fileSystem, clock.instant());
atomicCopy(configPath, rotatedConfig);
} catch (NoSuchFileException ignored) {
// Fine, not enough files exist to compare or rotate
}
Files.move(tempConfigPath, configPath, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING);
metric.add(CONFIG_RELOADS_METRIC, 1, null);
// Retry reload. Same rationale for retrying as in testConfig()
LOG.info("Loading new configuration file from " + configPath);
retryingExec("/usr/bin/sudo /opt/vespa/bin/vespa-reload-nginx");
metric.add(OK_CONFIG_RELOADS_METRIC, 1, null);
metric.set(GENERATED_UPSTREAMS_METRIC, upstreamCount, null);
}
/** Remove old config files */
private void gcConfig() throws IOException {
Instant oneWeekAgo = clock.instant().minus(Duration.ofDays(7));
// Rotated files have the format <basename>-yyyy-MM-dd-HH:mm:ss.SSS
String configBasename = NginxPath.config.in(fileSystem).getFileName().toString();
try (var entries = Files.list(NginxPath.root.in(fileSystem))) {
entries.filter(Files::isRegularFile)
.filter(path -> path.getFileName().toString().startsWith(configBasename))
.filter(path -> rotatedAt(path).map(instant -> instant.isBefore(oneWeekAgo))
.orElse(false))
.forEach(path -> Exceptions.uncheck(() -> Files.deleteIfExists(path)));
}
}
/** Returns the time given path was rotated */
private Optional<Instant> rotatedAt(Path path) {
String[] parts = path.getFileName().toString().split("-", 2);
if (parts.length != 2) return Optional.empty();
return Optional.of(LocalDateTime.from(NginxPath.ROTATED_SUFFIX_FORMAT.parse(parts[1])).toInstant(ZoneOffset.UTC));
}
/** Run given command. Retries after a delay on failure */
private void retryingExec(String command) {
boolean success = false;
for (int attempt = 1; attempt <= EXEC_ATTEMPTS; attempt++) {
String errorMessage;
try {
Pair<Integer, String> result = processExecuter.exec(command);
if (result.getFirst() == 0) {
success = true;
break;
}
errorMessage = result.getSecond();
} catch (IOException e) {
errorMessage = Exceptions.toMessageString(e);
}
Duration duration = Duration.ofSeconds((long) Math.pow(2, attempt));
LOG.log(Level.WARNING, "Failed to run " + command + " on attempt " + attempt + ": " + errorMessage +
". Retrying in " + duration);
sleeper.sleep(duration);
}
if (!success) {
throw new RuntimeException("Failed to run " + command + " successfully after " + EXEC_ATTEMPTS +
" attempts, giving up");
}
}
/** Apply pathOperation to a temporary file, then atomically move the temporary file to path */
private void atomicWrite(Path path, PathOperation pathOperation) throws IOException {
Path tempFile = null;
try {
tempFile = Files.createTempFile(path.getParent(), "nginx", "");
pathOperation.run(tempFile);
Files.move(tempFile, path, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING);
} finally {
if (tempFile != null) {
Files.deleteIfExists(tempFile);
}
}
}
private void atomicCopy(Path src, Path dst) throws IOException {
atomicWrite(dst, (tempFile) -> Files.copy(src, tempFile,
StandardCopyOption.REPLACE_EXISTING,
StandardCopyOption.COPY_ATTRIBUTES));
}
private void atomicWriteString(Path path, String content) throws IOException {
atomicWrite(path, (tempFile) -> Files.writeString(tempFile, content));
}
@FunctionalInterface
private interface PathOperation {
void run(Path path) throws IOException;
}
}
|
