summaryrefslogtreecommitdiffstats
path: root/container-disc/src/main/java/com/yahoo/container/jdisc/DataplaneProxyService.java
diff options
context:
space:
mode:
Diffstat (limited to 'container-disc/src/main/java/com/yahoo/container/jdisc/DataplaneProxyService.java')
-rw-r--r--container-disc/src/main/java/com/yahoo/container/jdisc/DataplaneProxyService.java265
1 files changed, 190 insertions, 75 deletions
diff --git a/container-disc/src/main/java/com/yahoo/container/jdisc/DataplaneProxyService.java b/container-disc/src/main/java/com/yahoo/container/jdisc/DataplaneProxyService.java
index 6d871b7283f..e6af65c0bc8 100644
--- a/container-disc/src/main/java/com/yahoo/container/jdisc/DataplaneProxyService.java
+++ b/container-disc/src/main/java/com/yahoo/container/jdisc/DataplaneProxyService.java
@@ -7,11 +7,14 @@ import com.yahoo.jdisc.http.server.jetty.DataplaneProxyCredentials;
import javax.inject.Inject;
import java.io.IOException;
-import java.net.URI;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.StandardCopyOption;
+import java.util.concurrent.ScheduledThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+import java.util.logging.Level;
+import java.util.logging.Logger;
/**
* Configures a data plane proxy. Currently using Nginx.
@@ -20,100 +23,137 @@ import java.nio.file.StandardCopyOption;
*/
public class DataplaneProxyService extends AbstractComponent {
+ private static Logger logger = Logger.getLogger(DataplaneProxyService.class.getName());
private static final String PREFIX = "/opt/vespa";
- private static final Path CONFIG_TEMPLATE = Paths.get(PREFIX, "conf/nginx/nginx.conf.template");
- private static final Path clientCertificateFile = Paths.get(PREFIX, "conf/nginx/client_cert.pem");
- private static final Path clientKeyFile = Paths.get(PREFIX, "conf/nginx/client_key.pem");
- private static final Path serverCertificateFile = Paths.get(PREFIX, "conf/nginx/server_cert.pem");
- private static final Path serverKeyFile = Paths.get(PREFIX, "conf/nginx/server_key.pem");
+ private final Path configTemplate;
+ private final Path serverCertificateFile;
+ private final Path serverKeyFile;
+ private final Path nginxConf;
- private static final Path nginxConf = Paths.get(PREFIX, "conf/nginx/nginx.conf");
+ private final ProxyCommands proxyCommands;
+ private final ScheduledThreadPoolExecutor executorService;
+ private final Path root;
+
+ enum NginxState {INITIALIZING, RUNNING, RELOAD_REQUIRED, STOPPED};
+ private NginxState state;
+ private NginxState wantedState;
+
+ private DataplaneProxyConfig cfg;
+ private Path proxyCredentialsCert;
+ private Path proxyCredentialsKey;
- private boolean started;
@Inject
public DataplaneProxyService() {
- this.started = false;
+ this(Paths.get(PREFIX), new NginxProxyCommands(), 1);
+ }
+
+ DataplaneProxyService(Path root, ProxyCommands proxyCommands, int reloadPeriodMinutes) {
+ this.root = root;
+ this.proxyCommands = proxyCommands;
+ changeState(NginxState.INITIALIZING);
+ wantedState = NginxState.RUNNING;
+ configTemplate = root.resolve("conf/nginx/nginx.conf.template");
+ serverCertificateFile = root.resolve("conf/nginx/server_cert.pem");
+ serverKeyFile = root.resolve("conf/nginx/server_key.pem");
+ nginxConf = root.resolve("conf/nginx/nginx.conf");
+
+ executorService = new ScheduledThreadPoolExecutor(1);
+ executorService.scheduleAtFixedRate(this::converge, reloadPeriodMinutes, reloadPeriodMinutes, TimeUnit.MINUTES);
}
public void reconfigure(DataplaneProxyConfig config, DataplaneProxyCredentials credentialsProvider) {
- try {
- String serverCert = config.serverCertificate();
- String serverKey = config.serverKey();
-
- boolean configChanged = false;
- configChanged |= writeFile(serverCertificateFile, serverCert);
- configChanged |= writeFile(serverKeyFile, serverKey);
- configChanged |= writeFile(nginxConf,
- nginxConfig(
- credentialsProvider.certificateFile(),
- credentialsProvider.keyFile(),
- serverCertificateFile,
- serverKeyFile,
- config.port(),
- PREFIX
- ));
- if (!started) {
- startNginx();
- started = true;
- } else if (configChanged){
- reloadNginx();
- }
- } catch (IOException e) {
- throw new RuntimeException("Error reconfiguring data plane proxy", e);
+ synchronized (this) {
+ this.cfg = config;
+ this.proxyCredentialsCert = credentialsProvider.certificateFile();
+ this.proxyCredentialsKey = credentialsProvider.keyFile();
}
}
- private void startNginx() {
- try {
- Process startCommand = new ProcessBuilder().command(
- "nginx",
- "-c", nginxConf.toString()
- ).start();
- int exitCode = startCommand.waitFor();
- if (exitCode != 0) {
- throw new RuntimeException("Non-zero exitcode from nginx: %d".formatted(exitCode));
- }
- } catch (IOException | InterruptedException e) {
- throw new RuntimeException("Could not start nginx", e);
- }
+ private void changeState(NginxState newState) {
+ state = newState;
}
- private void reloadNginx() {
- try {
- Process reloadCommand = new ProcessBuilder().command(
- "nginx",
- "-s", "reload"
- ).start();
- int exitCode = reloadCommand.waitFor();
- if (exitCode != 0) {
- throw new RuntimeException("Non-zero exitcode from nginx: %d".formatted(exitCode));
- }
- } catch (IOException | InterruptedException e) {
- throw new RuntimeException("Could not start nginx", e);
+ void converge() {
+ DataplaneProxyConfig config;
+ Path proxyCredentialsCert;
+ Path proxyCredentialsKey;
+ synchronized (this) {
+ config = cfg;
+ proxyCredentialsCert = this.proxyCredentialsCert;
+ proxyCredentialsKey = this.proxyCredentialsKey;
+ this.cfg = null;
+ this.proxyCredentialsCert = null;
+ this.proxyCredentialsKey = null;
}
- }
+ if (config != null) {
+ try {
- private void stopNginx() {
- try {
- Process stopCommand = new ProcessBuilder().command(
- "nginx",
- "-s", "stop"
- ).start();
- int exitCode = stopCommand.waitFor();
- if (exitCode != 0) {
- throw new RuntimeException("Non-zero exitcode from nginx: %d".formatted(exitCode));
+ String serverCert = config.serverCertificate();
+ String serverKey = config.serverKey();
+
+ boolean configChanged = false;
+ configChanged |= writeFile(serverCertificateFile, serverCert);
+ configChanged |= writeFile(serverKeyFile, serverKey);
+ configChanged |= writeFile(nginxConf,
+ nginxConfig(
+ configTemplate,
+ proxyCredentialsCert,
+ proxyCredentialsKey,
+ serverCertificateFile,
+ serverKeyFile,
+ config.port(),
+ root
+ ));
+ if (configChanged && state == NginxState.RUNNING) {
+ changeState(NginxState.RELOAD_REQUIRED);
+ }
+ } catch (IOException e) {
+ throw new RuntimeException("Error reconfiguring data plane proxy", e);
+ }
+ }
+ if (wantedState == NginxState.RUNNING) {
+ boolean nginxRunning = proxyCommands.isRunning();
+ if (!nginxRunning) {
+ try {
+ proxyCommands.start(nginxConf);
+ changeState(wantedState);
+ } catch (Exception e) {
+ logger.log(Level.INFO, "Failed to start nginx, will retry");
+ }
+ } else if (nginxRunning && state == NginxState.RELOAD_REQUIRED) {
+ try {
+ proxyCommands.reload();
+ changeState(wantedState);
+ } catch (Exception e) {
+ logger.log(Level.INFO, "Failed to reconfigure nginx, will retry.");
+ }
+ }
+ } else if (wantedState == NginxState.STOPPED) {
+ if (proxyCommands.isRunning()) {
+ try {
+ proxyCommands.stop();
+ changeState(wantedState);
+ executorService.shutdownNow();
+ } catch (Exception e) {
+ logger.log(Level.INFO, "Failed to stop nginx, will retry");
+ }
}
- } catch (IOException | InterruptedException e) {
- throw new RuntimeException("Could not start nginx", e);
+ } else {
+ logger.warning("Unknown state " + wantedState);
}
}
@Override
public void deconstruct() {
super.deconstruct();
- stopNginx();
+ wantedState = NginxState.STOPPED;
+ try {
+ executorService.awaitTermination(5, TimeUnit.MINUTES);
+ } catch (InterruptedException e) {
+ logger.log(Level.WARNING, "Error shutting down proxy reload thread");
+ }
}
/*
@@ -121,7 +161,7 @@ public class DataplaneProxyService extends AbstractComponent {
* return true if file was changed, false if no changes
*/
private boolean writeFile(Path file, String contents) throws IOException {
- Path tempPath = Paths.get(file.toFile().getAbsolutePath() + ".new");
+ Path tempPath = file.getParent().resolve(file.getFileName().toString() + ".new");
Files.createDirectories(tempPath.getParent());
Files.writeString(tempPath, contents);
@@ -135,21 +175,22 @@ public class DataplaneProxyService extends AbstractComponent {
}
static String nginxConfig(
+ Path configTemplate,
Path clientCert,
Path clientKey,
Path serverCert,
Path serverKey,
int vespaPort,
- String prefix) {
+ Path root) {
try {
- String nginxTemplate = Files.readString(CONFIG_TEMPLATE);
+ String nginxTemplate = Files.readString(configTemplate);
nginxTemplate = replace(nginxTemplate, "client_cert", clientCert.toString());
nginxTemplate = replace(nginxTemplate, "client_key", clientKey.toString());
nginxTemplate = replace(nginxTemplate, "server_cert", serverCert.toString());
nginxTemplate = replace(nginxTemplate, "server_key", serverKey.toString());
nginxTemplate = replace(nginxTemplate, "vespa_port", Integer.toString(vespaPort));
- nginxTemplate = replace(nginxTemplate, "prefix", prefix);
+ nginxTemplate = replace(nginxTemplate, "prefix", root.toString());
// TODO: verify that all template vars have been expanded
return nginxTemplate;
@@ -161,4 +202,78 @@ public class DataplaneProxyService extends AbstractComponent {
private static String replace(String template, String key, String value) {
return template.replaceAll("\\$\\{%s\\}".formatted(key), value);
}
+
+ NginxState state() {
+ return state;
+ }
+
+ NginxState wantedState() {
+ return wantedState;
+ }
+
+ public interface ProxyCommands {
+ void start(Path configFile);
+ void stop();
+ void reload();
+ boolean isRunning();
+ }
+
+ public static class NginxProxyCommands implements ProxyCommands {
+
+ @Override
+ public void start(Path configFile) {
+ try {
+ Process startCommand = new ProcessBuilder().command(
+ "nginx",
+ "-c", configFile.toString()
+ ).start();
+ int exitCode = startCommand.waitFor();
+ if (exitCode != 0) {
+ throw new RuntimeException("Non-zero exitcode from nginx: %d".formatted(exitCode));
+ }
+ } catch (IOException | InterruptedException e) {
+ throw new RuntimeException("Could not start nginx", e);
+ }
+ }
+
+ @Override
+ public void stop() {
+ try {
+ Process stopCommand = new ProcessBuilder().command(
+ "nginx",
+ "-s", "stop"
+ ).start();
+ int exitCode = stopCommand.waitFor();
+ if (exitCode != 0) {
+ throw new RuntimeException("Non-zero exitcode from nginx: %d".formatted(exitCode));
+ }
+ } catch (IOException | InterruptedException e) {
+ throw new RuntimeException("Could not start nginx", e);
+ }
+
+ }
+
+ @Override
+ public void reload() {
+ try {
+ Process reloadCommand = new ProcessBuilder().command(
+ "nginx",
+ "-s", "reload"
+ ).start();
+ int exitCode = reloadCommand.waitFor();
+ if (exitCode != 0) {
+ throw new RuntimeException("Non-zero exitcode from nginx: %d".formatted(exitCode));
+ }
+ } catch (IOException | InterruptedException e) {
+ throw new RuntimeException("Could not start nginx", e);
+ }
+ }
+
+ @Override
+ public boolean isRunning() {
+ return ProcessHandle.allProcesses()
+ .map(ProcessHandle::info)
+ .anyMatch(info -> info.command().orElse("").endsWith("nginx"));
+ }
+ }
}