diff options
20 files changed, 518 insertions, 178 deletions
diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetController.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetController.java index 8027cec4e3c..a5fa0d08bb8 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetController.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetController.java @@ -2,7 +2,6 @@ package com.yahoo.vespa.clustercontroller.core; import com.yahoo.document.FixedBucketSpaces; -import com.yahoo.exception.ExceptionUtils; import com.yahoo.vdslib.distribution.ConfiguredNode; import com.yahoo.vdslib.state.ClusterState; import com.yahoo.vdslib.state.Node; @@ -22,10 +21,9 @@ import com.yahoo.vespa.clustercontroller.core.status.LegacyIndexPageRequestHandl import com.yahoo.vespa.clustercontroller.core.status.LegacyNodePageRequestHandler; import com.yahoo.vespa.clustercontroller.core.status.NodeHealthRequestHandler; import com.yahoo.vespa.clustercontroller.core.status.StatusHandler; -import com.yahoo.vespa.clustercontroller.core.status.statuspage.StatusPageResponse; import com.yahoo.vespa.clustercontroller.core.status.statuspage.StatusPageServer; import com.yahoo.vespa.clustercontroller.utils.util.MetricReporter; -import java.io.FileNotFoundException; + import java.time.Duration; import java.time.Instant; import java.util.ArrayDeque; @@ -37,7 +35,6 @@ import java.util.List; import java.util.Map; import java.util.Queue; import java.util.Set; -import java.util.TimeZone; import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.atomic.AtomicBoolean; import java.util.logging.Level; @@ -508,43 +505,6 @@ public class FleetController implements NodeListener, SlobrokListener, SystemSta } } - public StatusPageResponse fetchStatusPage(StatusPageServer.HttpRequest httpRequest) { - verifyInControllerThread(); - StatusPageResponse.ResponseCode responseCode; - String message; - final String hiddenMessage; - try { - StatusPageServer.RequestHandler handler = statusRequestRouter.resolveHandler(httpRequest); - if (handler == null) { - throw new FileNotFoundException("No handler found for request: " + httpRequest.getPath()); - } - return handler.handle(httpRequest); - } catch (FileNotFoundException e) { - responseCode = StatusPageResponse.ResponseCode.NOT_FOUND; - message = e.getMessage(); - hiddenMessage = ""; - } catch (Exception e) { - responseCode = StatusPageResponse.ResponseCode.INTERNAL_SERVER_ERROR; - message = "Internal Server Error"; - hiddenMessage = ExceptionUtils.getStackTraceAsString(e); - context.log(logger, Level.FINE, () -> "Unknown exception thrown for request " + httpRequest.getRequest() + ": " + hiddenMessage); - } - - TimeZone tz = TimeZone.getTimeZone("UTC"); - long currentTime = timer.getCurrentTimeInMillis(); - StatusPageResponse response = new StatusPageResponse(); - StringBuilder content = new StringBuilder(); - response.setContentType("text/html"); - response.setResponseCode(responseCode); - content.append("<!-- Answer to request ").append(httpRequest.getRequest()).append(" -->\n"); - content.append("<p>UTC time when creating this page: ").append(RealTimer.printDateNoMilliSeconds(currentTime, tz)).append("</p>"); - response.writeHtmlHeader(content, message); - response.writeHtmlFooter(content, hiddenMessage); - response.writeContent(content.toString()); - - return response; - } - public void tick() throws Exception { synchronized (monitor) { boolean didWork; @@ -635,7 +595,8 @@ public class FleetController implements NodeListener, SlobrokListener, SystemSta private boolean processAnyPendingStatusPageRequest() { StatusPageServer.HttpRequest statusRequest = statusPageServer.getCurrentHttpRequest(); if (statusRequest != null) { - statusPageServer.answerCurrentStatusRequest(fetchStatusPage(statusRequest)); + verifyInControllerThread(); + statusPageServer.fetchStatusPage(statusRequest, statusRequestRouter, timer); return true; } return false; diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/restapiv2/MissingIdException.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/restapiv2/MissingIdException.java index 21229b4b358..18a3b923908 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/restapiv2/MissingIdException.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/restapiv2/MissingIdException.java @@ -4,14 +4,12 @@ package com.yahoo.vespa.clustercontroller.core.restapiv2; import com.yahoo.vdslib.state.Node; import com.yahoo.vespa.clustercontroller.utils.staterestapi.errors.MissingUnitException; +import java.util.List; + public class MissingIdException extends MissingUnitException { - private static String[] createPath(String cluster, Node n) { - String[] path = new String[3]; - path[0] = cluster; - path[1] = n.getType().toString(); - path[2] = String.valueOf(n.getIndex()); - return path; + private static List<String> createPath(String cluster, Node n) { + return List.of(cluster, n.getType().toString(), String.valueOf(n.getIndex())); } public MissingIdException(String cluster, Node n) { diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/restapiv2/UnitPathResolver.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/restapiv2/UnitPathResolver.java index 11fd5f39fad..0db11cad955 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/restapiv2/UnitPathResolver.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/restapiv2/UnitPathResolver.java @@ -7,6 +7,7 @@ import com.yahoo.vespa.clustercontroller.utils.staterestapi.errors.MissingUnitEx import com.yahoo.vespa.clustercontroller.utils.staterestapi.errors.OperationNotSupportedForUnitException; import com.yahoo.vespa.clustercontroller.utils.staterestapi.errors.StateRestApiException; import java.util.HashMap; +import java.util.List; import java.util.Map; public class UnitPathResolver<T> { @@ -22,10 +23,10 @@ public class UnitPathResolver<T> { public static abstract class AbstractVisitor<T> implements Visitor<T> { - private final String[] path; + private final List<String> path; private final String failureMessage; - public AbstractVisitor(String[] path, String failureMessage) { + public AbstractVisitor(List<String> path, String failureMessage) { this.path = path; this.failureMessage = failureMessage; } @@ -46,41 +47,41 @@ public class UnitPathResolver<T> { this.fleetControllers = new HashMap<>(fleetControllers); } - public RemoteClusterControllerTaskScheduler resolveFleetController(String[] path) throws StateRestApiException { - if (path.length == 0) return null; - RemoteClusterControllerTaskScheduler fc = fleetControllers.get(path[0]); + public RemoteClusterControllerTaskScheduler resolveFleetController(List<String> path) throws StateRestApiException { + if (path.size() == 0) return null; + RemoteClusterControllerTaskScheduler fc = fleetControllers.get(path.get(0)); if (fc == null) { throw new MissingUnitException(path, 0); } return fc; } - public Request<? extends T> visit(String[] path, Visitor<T> visitor) throws StateRestApiException { - if (path.length == 0) { + public Request<? extends T> visit(List<String> path, Visitor<T> visitor) throws StateRestApiException { + if (path.size() == 0) { return visitor.visitGlobal(); } - RemoteClusterControllerTaskScheduler fc = fleetControllers.get(path[0]); + RemoteClusterControllerTaskScheduler fc = fleetControllers.get(path.get(0)); if (fc == null) throw new MissingUnitException(path, 0); - Id.Cluster cluster = new Id.Cluster(path[0]); - if (path.length == 1) { + Id.Cluster cluster = new Id.Cluster(path.get(0)); + if (path.size() == 1) { return visitor.visitCluster(cluster); } Id.Service service; try{ - service = new Id.Service(cluster, NodeType.get(path[1])); + service = new Id.Service(cluster, NodeType.get(path.get(1))); } catch (IllegalArgumentException e) { throw new MissingUnitException(path, 1); } - if (path.length == 2) { + if (path.size() == 2) { return visitor.visitService(service); } Id.Node node; try{ - node = new Id.Node(service, Integer.valueOf(path[2])); + node = new Id.Node(service, Integer.parseInt(path.get(2))); } catch (NumberFormatException e) { throw new MissingUnitException(path, 2); } - if (path.length == 3) { + if (path.size() == 3) { return visitor.visitNode(node); } throw new MissingUnitException(path, 4); diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/status/StatusHandler.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/status/StatusHandler.java index 65b06afb0c5..6ed121284b6 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/status/StatusHandler.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/status/StatusHandler.java @@ -1,14 +1,20 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.clustercontroller.core.status; +import com.yahoo.exception.ExceptionUtils; +import com.yahoo.vespa.clustercontroller.core.RealTimer; +import com.yahoo.vespa.clustercontroller.core.Timer; import com.yahoo.vespa.clustercontroller.core.status.statuspage.StatusPageResponse; import com.yahoo.vespa.clustercontroller.core.status.statuspage.StatusPageServer; import com.yahoo.vespa.clustercontroller.utils.communication.http.HttpRequest; import com.yahoo.vespa.clustercontroller.utils.communication.http.HttpRequestHandler; import com.yahoo.vespa.clustercontroller.utils.communication.http.HttpResult; + +import java.io.FileNotFoundException; import java.io.StringWriter; import java.nio.charset.StandardCharsets; import java.util.Map; +import java.util.TimeZone; import java.util.logging.Logger; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -42,6 +48,46 @@ public class StatusHandler implements HttpRequestHandler { return r; } } + + public void fetchStatusPage(StatusPageServer.HttpRequest httpRequest, + StatusPageServer.PatternRequestRouter statusRequestRouter, + Timer timer) { + StatusPageResponse.ResponseCode responseCode; + String message; + final String hiddenMessage; + try { + StatusPageServer.RequestHandler handler = statusRequestRouter.resolveHandler(httpRequest); + if (handler == null) { + throw new FileNotFoundException("No handler found for request: " + httpRequest.getPath()); + } + answerCurrentStatusRequest(handler.handle(httpRequest)); + return; + } catch (FileNotFoundException e) { + responseCode = StatusPageResponse.ResponseCode.NOT_FOUND; + message = e.getMessage(); + hiddenMessage = ""; + } catch (Exception e) { + responseCode = StatusPageResponse.ResponseCode.INTERNAL_SERVER_ERROR; + message = "Internal Server Error"; + hiddenMessage = ExceptionUtils.getStackTraceAsString(e); + } + + TimeZone tz = TimeZone.getTimeZone("UTC"); + long currentTime = timer.getCurrentTimeInMillis(); + StatusPageResponse response = new StatusPageResponse(); + StringBuilder content = new StringBuilder(); + response.setContentType("text/html"); + response.setResponseCode(responseCode); + content.append("<!-- Answer to request ").append(httpRequest.getRequest()).append(" -->\n"); + content.append("<p>UTC time when creating this page: ").append(RealTimer.printDateNoMilliSeconds(currentTime, tz)).append("</p>"); + response.writeHtmlHeader(content, message); + response.writeHtmlFooter(content, hiddenMessage); + response.writeContent(content.toString()); + + + answerCurrentStatusRequest(response); + } + public void answerCurrentStatusRequest(StatusPageResponse r) { synchronized (answerMonitor) { response = r; diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/StateRequests.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/StateRequests.java new file mode 100644 index 00000000000..f0f98120d72 --- /dev/null +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/StateRequests.java @@ -0,0 +1,30 @@ +package com.yahoo.vespa.clustercontroller.core; + +import com.yahoo.vespa.clustercontroller.utils.staterestapi.requests.UnitStateRequest; + +import java.util.List; + +public class StateRequests { + + public static class Get extends StateRequests implements UnitStateRequest { + private final List<String> path; + private final int recursive; + + public Get(String req, int recursive) { + path = req.isEmpty() ? List.of() : List.of(req.split("/")); + this.recursive = recursive; + } + + @Override + public int getRecursiveLevels() { + return recursive; + } + + @Override + public List<String> getUnitPath() { + return path; + } + + } + +} diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/restapiv2/StateRestApiTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/restapiv2/StateRestApiTest.java index 205d5b05b29..bec12ccb195 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/restapiv2/StateRestApiTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/restapiv2/StateRestApiTest.java @@ -22,6 +22,7 @@ import java.util.Arrays; import java.util.Collection; import java.util.HashMap; import java.util.LinkedHashMap; +import java.util.List; import java.util.Map; import java.util.Set; import java.util.TreeMap; @@ -36,18 +37,18 @@ public abstract class StateRestApiTest { Map<Integer, ClusterControllerStateRestAPI.Socket> ccSockets; public static class StateRequest implements UnitStateRequest { - private final String[] path; + private final List<String> path; private final int recursive; StateRequest(String req, int recursive) { - path = req.isEmpty() ? new String[0] : req.split("/"); + path = req.isEmpty() ? List.of() : List.of(req.split("/")); this.recursive = recursive; } @Override public int getRecursiveLevels() { return recursive; } @Override - public String[] getUnitPath() { return path; } + public List<String> getUnitPath() { return path; } } protected void setUp(boolean dontInitializeNode2) { diff --git a/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/errors/MissingUnitException.java b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/errors/MissingUnitException.java index 838e13fc4ee..58a862f4878 100644 --- a/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/errors/MissingUnitException.java +++ b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/errors/MissingUnitException.java @@ -1,19 +1,21 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.clustercontroller.utils.staterestapi.errors; +import java.util.List; + public class MissingUnitException extends StateRestApiException { - private static String createMessage(String[] path, int level) { + private static String createMessage(List<String> path, int level) { StringBuilder sb = new StringBuilder(); sb.append("No such resource '"); for (int i=0; i<=level; ++i) { if (i != 0) sb.append('/'); - sb.append(path[i]); + sb.append(path.get(i)); } return sb.append("'.").toString(); } - public MissingUnitException(String[] path, int level) { + public MissingUnitException(List<String> path, int level) { super(createMessage(path, level)); setHtmlCode(404); setHtmlStatus(getMessage()); diff --git a/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/errors/OperationNotSupportedForUnitException.java b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/errors/OperationNotSupportedForUnitException.java index 342f568eacc..abc55d68bc6 100644 --- a/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/errors/OperationNotSupportedForUnitException.java +++ b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/errors/OperationNotSupportedForUnitException.java @@ -2,16 +2,17 @@ package com.yahoo.vespa.clustercontroller.utils.staterestapi.errors; import java.util.Arrays; +import java.util.List; public class OperationNotSupportedForUnitException extends StateRestApiException { - private static String createMessage(String[] path, String description) { + private static String createMessage(List<String> path, String description) { return new StringBuilder() - .append(Arrays.toString(path)).append(": ").append(description) + .append(Arrays.toString(path.toArray())).append(": ").append(description) .toString(); } - public OperationNotSupportedForUnitException(String path[], String description) { + public OperationNotSupportedForUnitException(List<String> path, String description) { super(createMessage(path, description)); setHtmlCode(405); setHtmlStatus("Operation not supported for resource"); diff --git a/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/requests/UnitRequest.java b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/requests/UnitRequest.java index 53abfff8aba..52bef3b09c0 100644 --- a/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/requests/UnitRequest.java +++ b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/requests/UnitRequest.java @@ -1,8 +1,10 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.clustercontroller.utils.staterestapi.requests; +import java.util.List; + public interface UnitRequest { - String[] getUnitPath(); + List<String> getUnitPath(); } diff --git a/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/response/SetResponse.java b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/response/SetResponse.java index 1c704ea3b63..2287abb5ca7 100644 --- a/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/response/SetResponse.java +++ b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/response/SetResponse.java @@ -24,7 +24,7 @@ public class SetResponse { public boolean getWasModified() { return wasModified; } /** - * Human readable reason. + * Human-readable reason. * * @return reason as string */ diff --git a/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/server/RestApiHandler.java b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/server/RestApiHandler.java index 654481aee33..ceec4f67e1b 100644 --- a/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/server/RestApiHandler.java +++ b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/server/RestApiHandler.java @@ -25,7 +25,6 @@ import java.time.Clock; import java.time.Duration; import java.time.Instant; import java.util.ArrayList; -import java.util.Arrays; import java.util.List; import java.util.Map; import java.util.Optional; @@ -64,7 +63,7 @@ public class RestApiHandler implements HttpRequestHandler { Instant start = clock.instant(); try{ - final String[] unitPath = createUnitPath(request); + List<String> unitPath = createUnitPath(request); if (request.getHttpOperation().equals(HttpRequest.HttpOp.GET)) { final int recursiveLevel = getRecursiveLevel(request); UnitResponse data = restApi.getState(new UnitStateRequest() { @@ -73,9 +72,7 @@ public class RestApiHandler implements HttpRequestHandler { return recursiveLevel; } @Override - public String[] getUnitPath() { - return unitPath; - } + public List<String> getUnitPath() { return unitPath; } }); return new JsonHttpResult().setJson(jsonWriter.createJson(data)); } else { @@ -87,7 +84,7 @@ public class RestApiHandler implements HttpRequestHandler { return setRequestData.stateMap; } @Override - public String[] getUnitPath() { + public List<String> getUnitPath() { return unitPath; } @Override @@ -137,9 +134,9 @@ public class RestApiHandler implements HttpRequestHandler { } } - private String[] createUnitPath(HttpRequest request) { - List<String> path = Arrays.asList(request.getPath().split("/")); - return path.subList(3, path.size()).toArray(new String[0]); + private List<String> createUnitPath(HttpRequest request) { + List<String> path = List.of(request.getPath().split("/")); + return path.subList(3, path.size()); } private int getRecursiveLevel(HttpRequest request) throws StateRestApiException { diff --git a/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/util/ComponentMetricReporter.java b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/util/ComponentMetricReporter.java index bed2ddcecd0..cde44b76fcb 100644 --- a/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/util/ComponentMetricReporter.java +++ b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/util/ComponentMetricReporter.java @@ -1,12 +1,12 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/** - * Metric reporter wrapper to add component name prefix and common dimensions. - */ package com.yahoo.vespa.clustercontroller.utils.util; import java.util.Map; import java.util.TreeMap; +/** + * Metric reporter wrapper to add component name prefix and common dimensions. + */ public class ComponentMetricReporter implements MetricReporter { private final MetricReporter impl; diff --git a/clustercontroller-utils/src/test/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/DummyStateApi.java b/clustercontroller-utils/src/test/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/DummyStateApi.java index d611b0c0ea8..2fcbf22aa59 100644 --- a/clustercontroller-utils/src/test/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/DummyStateApi.java +++ b/clustercontroller-utils/src/test/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/DummyStateApi.java @@ -10,6 +10,7 @@ import com.yahoo.vespa.clustercontroller.utils.staterestapi.requests.UnitStateRe import com.yahoo.vespa.clustercontroller.utils.staterestapi.response.*; import java.util.LinkedHashMap; +import java.util.List; import java.util.Map; public class DummyStateApi implements StateRestAPI { @@ -28,8 +29,8 @@ public class DummyStateApi implements StateRestAPI { } public class SubUnitListImpl implements SubUnitList { - private Map<String, String> links = new LinkedHashMap<>(); - private Map<String, UnitResponse> values = new LinkedHashMap<>(); + private final Map<String, String> links = new LinkedHashMap<>(); + private final Map<String, UnitResponse> values = new LinkedHashMap<>(); @Override public Map<String, String> getSubUnitLinks() { return links; } @@ -156,18 +157,18 @@ public class DummyStateApi implements StateRestAPI { @Override public UnitResponse getState(UnitStateRequest request) throws StateRestApiException { checkForInducedException(); - String[] path = request.getUnitPath(); - if (path.length == 0) { + List<String> path = request.getUnitPath(); + if (path.size() == 0) { return getClusterList(request.getRecursiveLevels()); } - final DummyBackend.Cluster c = backend.getClusters().get(path[0]); + DummyBackend.Cluster c = backend.getClusters().get(path.get(0)); if (c == null) throw new MissingUnitException(path, 0); - if (path.length == 1) { + if (path.size() == 1) { return getClusterState(c, request.getRecursiveLevels()); } - final DummyBackend.Node n = c.nodes.get(path[1]); + DummyBackend.Node n = c.nodes.get(path.get(1)); if (n == null) throw new MissingUnitException(path, 1); - if (path.length == 2) { + if (path.size() == 2) { return getNodeState(n); } throw new MissingUnitException(path, 3); @@ -176,15 +177,15 @@ public class DummyStateApi implements StateRestAPI { @Override public SetResponse setUnitState(SetUnitStateRequest request) throws StateRestApiException { checkForInducedException(); - String[] path = request.getUnitPath(); - if (path.length != 2) { + List<String> path = request.getUnitPath(); + if (path.size() != 2) { throw new OperationNotSupportedForUnitException( path, "You can only set states on nodes"); } DummyBackend.Node n = null; - DummyBackend.Cluster c = backend.getClusters().get(path[0]); + DummyBackend.Cluster c = backend.getClusters().get(path.get(0)); if (c != null) { - n = c.nodes.get(path[1]); + n = c.nodes.get(path.get(1)); } if (n == null) throw new MissingUnitException(path, 2); Map<String, UnitState> newState = request.getNewState(); diff --git a/clustercontroller-utils/src/test/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/StateRestAPITest.java b/clustercontroller-utils/src/test/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/StateRestAPITest.java index 47b12f883ff..29e33b7906a 100644 --- a/clustercontroller-utils/src/test/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/StateRestAPITest.java +++ b/clustercontroller-utils/src/test/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/StateRestAPITest.java @@ -19,6 +19,7 @@ import com.yahoo.vespa.clustercontroller.utils.staterestapi.server.RestApiHandle import com.yahoo.vespa.clustercontroller.utils.test.TestTransport; import org.junit.jupiter.api.Test; +import java.util.List; import java.util.Optional; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -486,9 +487,7 @@ public class StateRestAPITest { assertEquals(expected, result.getContent().toString()); } { - String path[] = new String[1]; - path[0] = "foo"; - stateApi.induceException(new OperationNotSupportedForUnitException(path, "Foo")); + stateApi.induceException(new OperationNotSupportedForUnitException(List.of("foo"), "Foo")); HttpResult result = execute(new HttpRequest().setPath("/cluster/v2")); assertEquals(405, result.getHttpReturnCode(), result.toString(true)); assertEquals("Operation not supported for resource", result.getHttpReturnCodeDescription(), result.toString(true)); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java index cab3e89c606..062133b6b6e 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java @@ -290,7 +290,8 @@ public class Deployment implements com.yahoo.config.provision.Deployment { .isBootstrap(isBootstrap) .force(force) .waitForResourcesInPrepare(waitForResourcesInPrepare) - .tenantSecretStores(session.getTenantSecretStores()); + .tenantSecretStores(session.getTenantSecretStores()) + .dataplaneTokens(session.getDataplaneTokens()); session.getDockerImageRepository().ifPresent(params::dockerImageRepository); session.getAthenzDomain().ifPresent(params::athenzDomain); session.getCloudAccount().ifPresent(params::cloudAccount); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java index ff661fbcc74..f82aa405380 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java @@ -283,6 +283,7 @@ public class SessionRepository { session.setTenantSecretStores(existingSession.getTenantSecretStores()); session.setOperatorCertificates(existingSession.getOperatorCertificates()); session.setCloudAccount(existingSession.getCloudAccount()); + session.setDataplaneTokens(existingSession.getDataplaneTokens()); return session; } diff --git a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/DataplaneProxyCredentials.java b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/DataplaneProxyCredentials.java index a30252b1626..05c6f5be467 100644 --- a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/DataplaneProxyCredentials.java +++ b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/DataplaneProxyCredentials.java @@ -37,8 +37,16 @@ public class DataplaneProxyCredentials extends AbstractComponent { @Inject public DataplaneProxyCredentials() { - certificateFile = Paths.get(Defaults.getDefaults().underVespaHome("tmp/proxy_cert.pem")); - keyFile = Paths.get(Defaults.getDefaults().underVespaHome("tmp/proxy_key.pem")); + this( + Paths.get(Defaults.getDefaults().underVespaHome("tmp/proxy_cert.pem")), + Paths.get(Defaults.getDefaults().underVespaHome("tmp/proxy_key.pem")) + ); + } + + public DataplaneProxyCredentials(Path certificateFile, Path keyFile){ + this.certificateFile = certificateFile; + this.keyFile = keyFile; + var existing = regenerateCredentials(certificateFile, keyFile).orElse(null); if (existing == null) { X509CertificateWithKey selfSigned = X509CertificateUtils.createSelfSigned("cn=vespa dataplane proxy", Duration.ofDays(30)); @@ -48,6 +56,7 @@ public class DataplaneProxyCredentials extends AbstractComponent { } else { this.certificate = existing; } + } /** diff --git a/container-disc/src/main/java/com/yahoo/container/jdisc/DataplaneProxyService.java b/container-disc/src/main/java/com/yahoo/container/jdisc/DataplaneProxyService.java index 6d871b7283f..e6af65c0bc8 100644 --- a/container-disc/src/main/java/com/yahoo/container/jdisc/DataplaneProxyService.java +++ b/container-disc/src/main/java/com/yahoo/container/jdisc/DataplaneProxyService.java @@ -7,11 +7,14 @@ import com.yahoo.jdisc.http.server.jetty.DataplaneProxyCredentials; import javax.inject.Inject; import java.io.IOException; -import java.net.URI; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.nio.file.StandardCopyOption; +import java.util.concurrent.ScheduledThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.logging.Level; +import java.util.logging.Logger; /** * Configures a data plane proxy. Currently using Nginx. @@ -20,100 +23,137 @@ import java.nio.file.StandardCopyOption; */ public class DataplaneProxyService extends AbstractComponent { + private static Logger logger = Logger.getLogger(DataplaneProxyService.class.getName()); private static final String PREFIX = "/opt/vespa"; - private static final Path CONFIG_TEMPLATE = Paths.get(PREFIX, "conf/nginx/nginx.conf.template"); - private static final Path clientCertificateFile = Paths.get(PREFIX, "conf/nginx/client_cert.pem"); - private static final Path clientKeyFile = Paths.get(PREFIX, "conf/nginx/client_key.pem"); - private static final Path serverCertificateFile = Paths.get(PREFIX, "conf/nginx/server_cert.pem"); - private static final Path serverKeyFile = Paths.get(PREFIX, "conf/nginx/server_key.pem"); + private final Path configTemplate; + private final Path serverCertificateFile; + private final Path serverKeyFile; + private final Path nginxConf; - private static final Path nginxConf = Paths.get(PREFIX, "conf/nginx/nginx.conf"); + private final ProxyCommands proxyCommands; + private final ScheduledThreadPoolExecutor executorService; + private final Path root; + + enum NginxState {INITIALIZING, RUNNING, RELOAD_REQUIRED, STOPPED}; + private NginxState state; + private NginxState wantedState; + + private DataplaneProxyConfig cfg; + private Path proxyCredentialsCert; + private Path proxyCredentialsKey; - private boolean started; @Inject public DataplaneProxyService() { - this.started = false; + this(Paths.get(PREFIX), new NginxProxyCommands(), 1); + } + + DataplaneProxyService(Path root, ProxyCommands proxyCommands, int reloadPeriodMinutes) { + this.root = root; + this.proxyCommands = proxyCommands; + changeState(NginxState.INITIALIZING); + wantedState = NginxState.RUNNING; + configTemplate = root.resolve("conf/nginx/nginx.conf.template"); + serverCertificateFile = root.resolve("conf/nginx/server_cert.pem"); + serverKeyFile = root.resolve("conf/nginx/server_key.pem"); + nginxConf = root.resolve("conf/nginx/nginx.conf"); + + executorService = new ScheduledThreadPoolExecutor(1); + executorService.scheduleAtFixedRate(this::converge, reloadPeriodMinutes, reloadPeriodMinutes, TimeUnit.MINUTES); } public void reconfigure(DataplaneProxyConfig config, DataplaneProxyCredentials credentialsProvider) { - try { - String serverCert = config.serverCertificate(); - String serverKey = config.serverKey(); - - boolean configChanged = false; - configChanged |= writeFile(serverCertificateFile, serverCert); - configChanged |= writeFile(serverKeyFile, serverKey); - configChanged |= writeFile(nginxConf, - nginxConfig( - credentialsProvider.certificateFile(), - credentialsProvider.keyFile(), - serverCertificateFile, - serverKeyFile, - config.port(), - PREFIX - )); - if (!started) { - startNginx(); - started = true; - } else if (configChanged){ - reloadNginx(); - } - } catch (IOException e) { - throw new RuntimeException("Error reconfiguring data plane proxy", e); + synchronized (this) { + this.cfg = config; + this.proxyCredentialsCert = credentialsProvider.certificateFile(); + this.proxyCredentialsKey = credentialsProvider.keyFile(); } } - private void startNginx() { - try { - Process startCommand = new ProcessBuilder().command( - "nginx", - "-c", nginxConf.toString() - ).start(); - int exitCode = startCommand.waitFor(); - if (exitCode != 0) { - throw new RuntimeException("Non-zero exitcode from nginx: %d".formatted(exitCode)); - } - } catch (IOException | InterruptedException e) { - throw new RuntimeException("Could not start nginx", e); - } + private void changeState(NginxState newState) { + state = newState; } - private void reloadNginx() { - try { - Process reloadCommand = new ProcessBuilder().command( - "nginx", - "-s", "reload" - ).start(); - int exitCode = reloadCommand.waitFor(); - if (exitCode != 0) { - throw new RuntimeException("Non-zero exitcode from nginx: %d".formatted(exitCode)); - } - } catch (IOException | InterruptedException e) { - throw new RuntimeException("Could not start nginx", e); + void converge() { + DataplaneProxyConfig config; + Path proxyCredentialsCert; + Path proxyCredentialsKey; + synchronized (this) { + config = cfg; + proxyCredentialsCert = this.proxyCredentialsCert; + proxyCredentialsKey = this.proxyCredentialsKey; + this.cfg = null; + this.proxyCredentialsCert = null; + this.proxyCredentialsKey = null; } - } + if (config != null) { + try { - private void stopNginx() { - try { - Process stopCommand = new ProcessBuilder().command( - "nginx", - "-s", "stop" - ).start(); - int exitCode = stopCommand.waitFor(); - if (exitCode != 0) { - throw new RuntimeException("Non-zero exitcode from nginx: %d".formatted(exitCode)); + String serverCert = config.serverCertificate(); + String serverKey = config.serverKey(); + + boolean configChanged = false; + configChanged |= writeFile(serverCertificateFile, serverCert); + configChanged |= writeFile(serverKeyFile, serverKey); + configChanged |= writeFile(nginxConf, + nginxConfig( + configTemplate, + proxyCredentialsCert, + proxyCredentialsKey, + serverCertificateFile, + serverKeyFile, + config.port(), + root + )); + if (configChanged && state == NginxState.RUNNING) { + changeState(NginxState.RELOAD_REQUIRED); + } + } catch (IOException e) { + throw new RuntimeException("Error reconfiguring data plane proxy", e); + } + } + if (wantedState == NginxState.RUNNING) { + boolean nginxRunning = proxyCommands.isRunning(); + if (!nginxRunning) { + try { + proxyCommands.start(nginxConf); + changeState(wantedState); + } catch (Exception e) { + logger.log(Level.INFO, "Failed to start nginx, will retry"); + } + } else if (nginxRunning && state == NginxState.RELOAD_REQUIRED) { + try { + proxyCommands.reload(); + changeState(wantedState); + } catch (Exception e) { + logger.log(Level.INFO, "Failed to reconfigure nginx, will retry."); + } + } + } else if (wantedState == NginxState.STOPPED) { + if (proxyCommands.isRunning()) { + try { + proxyCommands.stop(); + changeState(wantedState); + executorService.shutdownNow(); + } catch (Exception e) { + logger.log(Level.INFO, "Failed to stop nginx, will retry"); + } } - } catch (IOException | InterruptedException e) { - throw new RuntimeException("Could not start nginx", e); + } else { + logger.warning("Unknown state " + wantedState); } } @Override public void deconstruct() { super.deconstruct(); - stopNginx(); + wantedState = NginxState.STOPPED; + try { + executorService.awaitTermination(5, TimeUnit.MINUTES); + } catch (InterruptedException e) { + logger.log(Level.WARNING, "Error shutting down proxy reload thread"); + } } /* @@ -121,7 +161,7 @@ public class DataplaneProxyService extends AbstractComponent { * return true if file was changed, false if no changes */ private boolean writeFile(Path file, String contents) throws IOException { - Path tempPath = Paths.get(file.toFile().getAbsolutePath() + ".new"); + Path tempPath = file.getParent().resolve(file.getFileName().toString() + ".new"); Files.createDirectories(tempPath.getParent()); Files.writeString(tempPath, contents); @@ -135,21 +175,22 @@ public class DataplaneProxyService extends AbstractComponent { } static String nginxConfig( + Path configTemplate, Path clientCert, Path clientKey, Path serverCert, Path serverKey, int vespaPort, - String prefix) { + Path root) { try { - String nginxTemplate = Files.readString(CONFIG_TEMPLATE); + String nginxTemplate = Files.readString(configTemplate); nginxTemplate = replace(nginxTemplate, "client_cert", clientCert.toString()); nginxTemplate = replace(nginxTemplate, "client_key", clientKey.toString()); nginxTemplate = replace(nginxTemplate, "server_cert", serverCert.toString()); nginxTemplate = replace(nginxTemplate, "server_key", serverKey.toString()); nginxTemplate = replace(nginxTemplate, "vespa_port", Integer.toString(vespaPort)); - nginxTemplate = replace(nginxTemplate, "prefix", prefix); + nginxTemplate = replace(nginxTemplate, "prefix", root.toString()); // TODO: verify that all template vars have been expanded return nginxTemplate; @@ -161,4 +202,78 @@ public class DataplaneProxyService extends AbstractComponent { private static String replace(String template, String key, String value) { return template.replaceAll("\\$\\{%s\\}".formatted(key), value); } + + NginxState state() { + return state; + } + + NginxState wantedState() { + return wantedState; + } + + public interface ProxyCommands { + void start(Path configFile); + void stop(); + void reload(); + boolean isRunning(); + } + + public static class NginxProxyCommands implements ProxyCommands { + + @Override + public void start(Path configFile) { + try { + Process startCommand = new ProcessBuilder().command( + "nginx", + "-c", configFile.toString() + ).start(); + int exitCode = startCommand.waitFor(); + if (exitCode != 0) { + throw new RuntimeException("Non-zero exitcode from nginx: %d".formatted(exitCode)); + } + } catch (IOException | InterruptedException e) { + throw new RuntimeException("Could not start nginx", e); + } + } + + @Override + public void stop() { + try { + Process stopCommand = new ProcessBuilder().command( + "nginx", + "-s", "stop" + ).start(); + int exitCode = stopCommand.waitFor(); + if (exitCode != 0) { + throw new RuntimeException("Non-zero exitcode from nginx: %d".formatted(exitCode)); + } + } catch (IOException | InterruptedException e) { + throw new RuntimeException("Could not start nginx", e); + } + + } + + @Override + public void reload() { + try { + Process reloadCommand = new ProcessBuilder().command( + "nginx", + "-s", "reload" + ).start(); + int exitCode = reloadCommand.waitFor(); + if (exitCode != 0) { + throw new RuntimeException("Non-zero exitcode from nginx: %d".formatted(exitCode)); + } + } catch (IOException | InterruptedException e) { + throw new RuntimeException("Could not start nginx", e); + } + } + + @Override + public boolean isRunning() { + return ProcessHandle.allProcesses() + .map(ProcessHandle::info) + .anyMatch(info -> info.command().orElse("").endsWith("nginx")); + } + } } diff --git a/container-disc/src/test/java/com/yahoo/container/jdisc/DataplaneProxyServiceTest.java b/container-disc/src/test/java/com/yahoo/container/jdisc/DataplaneProxyServiceTest.java new file mode 100644 index 00000000000..947c99adf51 --- /dev/null +++ b/container-disc/src/test/java/com/yahoo/container/jdisc/DataplaneProxyServiceTest.java @@ -0,0 +1,174 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.jdisc; + +import com.google.common.jimfs.Jimfs; +import com.yahoo.cloud.config.DataplaneProxyConfig; +import com.yahoo.jdisc.http.server.jetty.DataplaneProxyCredentials; +import com.yahoo.security.KeyUtils; +import com.yahoo.security.X509CertificateUtils; +import com.yahoo.security.X509CertificateWithKey; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.FileSystem; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Duration; + +import static com.yahoo.yolean.Exceptions.uncheck; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +public class DataplaneProxyServiceTest { + private FileSystem fileSystem = Jimfs.newFileSystem(); + DataplaneProxyService.ProxyCommands proxyCommandsMock = Mockito.mock(DataplaneProxyService.ProxyCommands.class); + + @Test + public void starts_and_reloads_if_no_errors() throws IOException { + DataplaneProxyService service = dataplaneProxyService(proxyCommandsMock); + + assertEquals(DataplaneProxyService.NginxState.INITIALIZING, service.state()); + service.reconfigure(proxyConfig(), credentials(fileSystem)); + + // Simulate executor next tick + service.converge(); + assertEquals(DataplaneProxyService.NginxState.RUNNING, service.state()); + + // Trigger reload by recreating the proxy config (generates new server cert) + service.reconfigure(proxyConfig(), credentials(fileSystem)); + service.converge(); + assertEquals(DataplaneProxyService.NginxState.RUNNING, service.state()); + } + + @Test + public void retries_startup_errors() throws IOException { + Mockito.doThrow(new RuntimeException("IO error")).doNothing().when(proxyCommandsMock).start(any()); + DataplaneProxyService service = dataplaneProxyService(proxyCommandsMock); + + assertEquals(DataplaneProxyService.NginxState.INITIALIZING, service.state()); + service.reconfigure(proxyConfig(), credentials(fileSystem)); + + // Start nginx, starting will fail, so the service should be in INITIALIZING state + service.converge(); + assertEquals(DataplaneProxyService.NginxState.INITIALIZING, service.state()); + service.converge(); + assertEquals(DataplaneProxyService.NginxState.RUNNING, service.state()); + } + + @Test + public void retries_reload_errors() throws IOException { + Mockito.doThrow(new RuntimeException("IO error")).doNothing().when(proxyCommandsMock).reload(); + when(proxyCommandsMock.isRunning()).thenReturn(false); + DataplaneProxyService service = dataplaneProxyService(proxyCommandsMock); + + // Make sure service in running state + service.reconfigure(proxyConfig(), credentials(fileSystem)); + service.converge(); + assertEquals(DataplaneProxyService.NginxState.RUNNING, service.state()); + when(proxyCommandsMock.isRunning()).thenReturn(true); + + // Trigger reload, verifies 2nd attempt succeeds + service.reconfigure(proxyConfig(), credentials(fileSystem)); + service.converge(); + assertEquals(DataplaneProxyService.NginxState.RELOAD_REQUIRED, service.state()); + service.converge(); + assertEquals(DataplaneProxyService.NginxState.RUNNING, service.state()); + verify(proxyCommandsMock, times(2)).reload(); + } + + @Test + public void converges_to_wanted_state_when_nginx_not_running() throws IOException { + DataplaneProxyService.ProxyCommands proxyCommands = new TestProxyCommands(); + DataplaneProxyService service = dataplaneProxyService(proxyCommands); + + assertFalse(proxyCommands.isRunning()); + service.reconfigure(proxyConfig(), credentials(fileSystem)); + service.converge(); + assertEquals(DataplaneProxyService.NginxState.RUNNING, service.state()); + assertTrue(proxyCommands.isRunning()); + + // Simulate nginx process dying + proxyCommands.stop(); + assertFalse(proxyCommands.isRunning()); + service.converge(); + assertTrue(proxyCommands.isRunning()); + } + + @Test + public void shuts_down() throws IOException { + DataplaneProxyService.ProxyCommands proxyCommands = new TestProxyCommands(); + DataplaneProxyService service = dataplaneProxyService(proxyCommands); + service.converge(); + assertTrue(proxyCommands.isRunning()); + assertEquals(DataplaneProxyService.NginxState.RUNNING, service.state()); + + new Thread(service::deconstruct).start(); // deconstruct will block until nginx is stopped + // Wait for above thread to set the wanted state to STOPPED + while (service.wantedState() != DataplaneProxyService.NginxState.STOPPED) { + try { + Thread.sleep(10); + } catch (InterruptedException e) { + } + } + service.converge(); + assertEquals(service.state(), DataplaneProxyService.NginxState.STOPPED); + assertFalse(proxyCommands.isRunning()); + } + + private DataplaneProxyService dataplaneProxyService(DataplaneProxyService.ProxyCommands proxyCommands) throws IOException { + Path root = fileSystem.getPath("/opt/vespa"); + + Path nginxConf = root.resolve("conf/nginx/nginx.conf.template"); + Files.createDirectories(nginxConf.getParent()); + Files.write(nginxConf, "".getBytes(StandardCharsets.UTF_8)); + + DataplaneProxyService service = new DataplaneProxyService(root, proxyCommands, 100); + return service; + } + + private DataplaneProxyConfig proxyConfig() { + X509CertificateWithKey selfSigned = X509CertificateUtils.createSelfSigned("cn=test", Duration.ofMinutes(10)); + return new DataplaneProxyConfig.Builder() + .port(1234) + .serverCertificate(X509CertificateUtils.toPem(selfSigned.certificate())) + .serverKey(KeyUtils.toPem(selfSigned.privateKey())) + .build(); + } + + private DataplaneProxyCredentials credentials(FileSystem fileSystem) { + Path path = fileSystem.getPath("/tmp"); + uncheck(() -> Files.createDirectories(path)); + return new DataplaneProxyCredentials(path.resolve("cert.pem"), path.resolve("key.pem")); + } + + private static class TestProxyCommands implements DataplaneProxyService.ProxyCommands { + private boolean running = false; + + @Override + public void start(Path configFile) { + running = true; + } + + @Override + public void stop() { + running = false; + } + + @Override + public void reload() { + + } + + @Override + public boolean isRunning() { + return running; + } + } +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Nodes.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Nodes.java index b10a371e8bd..c09bca272c3 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Nodes.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Nodes.java @@ -376,6 +376,7 @@ public class Nodes { Predicate<Node> nodeInConfig = (node) -> hostIpConfig.contains(node.hostname()); performOn(nodeInConfig, (node, lock) -> { IP.Config ipConfig = hostIpConfig.require(node.hostname()); + log.info("Setting IP config for " + node.hostname() + " to " + ipConfig); return write(node.with(ipConfig), lock); }); } |