diff options
41 files changed, 544 insertions, 79 deletions
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/AbstractService.java b/config-model/src/main/java/com/yahoo/vespa/model/AbstractService.java index 60a49598c42..daa237d90c1 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/AbstractService.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/AbstractService.java @@ -482,8 +482,8 @@ public abstract class AbstractService extends AbstractConfigProducer<AbstractCon * Must be done this way since the system test framework * currently uses the first port as container http port. */ - public void reservePortPrepended(int port) { - hostResource.reservePort(this, port); + public void reservePortPrepended(int port, String suffix) { + hostResource.reservePort(this, port, suffix); ports.add(0, port); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/ConfigProxy.java b/config-model/src/main/java/com/yahoo/vespa/model/ConfigProxy.java index c540a5f62d2..7ab28faa434 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/ConfigProxy.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/ConfigProxy.java @@ -47,6 +47,11 @@ public class ConfigProxy extends AbstractService { */ public int getPortCount() { return 1; } + @Override + public String[] getPortSuffixes() { + return new String[]{"rpc"}; + } + /** * The config proxy is not started by the config system! */ diff --git a/config-model/src/main/java/com/yahoo/vespa/model/ConfigSentinel.java b/config-model/src/main/java/com/yahoo/vespa/model/ConfigSentinel.java index cd92f27cc50..1b5c5e4a579 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/ConfigSentinel.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/ConfigSentinel.java @@ -26,7 +26,7 @@ public class ConfigSentinel extends AbstractService implements SentinelConfig.Pr super(host, "sentinel"); this.applicationId = applicationId; this.zone = zone; - portsMeta.on(0).tag("rpc").tag("notyet"); + portsMeta.on(0).tag("rpc").tag("admin"); portsMeta.on(1).tag("telnet").tag("interactive").tag("http").tag("state"); setProp("clustertype", "hosts"); setProp("clustername", "admin"); @@ -48,6 +48,11 @@ public class ConfigSentinel extends AbstractService implements SentinelConfig.Pr public int getPortCount() { return 2; } @Override + public String[] getPortSuffixes() { + return new String[]{ "rpc", "http" }; + } + + @Override public int getHealthPort() {return getRelativePort(1); } /** diff --git a/config-model/src/main/java/com/yahoo/vespa/model/HostResource.java b/config-model/src/main/java/com/yahoo/vespa/model/HostResource.java index a27b33173ee..3a8cc5c2e4c 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/HostResource.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/HostResource.java @@ -6,6 +6,7 @@ import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.model.api.HostInfo; import com.yahoo.config.provision.ClusterMembership; import com.yahoo.config.provision.Flavor; +import com.yahoo.config.provision.NetworkPorts; import javax.annotation.Nullable; import java.util.ArrayList; @@ -36,10 +37,23 @@ public class HostResource implements Comparable<HostResource> { /** Map from "sentinel name" to service */ private final Map<String, Service> services = new LinkedHashMap<>(); - private final Map<Integer, Service> portDB = new LinkedHashMap<>(); + private final Map<Integer, NetworkPortRequestor> portDB = new LinkedHashMap<>(); private int allocatedPorts = 0; + static class PortReservation { + int gotPort; + NetworkPortRequestor service; + String suffix; + PortReservation(int port, NetworkPortRequestor svc, String suf) { + this.gotPort = port; + this.service = svc; + this.suffix = suf; + } + } + + private List<PortReservation> portReservations = new ArrayList<>(); + private Set<ClusterMembership> clusterMemberships = new LinkedHashSet<>(); // Empty for self-hosted Vespa. @@ -48,6 +62,12 @@ public class HostResource implements Comparable<HostResource> { /** The current Vespa version running on this node, or empty if not known */ private final Optional<Version> version; + private Optional<NetworkPorts> networkPortsList = Optional.empty(); + + public Optional<NetworkPorts> networkPorts() { return networkPortsList; } + + public void addNetworkPorts(NetworkPorts ports) { this.networkPortsList = Optional.of(ports); } + /** * Create a new {@link HostResource} bound to a specific {@link com.yahoo.vespa.model.Host}. * @@ -108,22 +128,22 @@ public class HostResource implements Comparable<HostResource> { return ports; } - private List<Integer> allocatePorts(DeployLogger deployLogger, AbstractService service, int wantedPort) { + private List<Integer> allocatePorts(DeployLogger deployLogger, NetworkPortRequestor service, int wantedPort) { List<Integer> ports = new ArrayList<>(); if (service.getPortCount() < 1) return ports; int serviceBasePort = BASE_PORT + allocatedPorts; if (wantedPort > 0) { - if (service.getPortCount() < 1) { - throw new RuntimeException(service + " wants baseport " + wantedPort + - ", but it has not reserved any ports, so it cannot name a desired baseport."); - } if (service.requiresWantedPort() || canUseWantedPort(deployLogger, service, wantedPort, serviceBasePort)) serviceBasePort = wantedPort; } + String[] suffixes = service.getPortSuffixes(); + if (suffixes.length != service.getPortCount()) { + throw new IllegalArgumentException("service "+service+" had "+suffixes.length+" port suffixes, but port count "+service.getPortCount()+", mismatch"); + } - reservePort(service, serviceBasePort); + reservePort(service, serviceBasePort, suffixes[0]); ports.add(serviceBasePort); int remainingPortsStart = service.requiresConsecutivePorts() ? @@ -131,17 +151,30 @@ public class HostResource implements Comparable<HostResource> { BASE_PORT + allocatedPorts; for (int i = 0; i < service.getPortCount() - 1; i++) { int port = remainingPortsStart + i; - reservePort(service, port); + reservePort(service, port, suffixes[i+1]); ports.add(port); } + if (suffixes.length != service.getPortCount()) { + throw new IllegalArgumentException("service "+service+" had "+suffixes.length+" port suffixes, but port count "+service.getPortCount()+", mismatch"); + } return ports; } - private boolean canUseWantedPort(DeployLogger deployLogger, AbstractService service, int wantedPort, int serviceBasePort) { + public void flushPortReservations() { + List<NetworkPorts.Allocation> list = new ArrayList<>(); + for (PortReservation pr : portReservations) { + String servType = pr.service.getServiceType(); + String configId = pr.service.getConfigId(); + list.add(new NetworkPorts.Allocation(pr.gotPort, servType, configId, pr.suffix)); + } + this.networkPortsList = Optional.of(new NetworkPorts(list)); + } + + private boolean canUseWantedPort(DeployLogger deployLogger, NetworkPortRequestor service, int wantedPort, int serviceBasePort) { for (int i = 0; i < service.getPortCount(); i++) { int port = wantedPort + i; if (portDB.containsKey(port)) { - AbstractService s = (AbstractService)portDB.get(port); + NetworkPortRequestor s = portDB.get(port); deployLogger.log(Level.WARNING, service.getServiceName() +" cannot reserve port " + port + " on " + this + ": Already reserved for " + s.getServiceName() + ". Using default port range from " + serviceBasePort); @@ -159,7 +192,7 @@ public class HostResource implements Comparable<HostResource> { * @param service the service that wishes to reserve the port. * @param port the port to be reserved. */ - void reservePort(AbstractService service, int port) { + void reservePort(NetworkPortRequestor service, int port, String suffix) { if (portDB.containsKey(port)) { portAlreadyReserved(service, port); } else { @@ -170,6 +203,7 @@ public class HostResource implements Comparable<HostResource> { } } portDB.put(port, service); + portReservations.add(new PortReservation(port, service, suffix)); } } @@ -178,8 +212,8 @@ public class HostResource implements Comparable<HostResource> { port < BASE_PORT + MAX_PORTS; } - private void portAlreadyReserved(AbstractService service, int port) { - AbstractService otherService = (AbstractService)portDB.get(port); + private void portAlreadyReserved(NetworkPortRequestor service, int port) { + NetworkPortRequestor otherService = portDB.get(port); int nextAvailablePort = nextAvailableBaseport(service.getPortCount()); if (nextAvailablePort == 0) { noMoreAvailablePorts(); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/HostSystem.java b/config-model/src/main/java/com/yahoo/vespa/model/HostSystem.java index fdfe4f01790..a1b030ffc61 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/HostSystem.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/HostSystem.java @@ -8,6 +8,7 @@ import com.yahoo.config.provision.Capacity; import com.yahoo.config.provision.ClusterMembership; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.HostSpec; +import com.yahoo.config.provision.NetworkPorts; import com.yahoo.config.provision.ProvisionLogger; import java.net.UnknownHostException; @@ -126,9 +127,11 @@ public class HostSystem extends AbstractConfigProducer<Host> { private HostResource addNewHost(HostSpec hostSpec) { Host host = Host.createHost(this, hostSpec.hostname()); - HostResource hostResource = new HostResource(host, hostSpec.version()); + HostResource hostResource = new HostResource(host, + hostSpec.version()); hostResource.setFlavor(hostSpec.flavor()); hostSpec.membership().ifPresent(hostResource::addClusterMembership); + hostSpec.networkPorts().ifPresent(hostResource::addNetworkPorts); hostname2host.put(host.getHostname(), hostResource); log.log(DEBUG, () -> "Added new host resource for " + host.getHostname() + " with flavor " + hostResource.getFlavor()); return hostResource; @@ -141,6 +144,19 @@ public class HostSystem extends AbstractConfigProducer<Host> { .collect(Collectors.toList()); } + public void dumpPortAllocations() { + for (HostResource hr : getHosts()) { + hr.flushPortReservations(); +/* + System.out.println("port allocations for: "+hr.getHostname()); + NetworkPorts ports = hr.networkPorts().get(); + for (NetworkPorts.Allocation allocation: ports.allocations()) { + System.out.println("port="+allocation.port+" [type="+allocation.serviceType+", cfgId="+allocation.configId+", suffix="+allocation.portSuffix+"]"); + } +*/ + } + } + public Map<HostResource, ClusterMembership> allocateHosts(ClusterSpec cluster, Capacity capacity, int groups, DeployLogger logger) { List<HostSpec> allocatedHosts = provisioner.prepare(cluster, capacity, groups, new ProvisionDeployLogger(logger)); // TODO: Even if HostResource owns a set of memberships, we need to return a map because the caller needs the current membership. @@ -177,7 +193,7 @@ public class HostSystem extends AbstractConfigProducer<Host> { Set<HostSpec> getHostSpecs() { return getHosts().stream() .map(host -> new HostSpec(host.getHostname(), Collections.emptyList(), - host.getFlavor(), host.primaryClusterMembership(), host.version())) + host.getFlavor(), host.primaryClusterMembership(), host.version(), host.networkPorts())) .collect(Collectors.toCollection(LinkedHashSet::new)); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/Logd.java b/config-model/src/main/java/com/yahoo/vespa/model/Logd.java index 0f7418582a3..3c7f1ba6cfa 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/Logd.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/Logd.java @@ -32,6 +32,11 @@ public class Logd */ public int getPortCount() { return 1; } + @Override + public String[] getPortSuffixes() { + return new String[]{"http"}; + } + /** Returns the desired base port for this service. */ public int getWantedPort() { return 19089; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/NetworkPortRequestor.java b/config-model/src/main/java/com/yahoo/vespa/model/NetworkPortRequestor.java new file mode 100644 index 00000000000..52319f71810 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/NetworkPortRequestor.java @@ -0,0 +1,53 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.vespa.model; + +/** + * Interface implemented by services using network ports, identifying its requirements. + * @author arnej + */ +public interface NetworkPortRequestor { + + /** Returns the type of service */ + String getServiceType(); + + /** Returns the name that identifies this service for the config-sentinel */ + String getServiceName(); + + /** Returns the config id */ + String getConfigId(); + + /** + * Returns the desired base port for this service, or '0' if this + * service should use the default port allocation mechanism. + * + * @return The desired base port for this service. + */ + default int getWantedPort() { return 0; } + + /** Returns the number of ports needed by this service. */ + int getPortCount(); + + /** + * Returns true if the desired base port (returned by + * getWantedPort()) for this service is the only allowed base + * port. + * + * @return true if this Service requires the wanted base port. + */ + default boolean requiresWantedPort() { return false; } + + /** + * Override if the services does not require consecutive port numbers. I.e. if any ports + * in addition to the baseport should be allocated from Vespa's default port range. + * + * @return true by default + */ + default boolean requiresConsecutivePorts() { return true; } + + /** + * Return names for each port requested. + * The size of the returned array must be equal to getPortCount(). + **/ + String[] getPortSuffixes(); +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/Service.java b/config-model/src/main/java/com/yahoo/vespa/model/Service.java index d5d33a08b5d..0af4355764c 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/Service.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/Service.java @@ -11,7 +11,7 @@ import java.util.Optional; * * @author gjoranv */ -public interface Service extends ConfigProducer { +public interface Service extends ConfigProducer, NetworkPortRequestor { /** * Services that should be started by config-sentinel must return @@ -44,39 +44,6 @@ public interface Service extends ConfigProducer { boolean getAutorestartFlag(); /** - * Returns the type of service. E.g. the class-name without the - * package prefix. - */ - String getServiceType(); - - /** - * Returns the name that identifies this service for the config-sentinel. - */ - String getServiceName(); - - /** - * Returns the desired base port for this service, or '0' if this - * service should use the default port allocation mechanism. - * - * @return The desired base port for this service. - */ - int getWantedPort(); - - /** - * Returns true if the desired base port (returned by - * getWantedPort()) for this service is the only allowed base - * port. - * - * @return true if this Service requires the wanted base port. - */ - boolean requiresWantedPort(); - - /** - * Returns the number of ports needed by this service. - */ - int getPortCount(); - - /** * Returns a PortsMeta object, giving access to more information * about the different ports of this service. */ diff --git a/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java b/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java index 20089dc3980..d954c69d144 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java @@ -172,7 +172,6 @@ public final class VespaModel extends AbstractConfigProducerRoot implements Seri if (complete) { // create a a completed, frozen model configModelRepo.readConfigModels(deployState, this, builder, root, configModelRegistry); addServiceClusters(deployState, builder); - this.allocatedHosts = AllocatedHosts.withHosts(hostSystem.getHostSpecs()); // must happen after the two lines above setupRouting(deployState); this.fileDistributor = root.getFileDistributionConfigProducer().getFileDistributor(); getAdmin().addPerHostServices(hostSystem.getHosts(), deployState); @@ -180,6 +179,9 @@ public final class VespaModel extends AbstractConfigProducerRoot implements Seri root.prepare(configModelRepo); configModelRepo.prepareConfigModels(deployState); validateWrapExceptions(); + hostSystem.dumpPortAllocations(); + // must happen after stuff above + this.allocatedHosts = AllocatedHosts.withHosts(hostSystem.getHostSpecs()); } else { // create a model with no services instantiated and the given file distributor this.allocatedHosts = AllocatedHosts.withHosts(hostSystem.getHostSpecs()); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/Configserver.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/Configserver.java index 2a32549b6bf..a2839ec0fb6 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/Configserver.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/Configserver.java @@ -51,6 +51,11 @@ public class Configserver extends AbstractService { */ public int getPortCount() { return 2; } + @Override + public String[] getPortSuffixes() { + return new String[]{ "rpc", "http" }; + } + /** * The configserver is not started by the config system! */ diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/LogForwarder.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/LogForwarder.java index 2693a4c7409..d766507c75f 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/LogForwarder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/LogForwarder.java @@ -55,6 +55,11 @@ public class LogForwarder extends AbstractService implements LogforwarderConfig. */ public int getPortCount() { return 0; } + @Override + public String[] getPortSuffixes() { + return null; + } + /** * @return The command used to start LogForwarder */ diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/Logserver.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/Logserver.java index c354445b690..4dcbfb5b3c3 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/Logserver.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/Logserver.java @@ -70,4 +70,9 @@ public class Logserver extends AbstractService { return 4; } + @Override + public String[] getPortSuffixes() { + return new String[]{ "unused", "logtp", "last.errors", "replicator" }; + } + } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/Slobrok.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/Slobrok.java index 12a0d35de5e..99738c13d4a 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/Slobrok.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/Slobrok.java @@ -56,6 +56,11 @@ public class Slobrok extends AbstractService implements StateserverConfig.Produc return 2; } + @Override + public String[] getPortSuffixes() { + return new String[] { "rpc", "http" }; + } + /** * @return The port on which this slobrok should respond, as a String. */ diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java b/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java index dc962ed5931..f61fc3d4df8 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java @@ -174,7 +174,7 @@ public class Container extends AbstractService implements private void reserveHttpPortsPrepended() { if (getHttp().getHttpServer() != null) { for (ConnectorFactory connectorFactory : getHttp().getHttpServer().getConnectorFactories()) { - reservePortPrepended(getPort(connectorFactory)); + reservePortPrepended(getPort(connectorFactory), "http/" + connectorFactory.getName()); } } } @@ -240,6 +240,34 @@ public class Container extends AbstractService implements return httpPorts + rpcPorts; } + @Override + public String[] getPortSuffixes() { + // TODO clean up this mess + int n = getPortCount(); + String[] suffixes = new String[n]; + int off = 0; + int httpPorts = (getHttp() != null) ? 0 : numHttpServerPorts; + if (httpPorts > 0) { + suffixes[off++] = "http"; + } + for (int i = 1; i < httpPorts; i++) { + suffixes[off++] = "http/" + i; + } + int rpcPorts = (rpcServerEnabled()) ? numRpcServerPorts : 0; + if (rpcPorts > 0) { + suffixes[off++] = "messaging"; + } + if (rpcPorts > 1) { + suffixes[off++] = "rpc"; + } + while (off < n) { + suffixes[off] = "unused/" + off; + ++off; + } + assert (off == n); + return suffixes; + } + /** * @return the actual search port * TODO: Remove. Use {@link #getPortsMeta()} and check tags in conjunction with {@link #getRelativePort(int)}. diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentNode.java b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentNode.java index c75421c9636..dc9372c463b 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentNode.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentNode.java @@ -57,6 +57,10 @@ public abstract class ContentNode extends AbstractService @Override public int getPortCount() { return 3; } + @Override + public String[] getPortSuffixes() { + return new String[] { "messaging", "rpc", "http" }; + } @Override public void getConfig(StorCommunicationmanagerConfig.Builder builder) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/generic/service/Service.java b/config-model/src/main/java/com/yahoo/vespa/model/generic/service/Service.java index 9ccf5103175..0d30bade53c 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/generic/service/Service.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/generic/service/Service.java @@ -24,6 +24,11 @@ public class Service extends AbstractService { } @Override + public String[] getPortSuffixes() { + return null; + } + + @Override public String getStartupCommand() { return ((ServiceCluster) getParent()).getCommand(); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/Dispatch.java b/config-model/src/main/java/com/yahoo/vespa/model/search/Dispatch.java index b9c937b4a4c..9b4fe93d6ea 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/Dispatch.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/Dispatch.java @@ -234,4 +234,10 @@ public class Dispatch extends AbstractService implements SearchInterface, * @return the number of ports needed */ public int getPortCount() { return 3; } + + @Override + public String[] getPortSuffixes() { + return new String[]{ "rpc", "fs4", "health" }; + } + } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java index 5be51310504..934184d5972 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java @@ -160,6 +160,11 @@ public class SearchNode extends AbstractService implements return 5; } + @Override + public String[] getPortSuffixes() { + return new String[] { "rpc", "fs4", "future/4", "unused/3", "health" }; + } + /** * Returns the RPC port used by this searchnode. * diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/TransactionLogServer.java b/config-model/src/main/java/com/yahoo/vespa/model/search/TransactionLogServer.java index 61cac8afb91..c42579085a5 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/TransactionLogServer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/TransactionLogServer.java @@ -38,6 +38,11 @@ public class TransactionLogServer extends AbstractService { return 1; } + @Override + public String[] getPortSuffixes() { + return new String[]{"tls"}; + } + /** * Returns the port used by the TLS. * diff --git a/config-model/src/test/java/com/yahoo/vespa/model/HostResourceTest.java b/config-model/src/test/java/com/yahoo/vespa/model/HostResourceTest.java index abf4ec02a3e..d16bbe72a95 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/HostResourceTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/HostResourceTest.java @@ -37,7 +37,7 @@ public class HostResourceTest { public void next_available_baseport_is_BASE_PORT_plus_one_when_one_port_has_been_reserved() { MockRoot root = new MockRoot(); HostResource host = mockHostResource(root); - host.reservePort(new TestService(1), HostResource.BASE_PORT); + host.reservePort(new TestService(1), HostResource.BASE_PORT, "foo"); assertThat(host.nextAvailableBaseport(1), is(HostResource.BASE_PORT + 1)); } @@ -47,12 +47,12 @@ public class HostResourceTest { HostResource host = mockHostResource(root); for (int p = HostResource.BASE_PORT; p < HostResource.BASE_PORT + HostResource.MAX_PORTS; p += 2) { - host.reservePort(new TestService(1), p); + host.reservePort(new TestService(1), p, "foo"); } assertThat(host.nextAvailableBaseport(2), is(0)); try { - host.reservePort(new TestService(2), HostResource.BASE_PORT); + host.reservePort(new TestService(2), HostResource.BASE_PORT, "bar"); } catch (RuntimeException e) { assertThat(e.getMessage(), containsString("Too many ports are reserved")); } @@ -181,5 +181,14 @@ public class HostResourceTest { @Override public int getPortCount() { return portCount; } + + @Override + public String[] getPortSuffixes() { + String[] suffixes = new String[portCount]; + for (int i = 0; i < portCount; i++) { + suffixes[i] = "generic." + i; + } + return suffixes; + } } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ConfigValueChangeValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ConfigValueChangeValidatorTest.java index 2456113f40d..f13f53e8648 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ConfigValueChangeValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ConfigValueChangeValidatorTest.java @@ -246,6 +246,9 @@ public class ConfigValueChangeValidatorTest { public int getPortCount() { return 0; } + + @Override + public String[] getPortSuffixes() { return null; } } private static class SimpleConfigProducer extends AbstractConfigProducer<AbstractConfigProducer<?>> diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/StartupCommandChangeValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/StartupCommandChangeValidatorTest.java index 4f6a1ddf7b3..2b04b026ee7 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/StartupCommandChangeValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/StartupCommandChangeValidatorTest.java @@ -75,5 +75,8 @@ public class StartupCommandChangeValidatorTest { public int getPortCount() { return 0; } + + @Override + public String[] getPortSuffixes() { return null; } } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/test/ApiService.java b/config-model/src/test/java/com/yahoo/vespa/model/test/ApiService.java index 28c56e1e45f..6f06eb3e482 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/test/ApiService.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/test/ApiService.java @@ -36,4 +36,6 @@ public class ApiService extends AbstractService implements com.yahoo.test.Standa public int getPortCount() { return 0; } + @Override + public String[] getPortSuffixes() { return null; } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/test/ModelAmendingTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/test/ModelAmendingTestCase.java index 57b0606457d..82da14f0d29 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/test/ModelAmendingTestCase.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/test/ModelAmendingTestCase.java @@ -129,6 +129,8 @@ public class ModelAmendingTestCase { return 0; } + @Override + public String[] getPortSuffixes() { return null; } } public static class AdminModelAmender extends ConfigModel { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/test/ParentService.java b/config-model/src/test/java/com/yahoo/vespa/model/test/ParentService.java index 325cc78a361..c7559c68592 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/test/ParentService.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/test/ParentService.java @@ -58,4 +58,7 @@ public class ParentService extends AbstractService implements com.yahoo.test.Sta } public int getPortCount() { return 0; } + + @Override + public String[] getPortSuffixes() { return null; } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/test/SimpleService.java b/config-model/src/test/java/com/yahoo/vespa/model/test/SimpleService.java index 0037e40a20f..a38916463c4 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/test/SimpleService.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/test/SimpleService.java @@ -38,6 +38,11 @@ public class SimpleService extends AbstractService implements com.yahoo.test.Sta public int getWantedPort(){ return 10000; } public int getPortCount() { return 5; } + @Override + public String[] getPortSuffixes() { + return new String[]{ "a", "b", "c", "d", "e" }; + } + // Make sure this service is listed in the sentinel config public String getStartupCommand() { return "sleep 0"; } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/utils/FileSenderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/utils/FileSenderTest.java index 9b5bd71274a..79646dacaa9 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/utils/FileSenderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/utils/FileSenderTest.java @@ -172,5 +172,8 @@ public class FileSenderTest { public int getPortCount() { return 0; } + + @Override + public String[] getPortSuffixes() { return null; } } } diff --git a/config-provisioning/abi-spec.json b/config-provisioning/abi-spec.json index 04e4a7276e2..af61fb46e50 100644 --- a/config-provisioning/abi-spec.json +++ b/config-provisioning/abi-spec.json @@ -462,11 +462,13 @@ "public void <init>(java.lang.String, java.util.List, com.yahoo.config.provision.ClusterMembership)", "public void <init>(java.lang.String, java.util.List, java.util.Optional, java.util.Optional)", "public void <init>(java.lang.String, java.util.List, java.util.Optional, java.util.Optional, java.util.Optional)", + "public void <init>(java.lang.String, java.util.List, java.util.Optional, java.util.Optional, java.util.Optional, java.util.Optional)", "public java.lang.String hostname()", "public java.util.List aliases()", "public java.util.Optional flavor()", "public java.util.Optional version()", "public java.util.Optional membership()", + "public java.util.Optional networkPorts()", "public java.lang.String toString()", "public boolean equals(java.lang.Object)", "public int hashCode()", @@ -497,6 +499,50 @@ ], "fields": [] }, + "com.yahoo.config.provision.NetworkPorts$Allocation": { + "superClass": "java.lang.Object", + "interfaces": [], + "attributes": [ + "public" + ], + "methods": [ + "public void <init>(int, java.lang.String, java.lang.String, java.lang.String)", + "public java.lang.String key()", + "public java.lang.String toString()" + ], + "fields": [ + "public final int port", + "public final java.lang.String serviceType", + "public final java.lang.String configId", + "public final java.lang.String portSuffix" + ] + }, + "com.yahoo.config.provision.NetworkPorts": { + "superClass": "java.lang.Object", + "interfaces": [], + "attributes": [ + "public" + ], + "methods": [ + "public void <init>(java.util.Collection)", + "public java.util.Collection allocations()", + "public int size()" + ], + "fields": [] + }, + "com.yahoo.config.provision.NetworkPortsSerializer": { + "superClass": "java.lang.Object", + "interfaces": [], + "attributes": [ + "public" + ], + "methods": [ + "public void <init>()", + "public static void toSlime(com.yahoo.config.provision.NetworkPorts, com.yahoo.slime.Cursor)", + "public static java.util.Optional fromSlime(com.yahoo.slime.Inspector)" + ], + "fields": [] + }, "com.yahoo.config.provision.NodeFlavors": { "superClass": "java.lang.Object", "interfaces": [], diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/AllocatedHosts.java b/config-provisioning/src/main/java/com/yahoo/config/provision/AllocatedHosts.java index 4c1798c549f..28c5d475e19 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/AllocatedHosts.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/AllocatedHosts.java @@ -35,6 +35,7 @@ public class AllocatedHosts { /** Current version */ private static final String hostSpecCurrentVespaVersion = "currentVespaVersion"; + private static final String hostSpecNetworkPorts = "ports"; private final ImmutableSet<HostSpec> hosts; @@ -60,6 +61,7 @@ public class AllocatedHosts { }); host.flavor().ifPresent(flavor -> cursor.setString(hostSpecFlavor, flavor.name())); host.version().ifPresent(version -> cursor.setString(hostSpecCurrentVespaVersion, version.toFullString())); + host.networkPorts().ifPresent(ports -> NetworkPortsSerializer.toSlime(ports, cursor.setArray(hostSpecNetworkPorts))); } /** Returns the hosts of this allocation */ @@ -84,8 +86,9 @@ public class AllocatedHosts { object.field(hostSpecFlavor).valid() ? flavorFromSlime(object, nodeFlavors) : Optional.empty(); Optional<com.yahoo.component.Version> version = optionalString(object.field(hostSpecCurrentVespaVersion)).map(com.yahoo.component.Version::new); - - return new HostSpec(object.field(hostSpecHostName).asString(), Collections.emptyList(), flavor, membership, version); + Optional<NetworkPorts> networkPorts = + NetworkPortsSerializer.fromSlime(object.field(hostSpecNetworkPorts)); + return new HostSpec(object.field(hostSpecHostName).asString(), Collections.emptyList(), flavor, membership, version, networkPorts); } private static ClusterMembership membershipFromSlime(Inspector object) { diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/HostSpec.java b/config-provisioning/src/main/java/com/yahoo/config/provision/HostSpec.java index f0e8774759d..e5d4aadb988 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/HostSpec.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/HostSpec.java @@ -29,6 +29,8 @@ public class HostSpec implements Comparable<HostSpec> { private final Optional<com.yahoo.component.Version> version; + private final Optional<NetworkPorts> networkPorts; + public HostSpec(String hostname, Optional<ClusterMembership> membership) { this(hostname, new ArrayList<>(), Optional.empty(), membership); } @@ -40,6 +42,7 @@ public class HostSpec implements Comparable<HostSpec> { public HostSpec(String hostname, List<String> aliases) { this(hostname, aliases, Optional.empty(), Optional.empty()); } + public HostSpec(String hostname, List<String> aliases, Flavor flavor) { this(hostname, aliases, Optional.of(flavor), Optional.empty()); } @@ -54,13 +57,21 @@ public class HostSpec implements Comparable<HostSpec> { public HostSpec(String hostname, List<String> aliases, Optional<Flavor> flavor, Optional<ClusterMembership> membership, Optional<com.yahoo.component.Version> version) { + this(hostname, aliases, flavor, membership, version, Optional.empty()); + } + + public HostSpec(String hostname, List<String> aliases, Optional<Flavor> flavor, + Optional<ClusterMembership> membership, Optional<com.yahoo.component.Version> version, + Optional<NetworkPorts> networkPorts) { if (hostname == null || hostname.isEmpty()) throw new IllegalArgumentException("Hostname must be specified"); Objects.requireNonNull(version, "Version cannot be null but can be empty"); + Objects.requireNonNull(networkPorts, "Network ports cannot be null but can be empty"); this.hostname = hostname; this.aliases = ImmutableList.copyOf(aliases); this.flavor = flavor; this.membership = membership; this.version = version; + this.networkPorts = networkPorts; } /** Returns the name identifying this host */ @@ -77,6 +88,9 @@ public class HostSpec implements Comparable<HostSpec> { /** Returns the membership of this host, or an empty value if not present */ public Optional<ClusterMembership> membership() { return membership; } + /** Returns the network port allocations on this host, or empty if not present */ + public Optional<NetworkPorts> networkPorts() { return networkPorts; } + @Override public String toString() { return hostname + diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/NetworkPorts.java b/config-provisioning/src/main/java/com/yahoo/config/provision/NetworkPorts.java new file mode 100644 index 00000000000..90ac3651bb2 --- /dev/null +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/NetworkPorts.java @@ -0,0 +1,55 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.config.provision; + +import java.util.Collection; +import java.util.List; + +/** + * Models an immutable list of network port allocations + * @author arnej + */ +public class NetworkPorts { + + public static class Allocation { + public final int port; + public final String serviceType; + public final String configId; + public final String portSuffix; + + public Allocation(int port, String serviceType, String configId, String portSuffix) { + this.port = port; + this.serviceType = serviceType; + this.configId = configId; + this.portSuffix = portSuffix; + } + public String key() { + StringBuilder buf = new StringBuilder(); + buf.append("t=").append(serviceType); + buf.append(" cfg=").append(configId); + buf.append(" suf=").append(portSuffix); + return buf.toString(); + } + public String toString() { + StringBuilder buf = new StringBuilder(); + buf.append("[port=").append(port); + buf.append(" serviceType=").append(serviceType); + buf.append(" configId=").append(configId); + buf.append(" suffix=").append(portSuffix); + buf.append("]"); + return buf.toString(); + } + } + + private final List<Allocation> allocations; + + public NetworkPorts(Collection<Allocation> allocations) { + this.allocations = List.copyOf(allocations); + } + + public Collection<Allocation> allocations() { + return this.allocations; + } + + public int size() { return allocations.size(); } +} diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/NetworkPortsSerializer.java b/config-provisioning/src/main/java/com/yahoo/config/provision/NetworkPortsSerializer.java new file mode 100644 index 00000000000..d3af337e9be --- /dev/null +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/NetworkPortsSerializer.java @@ -0,0 +1,56 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.config.provision; + +import com.yahoo.slime.ArrayTraverser; +import com.yahoo.slime.Cursor; +import com.yahoo.slime.Inspector; + +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; + +/** + * Serializes network port allocations to/from JSON. + * + * @author arnej + */ +public class NetworkPortsSerializer { + + // Network port fields + private static final String portNumberKey = "port"; + private static final String serviceTypeKey = "type"; + private static final String configIdKey = "cfg"; + private static final String portSuffixKey = "suf"; + + // ---------------- Serialization ---------------------------------------------------- + + public static void toSlime(NetworkPorts networkPorts, Cursor array) { + for (NetworkPorts.Allocation allocation : networkPorts.allocations()) { + Cursor object = array.addObject(); + object.setLong(portNumberKey, allocation.port); + object.setString(serviceTypeKey, allocation.serviceType); + object.setString(configIdKey, allocation.configId); + object.setString(portSuffixKey, allocation.portSuffix); + } + } + + // ---------------- Deserialization -------------------------------------------------- + + public static Optional<NetworkPorts> fromSlime(Inspector array) { + List<NetworkPorts.Allocation> list = new ArrayList<>(array.entries()); + array.traverse((ArrayTraverser) (int i, Inspector item) -> { + list.add(new NetworkPorts.Allocation((int)item.field(portNumberKey).asLong(), + item.field(serviceTypeKey).asString(), + item.field(configIdKey).asString(), + item.field(portSuffixKey).asString())); + } + ); + if (list.size() > 0) { + NetworkPorts allocator = new NetworkPorts(list); + return Optional.of(allocator); + } + return Optional.empty(); + } + +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/provision/ProvisionerAdapter.java b/configserver/src/main/java/com/yahoo/vespa/config/server/provision/ProvisionerAdapter.java index 32380b296dd..ee4cc4a3043 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/provision/ProvisionerAdapter.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/provision/ProvisionerAdapter.java @@ -8,6 +8,7 @@ import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.HostSpec; import com.yahoo.config.provision.ProvisionLogger; import com.yahoo.config.provision.Provisioner; +import com.yahoo.config.provision.NetworkPorts; import java.util.*; diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/session/LocalSessionTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/session/LocalSessionTest.java index c4b3e5f24dc..10ee1a22bab 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/session/LocalSessionTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/session/LocalSessionTest.java @@ -7,6 +7,7 @@ import com.yahoo.config.application.api.ApplicationFile; import com.yahoo.config.provision.AllocatedHosts; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.HostSpec; +import com.yahoo.config.provision.NetworkPorts; import com.yahoo.config.provision.TenantName; import com.yahoo.path.Path; import com.yahoo.config.model.application.provider.*; @@ -136,7 +137,15 @@ public class LocalSessionTest { @Test public void require_that_provision_info_can_be_read() throws Exception { - AllocatedHosts input = AllocatedHosts.withHosts(Collections.singleton(new HostSpec("myhost", Collections.<String>emptyList()))); + List<NetworkPorts.Allocation> list = new ArrayList<>(); + list.add(new NetworkPorts.Allocation(8080, "container", "default/0", "http")); + list.add(new NetworkPorts.Allocation(19101, "searchnode", "other/1", "rpc")); + NetworkPorts ports = new NetworkPorts(list); + + AllocatedHosts input = AllocatedHosts.withHosts(Collections.singleton( + new HostSpec("myhost", Collections.<String>emptyList(), + Optional.empty(), Optional.empty(), Optional.empty(), + Optional.of(ports)))); LocalSession session = createSession(TenantName.defaultName(), 3, new SessionTest.MockSessionPreparer(), Optional.of(input)); ApplicationId origId = new ApplicationId.Builder() @@ -147,6 +156,9 @@ public class LocalSessionTest { assertNotNull(info); assertThat(info.getHosts().size(), is(1)); assertTrue(info.getHosts().contains(new HostSpec("myhost", Collections.emptyList()))); + Optional<NetworkPorts> portsCopy = info.getHosts().iterator().next().networkPorts(); + assertTrue(portsCopy.isPresent()); + assertThat(portsCopy.get().allocations(), is(list)); } @Test diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java index 668795f362b..4fef3d8ebf7 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java @@ -5,6 +5,7 @@ import com.google.common.collect.ImmutableSet; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterMembership; import com.yahoo.config.provision.Flavor; +import com.yahoo.config.provision.NetworkPorts; import com.yahoo.config.provision.NodeType; import com.yahoo.vespa.hosted.provision.node.Agent; import com.yahoo.vespa.hosted.provision.node.Allocation; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Allocation.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Allocation.java index 8a331209efc..53e1ae3721e 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Allocation.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Allocation.java @@ -3,6 +3,9 @@ package com.yahoo.vespa.hosted.provision.node; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterMembership; +import com.yahoo.config.provision.NetworkPorts; + +import java.util.Optional; /** * The allocation of a node @@ -24,12 +27,21 @@ public class Allocation { /** This node can (and should) be removed from the cluster on the next deployment */ private final boolean removable; + private final Optional<NetworkPorts> networkPorts; + + public Allocation(ApplicationId owner, ClusterMembership clusterMembership, Generation restartGeneration, boolean removable) { + this(owner, clusterMembership, restartGeneration, removable, Optional.empty()); + } + + public Allocation(ApplicationId owner, ClusterMembership clusterMembership, + Generation restartGeneration, boolean removable, Optional<NetworkPorts> networkPorts) { this.owner = owner; this.clusterMembership = clusterMembership; this.restartGeneration = restartGeneration; this.removable = removable; + this.networkPorts = networkPorts; } /** Returns the id of the application this is allocated to */ @@ -41,14 +53,17 @@ public class Allocation { /** Returns the restart generation (wanted and current) of this */ public Generation restartGeneration() { return restartGeneration; } + /** Returns network ports allocations (or empty if not recorded) */ + public Optional<NetworkPorts> networkPorts() { return networkPorts; } + /** Returns a copy of this which is retired */ public Allocation retire() { - return new Allocation(owner, clusterMembership.retire(), restartGeneration, removable); + return new Allocation(owner, clusterMembership.retire(), restartGeneration, removable, networkPorts); } /** Returns a copy of this which is not retired */ public Allocation unretire() { - return new Allocation(owner, clusterMembership.unretire(), restartGeneration, removable); + return new Allocation(owner, clusterMembership.unretire(), restartGeneration, removable, networkPorts); } /** Return whether this node is ready to be removed from the application */ @@ -56,16 +71,20 @@ public class Allocation { /** Returns a copy of this with the current restart generation set to generation */ public Allocation withRestart(Generation generation) { - return new Allocation(owner, clusterMembership, generation, removable); + return new Allocation(owner, clusterMembership, generation, removable, networkPorts); } /** Returns a copy of this allocation where removable is set to true */ public Allocation removable() { - return new Allocation(owner, clusterMembership, restartGeneration, true); + return new Allocation(owner, clusterMembership, restartGeneration, true, networkPorts); } public Allocation with(ClusterMembership newMembership) { - return new Allocation(owner, newMembership, restartGeneration, removable); + return new Allocation(owner, newMembership, restartGeneration, removable, networkPorts); + } + + public Allocation withNetworkPorts(NetworkPorts ports) { + return new Allocation(owner, clusterMembership, restartGeneration, removable, Optional.of(ports)); } @Override diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java index 54668c4eda1..bb4dab3b97b 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java @@ -10,6 +10,8 @@ import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.NodeFlavors; import com.yahoo.config.provision.NodeType; +import com.yahoo.config.provision.NetworkPorts; +import com.yahoo.config.provision.NetworkPortsSerializer; import com.yahoo.config.provision.TenantName; import com.yahoo.slime.ArrayTraverser; import com.yahoo.slime.Cursor; @@ -85,6 +87,9 @@ public class NodeSerializer { private static final String atKey = "at"; private static final String agentKey = "agent"; // retired events only + // Network port fields + private static final String networkPortsKey = "networkPorts"; + // ---------------- Serialization ---------------------------------------------------- public NodeSerializer(NodeFlavors flavors) { @@ -136,6 +141,7 @@ public class NodeSerializer { object.setLong(currentRestartGenerationKey, allocation.restartGeneration().current()); object.setBool(removableKey, allocation.isRemovable()); object.setString(wantedVespaVersionKey, allocation.membership().cluster().vespaVersion().toString()); + allocation.networkPorts().ifPresent(ports -> NetworkPortsSerializer.toSlime(ports, object.setArray(networkPortsKey))); } private void toSlime(History history, Cursor array) { @@ -197,7 +203,8 @@ public class NodeSerializer { return Optional.of(new Allocation(applicationIdFromSlime(object), clusterMembershipFromSlime(object), generationFromSlime(object, restartGenerationKey, currentRestartGenerationKey), - object.field(removableKey).asBool())); + object.field(removableKey).asBool(), + NetworkPortsSerializer.fromSlime(object.field(networkPortsKey)))); } private ApplicationId applicationIdFromSlime(Inspector object) { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Activator.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Activator.java index f48f0c1bdce..4626a600d2c 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Activator.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Activator.java @@ -9,6 +9,7 @@ import com.yahoo.transaction.NestedTransaction; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeList; import com.yahoo.vespa.hosted.provision.NodeRepository; +import com.yahoo.vespa.hosted.provision.node.Allocation; import java.util.ArrayList; import java.util.Collection; @@ -73,7 +74,7 @@ class Activator { activeToRemove = activeToRemove.stream().map(Node::unretire).collect(Collectors.toList()); // only active nodes can be retired nodeRepository.deactivate(activeToRemove, transaction); nodeRepository.activate(updateFrom(hosts, continuedActive), transaction); // update active with any changes - nodeRepository.activate(reservedToActivate, transaction); + nodeRepository.activate(updatePortsFrom(hosts, reservedToActivate), transaction); } } @@ -133,7 +134,11 @@ class Activator { for (Node node : nodes) { HostSpec hostSpec = getHost(node.hostname(), hosts); node = hostSpec.membership().get().retired() ? node.retire(nodeRepository.clock().instant()) : node.unretire(); - node = node.with(node.allocation().get().with(hostSpec.membership().get())); + Allocation allocation = node.allocation().get().with(hostSpec.membership().get()); + if (hostSpec.networkPorts().isPresent()) { + allocation = allocation.withNetworkPorts(hostSpec.networkPorts().get()); + } + node = node.with(allocation); if (hostSpec.flavor().isPresent()) // Docker nodes may change flavor node = node.with(hostSpec.flavor().get()); updated.add(node); @@ -141,6 +146,23 @@ class Activator { return updated; } + /** + * Returns the input nodes with any port allocations from the hosts + */ + private List<Node> updatePortsFrom(Collection<HostSpec> hosts, List<Node> nodes) { + List<Node> updated = new ArrayList<>(); + for (Node node : nodes) { + HostSpec hostSpec = getHost(node.hostname(), hosts); + Allocation allocation = node.allocation().get(); + if (hostSpec.networkPorts().isPresent()) { + allocation = allocation.withNetworkPorts(hostSpec.networkPorts().get()); + node = node.with(allocation); + } + updated.add(node); + } + return updated; + } + private HostSpec getHost(String hostname, Collection<HostSpec> fromHosts) { for (HostSpec host : fromHosts) if (host.hostname().equals(hostname)) diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java index a0d76241533..246c56ee28b 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java @@ -22,11 +22,13 @@ import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.flag.FlagId; +import com.yahoo.vespa.hosted.provision.node.Allocation; import com.yahoo.vespa.hosted.provision.node.filter.ApplicationFilter; import com.yahoo.vespa.hosted.provision.node.filter.NodeHostFilter; import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; import java.util.Comparator; import java.util.List; import java.util.Optional; @@ -141,10 +143,16 @@ public class NodeRepositoryProvisioner implements Provisioner { List<HostSpec> hosts = new ArrayList<>(nodes.size()); for (Node node : nodes) { log.log(LogLevel.DEBUG, () -> "Prepared node " + node.hostname() + " - " + node.flavor()); + Allocation nodeAllocation = node.allocation().orElseThrow(IllegalStateException::new); hosts.add(new HostSpec(node.hostname(), - node.allocation().orElseThrow(IllegalStateException::new).membership(), - node.flavor(), - node.status().vespaVersion())); + Collections.emptyList(), + Optional.of(node.flavor()), + Optional.of(nodeAllocation.membership()), + node.status().vespaVersion(), + nodeAllocation.networkPorts())); + if (nodeAllocation.networkPorts().isPresent()) { + log.log(LogLevel.DEBUG, () -> "Prepared node " + node.hostname() + " has port allocations"); + } } return hosts; } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesResponse.java index ba513db5342..1254664eb78 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesResponse.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesResponse.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.hosted.provision.restapi.v2; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterMembership; +import com.yahoo.config.provision.NetworkPortsSerializer; import com.yahoo.config.provision.NodeType; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; @@ -153,17 +154,18 @@ class NodesResponse extends HttpResponse { object.setBool("fastDisk", node.flavor().hasFastDisk()); object.setDouble("bandwidth", node.flavor().getBandwidth()); object.setString("environment", node.flavor().getType().name()); - if (node.allocation().isPresent()) { - toSlime(node.allocation().get().owner(), object.setObject("owner")); - toSlime(node.allocation().get().membership(), object.setObject("membership")); - object.setLong("restartGeneration", node.allocation().get().restartGeneration().wanted()); - object.setLong("currentRestartGeneration", node.allocation().get().restartGeneration().current()); - object.setString("wantedDockerImage", nodeRepository.dockerImage().withTag(node.allocation().get().membership().cluster().vespaVersion()).asString()); - object.setString("wantedVespaVersion", node.allocation().get().membership().cluster().vespaVersion().toFullString()); + node.allocation().ifPresent(allocation -> { + toSlime(allocation.owner(), object.setObject("owner")); + toSlime(allocation.membership(), object.setObject("membership")); + object.setLong("restartGeneration", allocation.restartGeneration().wanted()); + object.setLong("currentRestartGeneration", allocation.restartGeneration().current()); + object.setString("wantedDockerImage", nodeRepository.dockerImage().withTag(allocation.membership().cluster().vespaVersion()).asString()); + object.setString("wantedVespaVersion", allocation.membership().cluster().vespaVersion().toFullString()); + allocation.networkPorts().ifPresent(ports -> NetworkPortsSerializer.toSlime(ports, object.setArray("networkPorts"))); orchestrator.apply(new HostName(node.hostname())) .map(status -> status == HostStatus.ALLOWED_TO_BE_DOWN) .ifPresent(allowedToBeDown -> object.setBool("allowedToBeDown", allowedToBeDown)); - } + }); object.setLong("rebootGeneration", node.status().reboot().wanted()); object.setLong("currentRebootGeneration", node.status().reboot().current()); node.status().osVersion().ifPresent(version -> object.setString("currentOsVersion", version.toFullString())); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/SerializationTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/SerializationTest.java index 29229efc662..53f6b745da1 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/SerializationTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/SerializationTest.java @@ -8,6 +8,7 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.ClusterMembership; import com.yahoo.config.provision.InstanceName; +import com.yahoo.config.provision.NetworkPorts; import com.yahoo.config.provision.NodeFlavors; import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.TenantName; @@ -29,8 +30,11 @@ import org.junit.Test; import java.nio.charset.StandardCharsets; import java.time.Duration; import java.time.Instant; +import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; import java.util.Collections; +import java.util.List; import java.util.Optional; import java.util.stream.Collectors; @@ -371,6 +375,31 @@ public class SerializationTest { assertEquals("some model", node.modelName().get()); } + @Test + public void testNodeWithNetworkPorts() { + Node node = createNode(); + List<NetworkPorts.Allocation> list = new ArrayList<>(); + list.add(new NetworkPorts.Allocation(8080, "container", "default/0", "http")); + list.add(new NetworkPorts.Allocation(19101, "searchnode", "other/1", "rpc")); + NetworkPorts ports = new NetworkPorts(list); + node = node.allocate(ApplicationId.from(TenantName.from("myTenant"), + ApplicationName.from("myApplication"), + InstanceName.from("myInstance")), + ClusterMembership.from("content/myId/0/0", Vtag.currentVersion), + clock.instant()); + assertTrue(node.allocation().isPresent()); + node = node.with(node.allocation().get().withNetworkPorts(ports)); + assertTrue(node.allocation().isPresent()); + assertTrue(node.allocation().get().networkPorts().isPresent()); + Node copy = nodeSerializer.fromJson(Node.State.provisioned, nodeSerializer.toJson(node)); + assertTrue(copy.allocation().isPresent()); + assertTrue(copy.allocation().get().networkPorts().isPresent()); + NetworkPorts portsCopy = node.allocation().get().networkPorts().get(); + Collection<NetworkPorts.Allocation> listCopy = portsCopy.allocations(); + assertEquals(list, listCopy); + } + + private byte[] createNodeJson(String hostname, String... ipAddress) { String ipAddressJsonPart = ""; if (ipAddress.length > 0) { |