diff options
31 files changed, 191 insertions, 234 deletions
diff --git a/application-model/pom.xml b/application-model/pom.xml index 3abf9851d5c..7eac247e249 100644 --- a/application-model/pom.xml +++ b/application-model/pom.xml @@ -23,6 +23,12 @@ </dependency> <dependency> <groupId>com.yahoo.vespa</groupId> + <artifactId>vespajlib</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> <artifactId>config-provisioning</artifactId> <version>${project.version}</version> <scope>provided</scope> diff --git a/application-model/src/main/java/com/yahoo/vespa/applicationmodel/TenantId.java b/application-model/src/main/java/com/yahoo/vespa/applicationmodel/TenantId.java index 7d94a0418e4..20aca379015 100644 --- a/application-model/src/main/java/com/yahoo/vespa/applicationmodel/TenantId.java +++ b/application-model/src/main/java/com/yahoo/vespa/applicationmodel/TenantId.java @@ -31,8 +31,6 @@ public class TenantId { return id; } - public TenantName toName() { return TenantName.from(id); } - @Override public boolean equals(Object o) { if (this == o) return true; diff --git a/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java b/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java index 824bf248b5c..1838f1e36b7 100644 --- a/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java +++ b/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java @@ -99,7 +99,7 @@ public class MockApplicationPackage implements ApplicationPackage { @SuppressWarnings("deprecation") // not redundant @Override public String getApplicationName() { - return "mock application"; + return "mock-application"; } @Override diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/ApplicationId.java b/config-provisioning/src/main/java/com/yahoo/config/provision/ApplicationId.java index 76d4ae3f1a6..aa70bf4d26a 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/ApplicationId.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/ApplicationId.java @@ -3,6 +3,10 @@ package com.yahoo.config.provision; import com.yahoo.cloud.config.ApplicationIdConfig; +import java.util.Comparator; +import java.util.Objects; +import java.util.regex.Pattern; + /** * A complete, immutable identification of an application instance. * @@ -10,27 +14,40 @@ import com.yahoo.cloud.config.ApplicationIdConfig; * @author vegard * @author bratseth */ -public final class ApplicationId implements Comparable<ApplicationId> { +public class ApplicationId implements Comparable<ApplicationId> { + + // TODO: remove '.' and '*' from this pattern. + static final Pattern namePattern = Pattern.compile("(?!\\.\\.)[a-zA-Z0-9_.*-]{1,256}"); + + private static final ApplicationId global = new ApplicationId(TenantName.from("*"), + ApplicationName.from("*"), + InstanceName.from("*")) { + @Override public boolean equals(Object other) { return this == other; } + }; + + private static final Comparator<ApplicationId> comparator = Comparator.comparing(ApplicationId::tenant) + .thenComparing(ApplicationId::application) + .thenComparing(ApplicationId::instance) + .thenComparing(global::equals, Boolean::compare); private final TenantName tenant; private final ApplicationName application; private final InstanceName instance; - - private final String stringValue; private final String serializedForm; - public ApplicationId(ApplicationIdConfig config) { - this(TenantName.from(config.tenant()), ApplicationName.from(config.application()), InstanceName.from(config.instance())); - } - private ApplicationId(TenantName tenant, ApplicationName applicationName, InstanceName instanceName) { this.tenant = tenant; this.application = applicationName; this.instance = instanceName; - this.stringValue = toStringValue(); this.serializedForm = toSerializedForm(); } + public static ApplicationId from(ApplicationIdConfig config) { + return from(TenantName.from(config.tenant()), + ApplicationName.from(config.application()), + InstanceName.from(config.instance())); + } + public static ApplicationId from(TenantName tenant, ApplicationName application, InstanceName instance) { return new ApplicationId(tenant, application, instance); } @@ -44,7 +61,7 @@ public final class ApplicationId implements Comparable<ApplicationId> { if (parts.length < 3) throw new IllegalArgumentException("Application ids must be on the form tenant:application:instance, but was " + idString); - return new Builder().tenant(parts[0]).applicationName(parts[1]).instanceName(parts[2]).build(); + return from(parts[0], parts[1], parts[2]); } public static ApplicationId fromFullString(String idString) { @@ -52,11 +69,11 @@ public final class ApplicationId implements Comparable<ApplicationId> { if (parts.length < 3) throw new IllegalArgumentException("Application ids must be on the form tenant.application.instance, but was " + idString); - return new Builder().tenant(parts[0]).applicationName(parts[1]).instanceName(parts[2]).build(); + return from(parts[0], parts[1], parts[2]); } @Override - public int hashCode() { return stringValue.hashCode(); } + public int hashCode() { return Objects.hash(tenant, application, instance); } @Override public boolean equals(Object other) { @@ -72,10 +89,6 @@ public final class ApplicationId implements Comparable<ApplicationId> { /** Returns a serialized form of the content of this: tenant:application:instance */ public String serializedForm() { return serializedForm; } - private String toStringValue() { - return "tenant '" + tenant + "', application '" + application + "', instance '" + instance + "'"; - } - /** Returns "dotted" string (tenant.application.instance) with instance name omitted if it is "default" */ public String toShortString() { return tenant().value() + "." + application().value() + @@ -88,7 +101,7 @@ public final class ApplicationId implements Comparable<ApplicationId> { } private String toSerializedForm() { - return tenant + ":" + application + ":" + instance; + return tenant.value() + ":" + application.value() + ":" + instance.value(); } @Override @@ -100,18 +113,7 @@ public final class ApplicationId implements Comparable<ApplicationId> { @Override public int compareTo(ApplicationId other) { - int diff; - - diff = tenant.compareTo(other.tenant); - if (diff != 0) { return diff; } - - diff = application.compareTo(other.application); - if (diff != 0) { return diff; } - - diff = instance.compareTo(other.instance); - if (diff != 0) { return diff; } - - return 0; + return comparator.compare(this, other); } /** Returns an application id where all fields are "default" */ @@ -119,12 +121,10 @@ public final class ApplicationId implements Comparable<ApplicationId> { return new ApplicationId(TenantName.defaultName(), ApplicationName.defaultName(), InstanceName.defaultName()); } - /** Returns an application id where all fields are "*" */ + // TODO: kill this + /** Returns a very special application id, which is not equal to any other id. */ public static ApplicationId global() { - return new Builder().tenant("*") - .applicationName("*") - .instanceName("*") - .build(); + return global; } public static class Builder { diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/ApplicationName.java b/config-provisioning/src/main/java/com/yahoo/config/provision/ApplicationName.java index f16c126dec2..f2585913015 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/ApplicationName.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/ApplicationName.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.config.provision; -import java.util.Objects; +import ai.vespa.validation.PatternedStringWrapper; /** * Represents an applications name, which may be any kind of string or default. This type is defined @@ -10,28 +10,12 @@ import java.util.Objects; * @author Ulf Lilleengen * @since 5.25 */ -public class ApplicationName implements Comparable<ApplicationName> { +public class ApplicationName extends PatternedStringWrapper<ApplicationName> { - private final String applicationName; + private static final ApplicationName defaultName = new ApplicationName("default"); - private ApplicationName(String applicationName) { - this.applicationName = applicationName; - } - - @Override - public int hashCode() { - return applicationName.hashCode(); - } - - @Override - public boolean equals(Object obj) { - if (!(obj instanceof ApplicationName)) return false; - return Objects.equals(((ApplicationName) obj).applicationName, applicationName); - } - - @Override - public String toString() { - return applicationName; + private ApplicationName(String name) { + super(name, ApplicationId.namePattern, "application name"); } public static ApplicationName from(String name) { @@ -39,20 +23,11 @@ public class ApplicationName implements Comparable<ApplicationName> { } public static ApplicationName defaultName() { - return new ApplicationName("default"); + return defaultName; } public boolean isDefault() { - return equals(ApplicationName.defaultName()); - } - - public String value() { - return applicationName; - } - - @Override - public int compareTo(ApplicationName name) { - return this.applicationName.compareTo(name.applicationName); + return equals(defaultName); } } diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/InstanceName.java b/config-provisioning/src/main/java/com/yahoo/config/provision/InstanceName.java index 8101b70b943..fc40d351465 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/InstanceName.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/InstanceName.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.config.provision; -import java.util.Objects; +import ai.vespa.validation.PatternedStringWrapper; /** * Represents an applications instance name, which may be any kind of string or default. This type is defined @@ -9,30 +9,12 @@ import java.util.Objects; * * @author Ulf Lilleengen */ -public class InstanceName implements Comparable<InstanceName> { +public class InstanceName extends PatternedStringWrapper<InstanceName> { - private static final InstanceName defaultInstance = new InstanceName("default"); + private static final InstanceName defaultName = new InstanceName("default"); - private final String instanceName; - - private InstanceName(String instanceName) { - this.instanceName = instanceName; - } - - @Override - public int hashCode() { - return instanceName.hashCode(); - } - - @Override - public boolean equals(Object obj) { - if (!(obj instanceof InstanceName)) return false; - return Objects.equals(((InstanceName) obj).instanceName, instanceName); - } - - @Override - public String toString() { - return instanceName; + private InstanceName(String name) { + super(name, ApplicationId.namePattern, "instance name"); } public static InstanceName from(String name) { @@ -40,22 +22,15 @@ public class InstanceName implements Comparable<InstanceName> { } public static InstanceName defaultName() { - return defaultInstance; + return defaultName; } public boolean isDefault() { - return equals(InstanceName.defaultName()); + return equals(defaultName); } public boolean isTester() { return value().endsWith("-t"); } - public String value() { return instanceName; } - - @Override - public int compareTo(InstanceName instance) { - return instanceName.compareTo(instance.instanceName); - } - } diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/TenantName.java b/config-provisioning/src/main/java/com/yahoo/config/provision/TenantName.java index 92fe5345b4e..9909ab360a0 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/TenantName.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/TenantName.java @@ -1,56 +1,31 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.config.provision; -import java.util.Objects; +import ai.vespa.validation.PatternedStringWrapper; /** * Represents a tenant in the provision API. * * @author Ulf Lilleengen */ -public class TenantName implements Comparable<TenantName> { +public class TenantName extends PatternedStringWrapper<TenantName> { - private final String name; + private static final TenantName defaultName = new TenantName("default"); private TenantName(String name) { - this.name = name; + super(name, ApplicationId.namePattern, "tenant name"); } - public String value() { return name; } - - /** - * Create a {@link TenantName} with a given name. - * - * @param name Name of tenant. - * @return instance of {@link TenantName}. - */ public static TenantName from(String name) { return new TenantName(name); } - @Override - public int hashCode() { - return name.hashCode(); - } - - @Override - public boolean equals(Object obj) { - if (!(obj instanceof TenantName)) return false; - return Objects.equals(((TenantName)obj).value(), value()); - } - - @Override - public String toString() { - return name; - } - public static TenantName defaultName() { - return from("default"); + return defaultName; } - @Override - public int compareTo(TenantName that) { - return this.name.compareTo(that.name); + public boolean isDefault() { + return equals(defaultName); } } diff --git a/config-provisioning/src/test/java/com/yahoo/config/provision/ApplicationIdTest.java b/config-provisioning/src/test/java/com/yahoo/config/provision/ApplicationIdTest.java index c82230f7edf..01904b5eece 100644 --- a/config-provisioning/src/test/java/com/yahoo/config/provision/ApplicationIdTest.java +++ b/config-provisioning/src/test/java/com/yahoo/config/provision/ApplicationIdTest.java @@ -106,7 +106,7 @@ public class ApplicationIdTest { builder.tenant("a"); builder.application("b"); builder.instance("c"); - ApplicationId applicationId = new ApplicationId(new ApplicationIdConfig(builder)); + ApplicationId applicationId = ApplicationId.from(new ApplicationIdConfig(builder)); assertEquals("a", applicationId.tenant().value()); assertEquals("b", applicationId.application().value()); assertEquals("c", applicationId.instance().value()); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/DelayedConfigResponses.java b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/DelayedConfigResponses.java index d3976464bde..7e6fccb6d2f 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/DelayedConfigResponses.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/DelayedConfigResponses.java @@ -193,10 +193,9 @@ public class DelayedConfigResponses { } private synchronized void metricDelayedResponses(ApplicationId app, int elems) { - if ( ! metrics.containsKey(app)) { - metrics.put(app, rpcServer.metricUpdaterFactory().getOrCreateMetricUpdater(Metrics.createDimensions(app))); - } - metrics.get(app).setDelayedResponses(elems); + metrics.computeIfAbsent(app, key -> rpcServer.metricUpdaterFactory() + .getOrCreateMetricUpdater(Metrics.createDimensions(key))) + .setDelayedResponses(elems); } private synchronized void createQueueIfNotExists(GetConfigContext context) { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/RpcServer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/RpcServer.java index 99ffff6403b..ebf1fb32141 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/RpcServer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/RpcServer.java @@ -243,13 +243,9 @@ public class RpcServer implements Runnable, ReloadListener, TenantListener { } private ApplicationState getState(ApplicationId id) { - ApplicationState state = applicationStateMap.get(id); - if (state == null) { - applicationStateMap.putIfAbsent(id, new ApplicationState(0)); - state = applicationStateMap.get(id); - } - return state; + return applicationStateMap.computeIfAbsent(id, __ -> new ApplicationState(0)); } + boolean hasNewerGeneration(ApplicationId id, long generation) { return getState(id).getActiveGeneration() > generation; } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/NodeRepoStats.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/NodeRepoStats.java index fe7beb538da..68ebc5e86aa 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/NodeRepoStats.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/NodeRepoStats.java @@ -20,6 +20,6 @@ public class NodeRepoStats { public Load load() { return load; } public Load activeLoad() { return activeLoad; } - public List<ApplicationStats> applicationStats() { return applicationStats; } + public List<ApplicationStats> applicationStats() { return applicationStats; } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java index 158cc6caede..1a4a42cb521 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java @@ -30,6 +30,7 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.TreeMap; import java.util.logging.Level; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -132,7 +133,7 @@ public class DeploymentApiHandler extends ThreadedHttpRequestHandler { entry -> entry.getValue().instanceJobs().get(entry.getKey()))); Cursor productionArray = versionObject.setArray("productionApplications"); statistics.productionSuccesses().stream() - .collect(groupingBy(run -> run.id().application())) + .collect(groupingBy(run -> run.id().application(), TreeMap::new, toList())) .forEach((id, runs) -> { Cursor applicationObject = productionArray.addObject(); toSlime(applicationObject, id, request); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java index 640e6860eb6..a464e3d7e9b 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java @@ -264,36 +264,38 @@ public class ApplicationPackageBuilder { xml.append(athenzIdentityAttributes); } xml.append(">\n"); - xml.append(" <instance id='").append(instances).append("'>\n"); - if (upgradePolicy != null || revisionTarget != null || revisionChange != null || upgradeRollout != null) { - xml.append(" <upgrade "); - if (upgradePolicy != null) xml.append("policy='").append(upgradePolicy).append("' "); - if (revisionTarget != null) xml.append("revision-target='").append(revisionTarget).append("' "); - if (revisionChange != null) xml.append("revision-change='").append(revisionChange).append("' "); - if (upgradeRollout != null) xml.append("rollout='").append(upgradeRollout).append("' "); - xml.append("/>\n"); - } - xml.append(notifications); - if (explicitSystemTest) - xml.append(" <test />\n"); - if (explicitStagingTest) - xml.append(" <staging />\n"); - xml.append(blockChange); - xml.append(" <prod"); - if (globalServiceId != null) { - xml.append(" global-service-id='"); - xml.append(globalServiceId); - xml.append("'"); - } - xml.append(">\n"); - xml.append(prodBody); - xml.append(" </prod>\n"); - if (endpointsBody.length() > 0 ) { - xml.append(" <endpoints>\n"); - xml.append(endpointsBody); - xml.append(" </endpoints>\n"); + for (String instance : instances.split(",")) { + xml.append(" <instance id='").append(instance).append("'>\n"); + if (upgradePolicy != null || revisionTarget != null || revisionChange != null || upgradeRollout != null) { + xml.append(" <upgrade "); + if (upgradePolicy != null) xml.append("policy='").append(upgradePolicy).append("' "); + if (revisionTarget != null) xml.append("revision-target='").append(revisionTarget).append("' "); + if (revisionChange != null) xml.append("revision-change='").append(revisionChange).append("' "); + if (upgradeRollout != null) xml.append("rollout='").append(upgradeRollout).append("' "); + xml.append("/>\n"); + } + xml.append(notifications); + if (explicitSystemTest) + xml.append(" <test />\n"); + if (explicitStagingTest) + xml.append(" <staging />\n"); + xml.append(blockChange); + xml.append(" <prod"); + if (globalServiceId != null) { + xml.append(" global-service-id='"); + xml.append(globalServiceId); + xml.append("'"); + } + xml.append(">\n"); + xml.append(prodBody); + xml.append(" </prod>\n"); + if (endpointsBody.length() > 0) { + xml.append(" <endpoints>\n"); + xml.append(endpointsBody); + xml.append(" </endpoints>\n"); + } + xml.append(" </instance>\n"); } - xml.append(" </instance>\n"); if (applicationEndpointsBody.length() > 0) { xml.append(" <endpoints>\n"); xml.append(applicationEndpointsBody); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/NodeRepositoryMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/NodeRepositoryMock.java index a2a1b4ba0a1..7b67db39350 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/NodeRepositoryMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/NodeRepositoryMock.java @@ -26,6 +26,7 @@ import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Optional; +import java.util.TreeMap; import java.util.function.UnaryOperator; import java.util.stream.Collectors; @@ -224,7 +225,7 @@ public class NodeRepositoryMock implements NodeRepository { } public void putApplication(ZoneId zone, Application application) { - applications.computeIfAbsent(zone, (k) -> new HashMap<>()) + applications.computeIfAbsent(zone, (k) -> new TreeMap<>()) .put(application.id(), application); } diff --git a/fnet/src/vespa/fnet/controlpacket.cpp b/fnet/src/vespa/fnet/controlpacket.cpp index 9ff69a76210..9aa03ad5e3a 100644 --- a/fnet/src/vespa/fnet/controlpacket.cpp +++ b/fnet/src/vespa/fnet/controlpacket.cpp @@ -95,7 +95,10 @@ FNET_ControlPacket FNET_ControlPacket::IOCClose(FNET_CMD_IOC_CLOSE); FNET_ControlPacket -FNET_ControlPacket::DetachServerAdapter(FNET_CMD_DETACH_SERVER_ADAPTER); +FNET_ControlPacket::DetachServerAdapterInit(FNET_CMD_DETACH_SERVER_ADAPTER_INIT); + +FNET_ControlPacket +FNET_ControlPacket::DetachServerAdapterFini(FNET_CMD_DETACH_SERVER_ADAPTER_FINI); FNET_ControlPacket FNET_ControlPacket::Execute(FNET_CMD_EXECUTE); diff --git a/fnet/src/vespa/fnet/controlpacket.h b/fnet/src/vespa/fnet/controlpacket.h index 8dd2d034ae6..ad846d37c30 100644 --- a/fnet/src/vespa/fnet/controlpacket.h +++ b/fnet/src/vespa/fnet/controlpacket.h @@ -38,7 +38,8 @@ public: FNET_CMD_IOC_ENABLE_WRITE, FNET_CMD_IOC_HANDSHAKE_ACT, FNET_CMD_IOC_CLOSE, - FNET_CMD_DETACH_SERVER_ADAPTER, + FNET_CMD_DETACH_SERVER_ADAPTER_INIT, + FNET_CMD_DETACH_SERVER_ADAPTER_FINI, FNET_CMD_EXECUTE, FNET_CMD_TIMEOUT, FNET_CMD_BAD_PACKET, @@ -51,7 +52,8 @@ public: static FNET_ControlPacket IOCEnableWrite; static FNET_ControlPacket IOCHandshakeACT; static FNET_ControlPacket IOCClose; - static FNET_ControlPacket DetachServerAdapter; + static FNET_ControlPacket DetachServerAdapterInit; + static FNET_ControlPacket DetachServerAdapterFini; static FNET_ControlPacket Execute; static FNET_ControlPacket Timeout; static FNET_ControlPacket BadPacket; diff --git a/fnet/src/vespa/fnet/frt/supervisor.cpp b/fnet/src/vespa/fnet/frt/supervisor.cpp index 0a31b9d882b..1681321b239 100644 --- a/fnet/src/vespa/fnet/frt/supervisor.cpp +++ b/fnet/src/vespa/fnet/frt/supervisor.cpp @@ -30,12 +30,10 @@ FRT_Supervisor::FRT_Supervisor(FNET_Transport *transport) FRT_Supervisor::~FRT_Supervisor() { + _transport->detach(this); if (_connector != nullptr) { - _connector->Owner()->Close(_connector, /* needref */ false); + _connector->SubRef(); } - _transport->wait_for_pending_resolves(); - _transport->detach(this); - _transport->sync(); } FNET_Scheduler * diff --git a/fnet/src/vespa/fnet/transport.cpp b/fnet/src/vespa/fnet/transport.cpp index 0a79f324bb9..1130b6d3e5e 100644 --- a/fnet/src/vespa/fnet/transport.cpp +++ b/fnet/src/vespa/fnet/transport.cpp @@ -125,6 +125,11 @@ TransportConfig::time_tools() const { } // fnet +void +FNET_Transport::wait_for_pending_resolves() { + _async_resolver->wait_for_pending_resolves(); +} + FNET_Transport::FNET_Transport(const fnet::TransportConfig &cfg) : _async_resolver(cfg.resolver()), _crypto_engine(cfg.crypto()), @@ -158,11 +163,6 @@ FNET_Transport::resolve_async(const vespalib::string &spec, _async_resolver->resolve_async(spec, std::move(result_handler)); } -void -FNET_Transport::wait_for_pending_resolves() { - _async_resolver->wait_for_pending_resolves(); -} - vespalib::CryptoSocket::UP FNET_Transport::create_client_crypto_socket(vespalib::SocketHandle socket, const vespalib::SocketSpec &spec) { @@ -221,8 +221,14 @@ void FNET_Transport::detach(FNET_IServerAdapter *server_adapter) { for (const auto &thread: _threads) { - thread->detach(server_adapter); + thread->init_detach(server_adapter); + } + wait_for_pending_resolves(); + sync(); + for (const auto &thread: _threads) { + thread->fini_detach(server_adapter); } + sync(); } FNET_Scheduler * diff --git a/fnet/src/vespa/fnet/transport.h b/fnet/src/vespa/fnet/transport.h index 7dbfd80dfe7..d6e4aefb02b 100644 --- a/fnet/src/vespa/fnet/transport.h +++ b/fnet/src/vespa/fnet/transport.h @@ -113,6 +113,11 @@ private: Threads _threads; const FNET_Config _config; + /** + * Wait for all pending resolve requests. + **/ + void wait_for_pending_resolves(); + public: FNET_Transport(const FNET_Transport &) = delete; FNET_Transport & operator = (const FNET_Transport &) = delete; @@ -160,11 +165,6 @@ public: vespalib::AsyncResolver::ResultHandler::WP result_handler); /** - * Wait for all pending resolve requests. - **/ - void wait_for_pending_resolves(); - - /** * Wrap a plain socket endpoint (client side) in a CryptoSocket. The * implementation will be determined by the CryptoEngine used by * this Transport. @@ -258,11 +258,8 @@ public: * Detach a server adapter from this transport. * * This will close all connectors and connections referencing the - * server adapter. Note that this is an async - * operation. 'wait_for_pending_resolves' should be called before - * this to make sure any in-flight connections are added - * first. 'sync' should be called after this to drain any pending - * call-backs. + * server adapter. Note that this function will also synchronize + * with async address resolving and underlying transport threads. **/ void detach(FNET_IServerAdapter *server_adapter); diff --git a/fnet/src/vespa/fnet/transport_thread.cpp b/fnet/src/vespa/fnet/transport_thread.cpp index 53363740bae..d46d174c670 100644 --- a/fnet/src/vespa/fnet/transport_thread.cpp +++ b/fnet/src/vespa/fnet/transport_thread.cpp @@ -162,7 +162,7 @@ FNET_TransportThread::SafeDiscardEvent(FNET_ControlPacket *cpacket, void FNET_TransportThread::handle_add_cmd(FNET_IOComponent *ioc) { - if (ioc->handle_add_event()) { + if ((_detaching.count(ioc->server_adapter()) == 0) && ioc->handle_add_event()) { AddComponent(ioc); ioc->_flags._ioc_added = true; ioc->attach_selector(_selector); @@ -186,8 +186,9 @@ FNET_TransportThread::handle_close_cmd(FNET_IOComponent *ioc) void -FNET_TransportThread::handle_detach_server_adapter_cmd(FNET_IServerAdapter *server_adapter) +FNET_TransportThread::handle_detach_server_adapter_init_cmd(FNET_IServerAdapter *server_adapter) { + _detaching.insert(server_adapter); FNET_IOComponent *component = _componentsHead; while (component != nullptr) { FNET_IOComponent *tmp = component; @@ -201,6 +202,12 @@ FNET_TransportThread::handle_detach_server_adapter_cmd(FNET_IServerAdapter *serv } +void +FNET_TransportThread::handle_detach_server_adapter_fini_cmd(FNET_IServerAdapter *server_adapter) +{ + _detaching.erase(server_adapter); +} + extern "C" { static void pipehandler(int) @@ -241,7 +248,8 @@ FNET_TransportThread::FNET_TransportThread(FNET_Transport &owner_in) _pseudo_thread(), _started(false), _shutdown(false), - _finished(false) + _finished(false), + _detaching() { trapsigpipe(); } @@ -348,9 +356,15 @@ FNET_TransportThread::Close(FNET_IOComponent *comp, bool needRef) } void -FNET_TransportThread::detach(FNET_IServerAdapter *server_adapter) +FNET_TransportThread::init_detach(FNET_IServerAdapter *server_adapter) { - PostEvent(&FNET_ControlPacket::DetachServerAdapter, FNET_Context(server_adapter)); + PostEvent(&FNET_ControlPacket::DetachServerAdapterInit, FNET_Context(server_adapter)); +} + +void +FNET_TransportThread::fini_detach(FNET_IServerAdapter *server_adapter) +{ + PostEvent(&FNET_ControlPacket::DetachServerAdapterFini, FNET_Context(server_adapter)); } bool @@ -432,8 +446,13 @@ FNET_TransportThread::handle_wakeup() continue; } - if (packet->GetCommand() == FNET_ControlPacket::FNET_CMD_DETACH_SERVER_ADAPTER) { - handle_detach_server_adapter_cmd(context._value.SERVER_ADAPTER); + if (packet->GetCommand() == FNET_ControlPacket::FNET_CMD_DETACH_SERVER_ADAPTER_INIT) { + handle_detach_server_adapter_init_cmd(context._value.SERVER_ADAPTER); + continue; + } + + if (packet->GetCommand() == FNET_ControlPacket::FNET_CMD_DETACH_SERVER_ADAPTER_FINI) { + handle_detach_server_adapter_fini_cmd(context._value.SERVER_ADAPTER); continue; } diff --git a/fnet/src/vespa/fnet/transport_thread.h b/fnet/src/vespa/fnet/transport_thread.h index c120894ac9c..b507c5dc31d 100644 --- a/fnet/src/vespa/fnet/transport_thread.h +++ b/fnet/src/vespa/fnet/transport_thread.h @@ -13,6 +13,7 @@ #include <mutex> #include <condition_variable> #include <chrono> +#include <set> namespace fnet { struct TimeTools; } class FNET_Transport; @@ -51,6 +52,7 @@ private: std::atomic<bool> _started; // event loop started ? std::atomic<bool> _shutdown; // should stop event loop ? std::atomic<bool> _finished; // event loop stopped ? + std::set<FNET_IServerAdapter*> _detaching; // server adapters being detached /** * Add an IOComponent to the list of components. This operation is @@ -143,7 +145,8 @@ private: void handle_add_cmd(FNET_IOComponent *ioc); void handle_close_cmd(FNET_IOComponent *ioc); - void handle_detach_server_adapter_cmd(FNET_IServerAdapter *server_adapter); + void handle_detach_server_adapter_init_cmd(FNET_IServerAdapter *server_adapter); + void handle_detach_server_adapter_fini_cmd(FNET_IServerAdapter *server_adapter); /** * This method is called to initialize the transport thread event @@ -336,16 +339,16 @@ public: void Close(FNET_IOComponent *comp, bool needRef = true); /** - * Detach a server adapter from this transport. - * - * This will close all connectors and connections referencing the - * server adapter. Note that this is an async - * operation. 'wait_for_pending_resolves' (on the owning - * Transport) should be called before this to make sure any - * in-flight connections are added first. 'sync' should be called - * after this to drain any pending call-backs. + * Start the operation of detaching a server adapter from this + * transport. + **/ + void init_detach(FNET_IServerAdapter *server_adapter); + + /** + * Complete the operation of detaching a server adapter from this + * transport. **/ - void detach(FNET_IServerAdapter *server_adapter); + void fini_detach(FNET_IServerAdapter *server_adapter); /** * Post an execution event on the transport event queue. The return diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java index 379bb2566df..a9abc352d8c 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java @@ -43,8 +43,8 @@ import java.util.logging.Logger; import java.util.stream.Collectors; import java.util.stream.IntStream; +import static com.yahoo.stream.CustomCollectors.toLinkedMap; import static java.util.stream.Collectors.collectingAndThen; -import static java.util.stream.Collectors.toMap; /** * Client which reads and writes nodes to a curator database. @@ -453,7 +453,8 @@ public class CuratorDatabaseClient { .map(this::readLoadBalancer) .filter(Optional::isPresent) .map(Optional::get) - .collect(collectingAndThen(toMap(LoadBalancer::id, Function.identity()), + .collect(collectingAndThen(toLinkedMap(LoadBalancer::id, + Function.identity()), Collections::unmodifiableMap)); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializerTest.java index cc121ba8104..48ee23c7b60 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializerTest.java @@ -334,6 +334,9 @@ public class NodeSerializerTest { " \"hostname\" : \"myHostname\",\n" + " \"ipAddresses\" : [\"127.0.0.1\"],\n" + " \"instance\": {\n" + + " \"tenantId\":\"t\",\n" + + " \"applicationId\":\"a\",\n" + + " \"instanceId\":\"i\",\n" + " \"serviceId\": \"content/myId/0/0/stateful\",\n" + " \"wantedVespaVersion\": \"6.42.2\"\n" + " }\n" + diff --git a/routing-generator/src/main/java/com/yahoo/vespa/hosted/routing/nginx/NginxMetricsReporter.java b/routing-generator/src/main/java/com/yahoo/vespa/hosted/routing/nginx/NginxMetricsReporter.java index 79381b8c99e..b9ab1dbe9b6 100644 --- a/routing-generator/src/main/java/com/yahoo/vespa/hosted/routing/nginx/NginxMetricsReporter.java +++ b/routing-generator/src/main/java/com/yahoo/vespa/hosted/routing/nginx/NginxMetricsReporter.java @@ -58,7 +58,7 @@ public class NginxMetricsReporter extends AbstractComponent implements Runnable @Inject public NginxMetricsReporter(ApplicationIdConfig applicationId, Metric metric, HealthStatus healthStatus, RoutingGenerator routingGenerator) { - this(new ApplicationId(applicationId), metric, healthStatus, FileSystems.getDefault(), interval, routingGenerator::routingTable); + this(ApplicationId.from(applicationId), metric, healthStatus, FileSystems.getDefault(), interval, routingGenerator::routingTable); } NginxMetricsReporter(ApplicationId application, Metric metric, HealthStatus healthStatus, FileSystem fileSystem, Duration interval, diff --git a/screwdriver/release-java-artifacts.sh b/screwdriver/release-java-artifacts.sh index 8030638cf5b..8d80bb45578 100755 --- a/screwdriver/release-java-artifacts.sh +++ b/screwdriver/release-java-artifacts.sh @@ -52,6 +52,8 @@ for MODULE in $(comm -2 -3 \ echo "No javadoc available for module" > $MODULE/src/main/javadoc/README done +# Workaround for broken nexus-staging-maven-plugin instead of swapping JDK +export MAVEN_OPTS="--add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.text=ALL-UNNAMED --add-opens=java.desktop/java.awt.font=ALL-UNNAMED" export VESPA_MAVEN_EXTRA_OPTS="--show-version --batch-mode" ./bootstrap.sh @@ -69,20 +71,12 @@ mvn $COMMON_MAVEN_OPTS --file ./maven-plugins/pom.xml -DskipStagingRepositoryClo # Deploy the rest of the artifacts mvn $COMMON_MAVEN_OPTS --threads 8 -DskipStagingRepositoryClose=true -DstagingRepositoryId=$STG_REPO deploy -# Workaround for nexus-staging-maven-plugin:1.6.12:rc-release not working with maven+jdk17 -SWAP_MAVEN_JAVA_WORKAROUND=false -if rpm -q maven-openjdk17 &> /dev/null; then SWAP_MAVEN_JAVA_WORKAROUND=true; fi -if $SWAP_MAVEN_JAVA_WORKAROUND; then dnf swap -y maven-openjdk17 maven-openjdk11; fi - # Close with checks mvn $COMMON_MAVEN_OPTS -N org.sonatype.plugins:nexus-staging-maven-plugin:1.6.12:rc-close -DnexusUrl=https://oss.sonatype.org/ -DserverId=ossrh -DstagingRepositoryId=$STG_REPO # Release if ok mvn $COMMON_MAVEN_OPTS -N org.sonatype.plugins:nexus-staging-maven-plugin:1.6.12:rc-release -DnexusUrl=https://oss.sonatype.org/ -DserverId=ossrh -DstagingRepositoryId=$STG_REPO -# Swap back if we swapped previously -if $SWAP_MAVEN_JAVA_WORKAROUND; then dnf swap -y maven-openjdk11 maven-openjdk17; fi - # Delete the GPG rings rm -rf $SD_SOURCE_DIR/screwdriver/deploy diff --git a/screwdriver/settings-publish.xml b/screwdriver/settings-publish.xml index 5524bf9d7ac..2d6dc2d187c 100644 --- a/screwdriver/settings-publish.xml +++ b/screwdriver/settings-publish.xml @@ -30,7 +30,7 @@ <gpg.publickeyring>pubring.gpg</gpg.publickeyring> <gpg.secretkeyring>secring.gpg</gpg.secretkeyring> <maven.gpg.plugin.version>1.6</maven.gpg.plugin.version> - <nexus.staging.maven.plugin.version>1.6.7</nexus.staging.maven.plugin.version> + <nexus.staging.maven.plugin.version>1.6.12</nexus.staging.maven.plugin.version> </properties> </profile> </profiles> diff --git a/testutil/src/main/java/com/yahoo/test/OrderTester.java b/testutil/src/main/java/com/yahoo/test/OrderTester.java index 4acba4ee7fe..cc28ca9f469 100644 --- a/testutil/src/main/java/com/yahoo/test/OrderTester.java +++ b/testutil/src/main/java/com/yahoo/test/OrderTester.java @@ -15,8 +15,8 @@ import java.util.List; * */ -public abstract class OrderTester<T extends Comparable<T>> { - private ArrayList<List<T>> groups = new ArrayList<>(); +public abstract class OrderTester<T extends Comparable<? super T>> { + private final ArrayList<List<T>> groups = new ArrayList<>(); abstract protected void lessTest(T a, T b); abstract protected void greaterTest(T a, T b); @@ -24,7 +24,7 @@ public abstract class OrderTester<T extends Comparable<T>> { @SafeVarargs @SuppressWarnings("varargs") - private final OrderTester<T> addGroup(T... group) { + private OrderTester<T> addGroup(T... group) { groups.add(Arrays.asList(group)); return this; } diff --git a/testutil/src/main/java/com/yahoo/test/TotalOrderTester.java b/testutil/src/main/java/com/yahoo/test/TotalOrderTester.java index 850369fbc2e..e95bc056ba8 100644 --- a/testutil/src/main/java/com/yahoo/test/TotalOrderTester.java +++ b/testutil/src/main/java/com/yahoo/test/TotalOrderTester.java @@ -20,7 +20,7 @@ import static org.junit.Assert.assertTrue; * @author Vegard Sjonfjell */ -public class TotalOrderTester<T extends Comparable<T>> extends OrderTester<T> { +public class TotalOrderTester<T extends Comparable<? super T>> extends OrderTester<T> { protected void lessTest(T a, T b) throws AssertionError { assertTrue(a + " must be less than " + b, a.compareTo(b) <= -1); } diff --git a/vespajlib/src/main/java/ai/vespa/validation/Name.java b/vespajlib/src/main/java/ai/vespa/validation/Name.java index b275917dcff..a6ab456c285 100644 --- a/vespajlib/src/main/java/ai/vespa/validation/Name.java +++ b/vespajlib/src/main/java/ai/vespa/validation/Name.java @@ -3,8 +3,6 @@ package ai.vespa.validation; import java.util.regex.Pattern; -import static ai.vespa.validation.Validation.requireMatch; - /** * A name has from 1 to 64 {@link String} characters which may be letters, numbers, * dashes or underscores, and must start with a letter. diff --git a/vespajlib/src/main/java/ai/vespa/validation/package-info.java b/vespajlib/src/main/java/ai/vespa/validation/package-info.java new file mode 100644 index 00000000000..edbab3a6fd1 --- /dev/null +++ b/vespajlib/src/main/java/ai/vespa/validation/package-info.java @@ -0,0 +1,5 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package ai.vespa.validation; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/zkfacade/src/main/java/com/yahoo/vespa/curator/mock/MockCurator.java b/zkfacade/src/main/java/com/yahoo/vespa/curator/mock/MockCurator.java index bd3389b8d4d..7c91e54dd4b 100644 --- a/zkfacade/src/main/java/com/yahoo/vespa/curator/mock/MockCurator.java +++ b/zkfacade/src/main/java/com/yahoo/vespa/curator/mock/MockCurator.java @@ -36,7 +36,7 @@ public class MockCurator extends Curator { /** * Creates a mock curator * - * @param stableOrdering if true children of a node are returned in the same order each time they are queries. + * @param stableOrdering if true children of a node are returned in the same order each time they are queried. * This is not what ZooKeeper does. */ public MockCurator(boolean stableOrdering) { |