diff options
-rw-r--r-- | config-provisioning/src/main/java/com/yahoo/config/provision/ApplicationId.java | 2 | ||||
-rw-r--r-- | fnet/src/vespa/fnet/controlpacket.cpp | 5 | ||||
-rw-r--r-- | fnet/src/vespa/fnet/controlpacket.h | 6 | ||||
-rw-r--r-- | fnet/src/vespa/fnet/frt/supervisor.cpp | 6 | ||||
-rw-r--r-- | fnet/src/vespa/fnet/transport.cpp | 18 | ||||
-rw-r--r-- | fnet/src/vespa/fnet/transport.h | 17 | ||||
-rw-r--r-- | fnet/src/vespa/fnet/transport_thread.cpp | 33 | ||||
-rw-r--r-- | fnet/src/vespa/fnet/transport_thread.h | 23 | ||||
-rwxr-xr-x | screwdriver/release-java-artifacts.sh | 10 | ||||
-rw-r--r-- | screwdriver/settings-publish.xml | 2 | ||||
-rw-r--r-- | vespa-hadoop/pom.xml | 16 | ||||
-rw-r--r-- | vespajlib/src/main/java/ai/vespa/validation/package-info.java | 5 |
12 files changed, 92 insertions, 51 deletions
diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/ApplicationId.java b/config-provisioning/src/main/java/com/yahoo/config/provision/ApplicationId.java index 796fe586fb9..aa70bf4d26a 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/ApplicationId.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/ApplicationId.java @@ -17,7 +17,7 @@ import java.util.regex.Pattern; public class ApplicationId implements Comparable<ApplicationId> { // TODO: remove '.' and '*' from this pattern. - static final Pattern namePattern = Pattern.compile("(?!\\.\\.)[a-zA-Z0-9_.*-]{1,64}"); + static final Pattern namePattern = Pattern.compile("(?!\\.\\.)[a-zA-Z0-9_.*-]{1,256}"); private static final ApplicationId global = new ApplicationId(TenantName.from("*"), ApplicationName.from("*"), diff --git a/fnet/src/vespa/fnet/controlpacket.cpp b/fnet/src/vespa/fnet/controlpacket.cpp index 9ff69a76210..9aa03ad5e3a 100644 --- a/fnet/src/vespa/fnet/controlpacket.cpp +++ b/fnet/src/vespa/fnet/controlpacket.cpp @@ -95,7 +95,10 @@ FNET_ControlPacket FNET_ControlPacket::IOCClose(FNET_CMD_IOC_CLOSE); FNET_ControlPacket -FNET_ControlPacket::DetachServerAdapter(FNET_CMD_DETACH_SERVER_ADAPTER); +FNET_ControlPacket::DetachServerAdapterInit(FNET_CMD_DETACH_SERVER_ADAPTER_INIT); + +FNET_ControlPacket +FNET_ControlPacket::DetachServerAdapterFini(FNET_CMD_DETACH_SERVER_ADAPTER_FINI); FNET_ControlPacket FNET_ControlPacket::Execute(FNET_CMD_EXECUTE); diff --git a/fnet/src/vespa/fnet/controlpacket.h b/fnet/src/vespa/fnet/controlpacket.h index 8dd2d034ae6..ad846d37c30 100644 --- a/fnet/src/vespa/fnet/controlpacket.h +++ b/fnet/src/vespa/fnet/controlpacket.h @@ -38,7 +38,8 @@ public: FNET_CMD_IOC_ENABLE_WRITE, FNET_CMD_IOC_HANDSHAKE_ACT, FNET_CMD_IOC_CLOSE, - FNET_CMD_DETACH_SERVER_ADAPTER, + FNET_CMD_DETACH_SERVER_ADAPTER_INIT, + FNET_CMD_DETACH_SERVER_ADAPTER_FINI, FNET_CMD_EXECUTE, FNET_CMD_TIMEOUT, FNET_CMD_BAD_PACKET, @@ -51,7 +52,8 @@ public: static FNET_ControlPacket IOCEnableWrite; static FNET_ControlPacket IOCHandshakeACT; static FNET_ControlPacket IOCClose; - static FNET_ControlPacket DetachServerAdapter; + static FNET_ControlPacket DetachServerAdapterInit; + static FNET_ControlPacket DetachServerAdapterFini; static FNET_ControlPacket Execute; static FNET_ControlPacket Timeout; static FNET_ControlPacket BadPacket; diff --git a/fnet/src/vespa/fnet/frt/supervisor.cpp b/fnet/src/vespa/fnet/frt/supervisor.cpp index 0a31b9d882b..1681321b239 100644 --- a/fnet/src/vespa/fnet/frt/supervisor.cpp +++ b/fnet/src/vespa/fnet/frt/supervisor.cpp @@ -30,12 +30,10 @@ FRT_Supervisor::FRT_Supervisor(FNET_Transport *transport) FRT_Supervisor::~FRT_Supervisor() { + _transport->detach(this); if (_connector != nullptr) { - _connector->Owner()->Close(_connector, /* needref */ false); + _connector->SubRef(); } - _transport->wait_for_pending_resolves(); - _transport->detach(this); - _transport->sync(); } FNET_Scheduler * diff --git a/fnet/src/vespa/fnet/transport.cpp b/fnet/src/vespa/fnet/transport.cpp index 0a79f324bb9..1130b6d3e5e 100644 --- a/fnet/src/vespa/fnet/transport.cpp +++ b/fnet/src/vespa/fnet/transport.cpp @@ -125,6 +125,11 @@ TransportConfig::time_tools() const { } // fnet +void +FNET_Transport::wait_for_pending_resolves() { + _async_resolver->wait_for_pending_resolves(); +} + FNET_Transport::FNET_Transport(const fnet::TransportConfig &cfg) : _async_resolver(cfg.resolver()), _crypto_engine(cfg.crypto()), @@ -158,11 +163,6 @@ FNET_Transport::resolve_async(const vespalib::string &spec, _async_resolver->resolve_async(spec, std::move(result_handler)); } -void -FNET_Transport::wait_for_pending_resolves() { - _async_resolver->wait_for_pending_resolves(); -} - vespalib::CryptoSocket::UP FNET_Transport::create_client_crypto_socket(vespalib::SocketHandle socket, const vespalib::SocketSpec &spec) { @@ -221,8 +221,14 @@ void FNET_Transport::detach(FNET_IServerAdapter *server_adapter) { for (const auto &thread: _threads) { - thread->detach(server_adapter); + thread->init_detach(server_adapter); + } + wait_for_pending_resolves(); + sync(); + for (const auto &thread: _threads) { + thread->fini_detach(server_adapter); } + sync(); } FNET_Scheduler * diff --git a/fnet/src/vespa/fnet/transport.h b/fnet/src/vespa/fnet/transport.h index 7dbfd80dfe7..d6e4aefb02b 100644 --- a/fnet/src/vespa/fnet/transport.h +++ b/fnet/src/vespa/fnet/transport.h @@ -113,6 +113,11 @@ private: Threads _threads; const FNET_Config _config; + /** + * Wait for all pending resolve requests. + **/ + void wait_for_pending_resolves(); + public: FNET_Transport(const FNET_Transport &) = delete; FNET_Transport & operator = (const FNET_Transport &) = delete; @@ -160,11 +165,6 @@ public: vespalib::AsyncResolver::ResultHandler::WP result_handler); /** - * Wait for all pending resolve requests. - **/ - void wait_for_pending_resolves(); - - /** * Wrap a plain socket endpoint (client side) in a CryptoSocket. The * implementation will be determined by the CryptoEngine used by * this Transport. @@ -258,11 +258,8 @@ public: * Detach a server adapter from this transport. * * This will close all connectors and connections referencing the - * server adapter. Note that this is an async - * operation. 'wait_for_pending_resolves' should be called before - * this to make sure any in-flight connections are added - * first. 'sync' should be called after this to drain any pending - * call-backs. + * server adapter. Note that this function will also synchronize + * with async address resolving and underlying transport threads. **/ void detach(FNET_IServerAdapter *server_adapter); diff --git a/fnet/src/vespa/fnet/transport_thread.cpp b/fnet/src/vespa/fnet/transport_thread.cpp index 53363740bae..d46d174c670 100644 --- a/fnet/src/vespa/fnet/transport_thread.cpp +++ b/fnet/src/vespa/fnet/transport_thread.cpp @@ -162,7 +162,7 @@ FNET_TransportThread::SafeDiscardEvent(FNET_ControlPacket *cpacket, void FNET_TransportThread::handle_add_cmd(FNET_IOComponent *ioc) { - if (ioc->handle_add_event()) { + if ((_detaching.count(ioc->server_adapter()) == 0) && ioc->handle_add_event()) { AddComponent(ioc); ioc->_flags._ioc_added = true; ioc->attach_selector(_selector); @@ -186,8 +186,9 @@ FNET_TransportThread::handle_close_cmd(FNET_IOComponent *ioc) void -FNET_TransportThread::handle_detach_server_adapter_cmd(FNET_IServerAdapter *server_adapter) +FNET_TransportThread::handle_detach_server_adapter_init_cmd(FNET_IServerAdapter *server_adapter) { + _detaching.insert(server_adapter); FNET_IOComponent *component = _componentsHead; while (component != nullptr) { FNET_IOComponent *tmp = component; @@ -201,6 +202,12 @@ FNET_TransportThread::handle_detach_server_adapter_cmd(FNET_IServerAdapter *serv } +void +FNET_TransportThread::handle_detach_server_adapter_fini_cmd(FNET_IServerAdapter *server_adapter) +{ + _detaching.erase(server_adapter); +} + extern "C" { static void pipehandler(int) @@ -241,7 +248,8 @@ FNET_TransportThread::FNET_TransportThread(FNET_Transport &owner_in) _pseudo_thread(), _started(false), _shutdown(false), - _finished(false) + _finished(false), + _detaching() { trapsigpipe(); } @@ -348,9 +356,15 @@ FNET_TransportThread::Close(FNET_IOComponent *comp, bool needRef) } void -FNET_TransportThread::detach(FNET_IServerAdapter *server_adapter) +FNET_TransportThread::init_detach(FNET_IServerAdapter *server_adapter) { - PostEvent(&FNET_ControlPacket::DetachServerAdapter, FNET_Context(server_adapter)); + PostEvent(&FNET_ControlPacket::DetachServerAdapterInit, FNET_Context(server_adapter)); +} + +void +FNET_TransportThread::fini_detach(FNET_IServerAdapter *server_adapter) +{ + PostEvent(&FNET_ControlPacket::DetachServerAdapterFini, FNET_Context(server_adapter)); } bool @@ -432,8 +446,13 @@ FNET_TransportThread::handle_wakeup() continue; } - if (packet->GetCommand() == FNET_ControlPacket::FNET_CMD_DETACH_SERVER_ADAPTER) { - handle_detach_server_adapter_cmd(context._value.SERVER_ADAPTER); + if (packet->GetCommand() == FNET_ControlPacket::FNET_CMD_DETACH_SERVER_ADAPTER_INIT) { + handle_detach_server_adapter_init_cmd(context._value.SERVER_ADAPTER); + continue; + } + + if (packet->GetCommand() == FNET_ControlPacket::FNET_CMD_DETACH_SERVER_ADAPTER_FINI) { + handle_detach_server_adapter_fini_cmd(context._value.SERVER_ADAPTER); continue; } diff --git a/fnet/src/vespa/fnet/transport_thread.h b/fnet/src/vespa/fnet/transport_thread.h index c120894ac9c..b507c5dc31d 100644 --- a/fnet/src/vespa/fnet/transport_thread.h +++ b/fnet/src/vespa/fnet/transport_thread.h @@ -13,6 +13,7 @@ #include <mutex> #include <condition_variable> #include <chrono> +#include <set> namespace fnet { struct TimeTools; } class FNET_Transport; @@ -51,6 +52,7 @@ private: std::atomic<bool> _started; // event loop started ? std::atomic<bool> _shutdown; // should stop event loop ? std::atomic<bool> _finished; // event loop stopped ? + std::set<FNET_IServerAdapter*> _detaching; // server adapters being detached /** * Add an IOComponent to the list of components. This operation is @@ -143,7 +145,8 @@ private: void handle_add_cmd(FNET_IOComponent *ioc); void handle_close_cmd(FNET_IOComponent *ioc); - void handle_detach_server_adapter_cmd(FNET_IServerAdapter *server_adapter); + void handle_detach_server_adapter_init_cmd(FNET_IServerAdapter *server_adapter); + void handle_detach_server_adapter_fini_cmd(FNET_IServerAdapter *server_adapter); /** * This method is called to initialize the transport thread event @@ -336,16 +339,16 @@ public: void Close(FNET_IOComponent *comp, bool needRef = true); /** - * Detach a server adapter from this transport. - * - * This will close all connectors and connections referencing the - * server adapter. Note that this is an async - * operation. 'wait_for_pending_resolves' (on the owning - * Transport) should be called before this to make sure any - * in-flight connections are added first. 'sync' should be called - * after this to drain any pending call-backs. + * Start the operation of detaching a server adapter from this + * transport. + **/ + void init_detach(FNET_IServerAdapter *server_adapter); + + /** + * Complete the operation of detaching a server adapter from this + * transport. **/ - void detach(FNET_IServerAdapter *server_adapter); + void fini_detach(FNET_IServerAdapter *server_adapter); /** * Post an execution event on the transport event queue. The return diff --git a/screwdriver/release-java-artifacts.sh b/screwdriver/release-java-artifacts.sh index 8030638cf5b..8d80bb45578 100755 --- a/screwdriver/release-java-artifacts.sh +++ b/screwdriver/release-java-artifacts.sh @@ -52,6 +52,8 @@ for MODULE in $(comm -2 -3 \ echo "No javadoc available for module" > $MODULE/src/main/javadoc/README done +# Workaround for broken nexus-staging-maven-plugin instead of swapping JDK +export MAVEN_OPTS="--add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.text=ALL-UNNAMED --add-opens=java.desktop/java.awt.font=ALL-UNNAMED" export VESPA_MAVEN_EXTRA_OPTS="--show-version --batch-mode" ./bootstrap.sh @@ -69,20 +71,12 @@ mvn $COMMON_MAVEN_OPTS --file ./maven-plugins/pom.xml -DskipStagingRepositoryClo # Deploy the rest of the artifacts mvn $COMMON_MAVEN_OPTS --threads 8 -DskipStagingRepositoryClose=true -DstagingRepositoryId=$STG_REPO deploy -# Workaround for nexus-staging-maven-plugin:1.6.12:rc-release not working with maven+jdk17 -SWAP_MAVEN_JAVA_WORKAROUND=false -if rpm -q maven-openjdk17 &> /dev/null; then SWAP_MAVEN_JAVA_WORKAROUND=true; fi -if $SWAP_MAVEN_JAVA_WORKAROUND; then dnf swap -y maven-openjdk17 maven-openjdk11; fi - # Close with checks mvn $COMMON_MAVEN_OPTS -N org.sonatype.plugins:nexus-staging-maven-plugin:1.6.12:rc-close -DnexusUrl=https://oss.sonatype.org/ -DserverId=ossrh -DstagingRepositoryId=$STG_REPO # Release if ok mvn $COMMON_MAVEN_OPTS -N org.sonatype.plugins:nexus-staging-maven-plugin:1.6.12:rc-release -DnexusUrl=https://oss.sonatype.org/ -DserverId=ossrh -DstagingRepositoryId=$STG_REPO -# Swap back if we swapped previously -if $SWAP_MAVEN_JAVA_WORKAROUND; then dnf swap -y maven-openjdk11 maven-openjdk17; fi - # Delete the GPG rings rm -rf $SD_SOURCE_DIR/screwdriver/deploy diff --git a/screwdriver/settings-publish.xml b/screwdriver/settings-publish.xml index 5524bf9d7ac..2d6dc2d187c 100644 --- a/screwdriver/settings-publish.xml +++ b/screwdriver/settings-publish.xml @@ -30,7 +30,7 @@ <gpg.publickeyring>pubring.gpg</gpg.publickeyring> <gpg.secretkeyring>secring.gpg</gpg.secretkeyring> <maven.gpg.plugin.version>1.6</maven.gpg.plugin.version> - <nexus.staging.maven.plugin.version>1.6.7</nexus.staging.maven.plugin.version> + <nexus.staging.maven.plugin.version>1.6.12</nexus.staging.maven.plugin.version> </properties> </profile> </profiles> diff --git a/vespa-hadoop/pom.xml b/vespa-hadoop/pom.xml index 2530e461354..6fe87541448 100644 --- a/vespa-hadoop/pom.xml +++ b/vespa-hadoop/pom.xml @@ -126,7 +126,21 @@ <groupId>com.yahoo.vespa</groupId> <artifactId>vespa-feed-client</artifactId> <version>${project.version}</version> - <scope>compile</scope> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>vespa-feed-client-api</artifactId> + <version>${project.version}</version> + </dependency> + + <!-- Jackson dependencies used in this module --> + <dependency> + <groupId>com.fasterxml.jackson.core</groupId> + <artifactId>jackson-core</artifactId> + </dependency> + <dependency> + <groupId>com.fasterxml.jackson.core</groupId> + <artifactId>jackson-databind</artifactId> </dependency> </dependencies> diff --git a/vespajlib/src/main/java/ai/vespa/validation/package-info.java b/vespajlib/src/main/java/ai/vespa/validation/package-info.java new file mode 100644 index 00000000000..edbab3a6fd1 --- /dev/null +++ b/vespajlib/src/main/java/ai/vespa/validation/package-info.java @@ -0,0 +1,5 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package ai.vespa.validation; + +import com.yahoo.osgi.annotation.ExportPackage; |