diff options
19 files changed, 101 insertions, 31 deletions
diff --git a/configserver/src/main/sh/start-configserver b/configserver/src/main/sh/start-configserver index 317af4b2fea..216eefec7a0 100755 --- a/configserver/src/main/sh/start-configserver +++ b/configserver/src/main/sh/start-configserver @@ -157,7 +157,7 @@ jvmargs="$baseuserargs $serveruserargs" export LD_PRELOAD=${VESPA_HOME}/lib64/vespa/malloc/libvespamalloc.so -printenv > $cfpfile +vespa-run-as-vespa-user sh -c "printenv > $cfpfile" fixddir $bundlecachedir vespa-run-as-vespa-user vespa-runserver -s configserver -r 30 -p $pidfile -- \ diff --git a/document/src/main/java/com/yahoo/document/DocumentTypeManagerConfigurer.java b/document/src/main/java/com/yahoo/document/DocumentTypeManagerConfigurer.java index 662fe8665f3..183f92d543c 100644 --- a/document/src/main/java/com/yahoo/document/DocumentTypeManagerConfigurer.java +++ b/document/src/main/java/com/yahoo/document/DocumentTypeManagerConfigurer.java @@ -450,8 +450,9 @@ public class DocumentTypeManagerConfigurer implements ConfigSubscriber.SingleSub void createEmptyStructs() { String docName = docTypeConfig.name(); for (var typeconf : docTypeConfig.structtype()) { - if (usev8geopositions && isPositionStruct(typeconf)) { - addNewType(typeconf.idx(), new GeoPosType(8)); + if (isPositionStruct(typeconf)) { + int geoVersion = usev8geopositions ? 8 : 7; + addNewType(typeconf.idx(), new GeoPosType(geoVersion)); } else { addNewType(typeconf.idx(), new StructDataType(typeconf.name())); } @@ -526,7 +527,7 @@ public class DocumentTypeManagerConfigurer implements ConfigSubscriber.SingleSub } void fillStructs() { for (var structCfg : docTypeConfig.structtype()) { - if (usev8geopositions && isPositionStruct(structCfg)) { + if (isPositionStruct(structCfg)) { continue; } int idx = structCfg.idx(); diff --git a/document/src/vespa/document/base/idstring.cpp b/document/src/vespa/document/base/idstring.cpp index a364ffd90ce..f3e41981c8e 100644 --- a/document/src/vespa/document/base/idstring.cpp +++ b/document/src/vespa/document/base/idstring.cpp @@ -110,13 +110,27 @@ fmemchr(const char * s, const char * e) #endif } +namespace { + +// Avoid issues with primitive alignment when reading from buffer. +// Requires caller to ensure buffer is big enough to read from. +template <typename T> +inline T read_unaligned(const char* buf) noexcept +{ + T tmp; + memcpy(&tmp, buf, sizeof(T)); + return tmp; +} + +} + void verifyIdString(const char * id, size_t sz_) { if (sz_ > 4) { - if (_G_id.as16 == *reinterpret_cast<const uint16_t *>(id) && id[2] == ':') { + if ((_G_id.as16 == read_unaligned<uint16_t>(id)) && (id[2] == ':')) { return; - } else if ((sz_ == 6) && (_G_null.as32 == *reinterpret_cast<const uint32_t *>(id)) && (id[4] == ':') && (id[5] == ':')) { + } else if ((sz_ == 6) && (_G_null.as32 == read_unaligned<uint32_t>(id)) && (id[4] == ':') && (id[5] == ':')) { reportNoId(id); } else if (sz_ > 8) { reportNoSchemeSeparator(id); diff --git a/fnet/src/vespa/fnet/databuffer.cpp b/fnet/src/vespa/fnet/databuffer.cpp index f6a7e1c25e5..7820ae2dfcf 100644 --- a/fnet/src/vespa/fnet/databuffer.cpp +++ b/fnet/src/vespa/fnet/databuffer.cpp @@ -94,7 +94,9 @@ FNET_DataBuffer::Pack(uint32_t needbytes) bufsize *= 2; Alloc newBuf(Alloc::alloc(bufsize)); - memcpy(newBuf.get(), _datapt, GetDataLen()); + if (_datapt != nullptr) { + memcpy(newBuf.get(), _datapt, GetDataLen()); + } _ownedBuf.swap(newBuf); _bufstart = static_cast<char *>(_ownedBuf.get()); _freept = _bufstart + GetDataLen(); diff --git a/fnet/src/vespa/fnet/frt/values.cpp b/fnet/src/vespa/fnet/frt/values.cpp index dedefd64323..85979e2af41 100644 --- a/fnet/src/vespa/fnet/frt/values.cpp +++ b/fnet/src/vespa/fnet/frt/values.cpp @@ -984,7 +984,7 @@ FRT_Values::DecodeBig(FNET_DataBuffer *src, uint32_t len) } if (len != 0) goto error; - if (strncmp(typeString, _typeString, numValues) != 0) goto error; + if ((numValues > 0) && strncmp(typeString, _typeString, numValues) != 0) goto error; return true; error: @@ -1385,6 +1385,9 @@ FRT_Values::EncodeBig(FNET_DataBuffer *dst) const char *p = _typeString; dst->WriteInt32Fast(numValues); + if (numValues == 0) { + return; // p may be nullptr, don't try to write what does not exist. + } dst->WriteBytesFast(p, numValues); for (uint32_t i = 0; i < numValues; i++, p++) { diff --git a/searchlib/src/vespa/searchlib/attribute/predicate_attribute.h b/searchlib/src/vespa/searchlib/attribute/predicate_attribute.h index 0d2931a1f04..40823faff8a 100644 --- a/searchlib/src/vespa/searchlib/attribute/predicate_attribute.h +++ b/searchlib/src/vespa/searchlib/attribute/predicate_attribute.h @@ -63,11 +63,12 @@ public: * The pointer is only guaranteed to be valid for as long as you hold the attribute guard. **/ MinFeatureHandle getMinFeatureVector() const { - return MinFeatureHandle(&_min_feature[0], getNumDocs()); + const auto* min_feature_vector = &_min_feature.acquire_elem_ref(0); + return MinFeatureHandle(min_feature_vector, getNumDocs()); } const IntervalRange * getIntervalRangeVector() const { - return &_interval_range_vector[0]; + return &_interval_range_vector.acquire_elem_ref(0); } IntervalRange getMaxIntervalRange() const { diff --git a/searchlib/src/vespa/searchlib/common/condensedbitvectors.cpp b/searchlib/src/vespa/searchlib/common/condensedbitvectors.cpp index d03f995f99e..39986c43ac5 100644 --- a/searchlib/src/vespa/searchlib/common/condensedbitvectors.cpp +++ b/searchlib/src/vespa/searchlib/common/condensedbitvectors.cpp @@ -71,7 +71,7 @@ private: } bool get(Key key, uint32_t index) const override { assert(key < getKeyCapacity()); - return (_v[index] & (B << key)) != 0; + return (_v.acquire_elem_ref(index) & (B << key)) != 0; } size_t getKeyCapacity() const override { return sizeof(T)*8; } @@ -89,7 +89,7 @@ CondensedBitVectorT<T>::computeCountVector(T mask, CountVector & cv, F func) con size_t i(0); const size_t UNROLL = 2; uint8_t *d = &cv[0]; - const T *v = &_v[0]; + const T *v = &_v.acquire_elem_ref(0); for (const size_t m(cv.size() - (UNROLL - 1)); i < m; i+=UNROLL) { for (size_t j(0); j < UNROLL; j++) { func(d[i+j], countBits(v[i+j] & mask)); @@ -103,8 +103,9 @@ template <typename F> void CondensedBitVectorT<T>::computeTail(T mask, CountVector & cv, F func, size_t i) const { + auto* v = &_v.acquire_elem_ref(0); for (; i < cv.size(); i++) { - func(cv[i], countBits(_v[i] & mask)); + func(cv[i], countBits(v[i] & mask)); } } diff --git a/searchlib/src/vespa/searchlib/predicate/simple_index.h b/searchlib/src/vespa/searchlib/predicate/simple_index.h index 25ba76ce7c4..df3d0686e82 100644 --- a/searchlib/src/vespa/searchlib/predicate/simple_index.h +++ b/searchlib/src/vespa/searchlib/predicate/simple_index.h @@ -88,8 +88,10 @@ public: PostingVectorIterator(const PostingVectorIterator&) = default; PostingVectorIterator& operator=(const PostingVectorIterator&) = default; - explicit PostingVectorIterator(const PostingVector & vector, size_t size) : - _vector(&vector[0]), _size(size) { + explicit PostingVectorIterator(const PostingVector & vector, size_t size) + : _vector(&vector.acquire_elem_ref(0)), + _size(size) + { assert(_size <= vector.size()); linearSeek(1); } diff --git a/vespa-feed-client-api/abi-spec.json b/vespa-feed-client-api/abi-spec.json index 16e532a2c9a..5bd0acf82d3 100644 --- a/vespa-feed-client-api/abi-spec.json +++ b/vespa-feed-client-api/abi-spec.json @@ -140,6 +140,7 @@ "public abstract ai.vespa.feed.client.FeedClientBuilder setCaCertificatesFile(java.nio.file.Path)", "public abstract ai.vespa.feed.client.FeedClientBuilder setCaCertificates(java.util.Collection)", "public abstract ai.vespa.feed.client.FeedClientBuilder setEndpointUris(java.util.List)", + "public abstract ai.vespa.feed.client.FeedClientBuilder setProxy(java.net.URI)", "public abstract ai.vespa.feed.client.FeedClient build()" ], "fields": [ diff --git a/vespa-feed-client-api/src/main/java/ai/vespa/feed/client/FeedClientBuilder.java b/vespa-feed-client-api/src/main/java/ai/vespa/feed/client/FeedClientBuilder.java index 95c9b2c95fe..7ec5fbb02b7 100644 --- a/vespa-feed-client-api/src/main/java/ai/vespa/feed/client/FeedClientBuilder.java +++ b/vespa-feed-client-api/src/main/java/ai/vespa/feed/client/FeedClientBuilder.java @@ -109,6 +109,9 @@ public interface FeedClientBuilder { /** Overrides endpoint URIs for this client */ FeedClientBuilder setEndpointUris(List<URI> endpoints); + /** Specify HTTP(S) proxy for all endpoints */ + FeedClientBuilder setProxy(URI uri); + /** Constructs instance of {@link FeedClient} from builder configuration */ FeedClient build(); diff --git a/vespa-feed-client-cli/src/main/java/ai/vespa/feed/client/impl/CliArguments.java b/vespa-feed-client-cli/src/main/java/ai/vespa/feed/client/impl/CliArguments.java index ac859359bfa..e024f961e26 100644 --- a/vespa-feed-client-cli/src/main/java/ai/vespa/feed/client/impl/CliArguments.java +++ b/vespa-feed-client-cli/src/main/java/ai/vespa/feed/client/impl/CliArguments.java @@ -55,6 +55,7 @@ class CliArguments { private static final String VERSION_OPTION = "version"; private static final String STDIN_OPTION = "stdin"; private static final String DOOM_OPTION = "max-failure-seconds"; + private static final String PROXY_OPTION = "proxy"; private final CommandLine arguments; @@ -165,6 +166,16 @@ class CliArguments { boolean dryrunEnabled() { return has(DRYRUN_OPTION); } + Optional<URI> proxy() throws CliArgumentsException { + try { + URL url = (URL) arguments.getParsedOptionValue(PROXY_OPTION); + if (url == null) return Optional.empty(); + return Optional.of(url.toURI()); + } catch (ParseException | URISyntaxException e) { + throw new CliArgumentsException("Invalid proxy: " + e.getMessage(), e); + } + } + private OptionalInt intValue(String option) throws CliArgumentsException { try { Number number = (Number) arguments.getParsedOptionValue(option); @@ -310,6 +321,12 @@ class CliArguments { .addOption(Option.builder() .longOpt(SHOW_ALL_OPTION) .desc("Print the result of every feed operation") + .build()) + .addOption(Option.builder() + .longOpt(PROXY_OPTION) + .desc("URI to proxy endpoint") + .hasArg() + .type(URL.class) .build()); } diff --git a/vespa-feed-client-cli/src/main/java/ai/vespa/feed/client/impl/CliClient.java b/vespa-feed-client-cli/src/main/java/ai/vespa/feed/client/impl/CliClient.java index 5e904b37588..68b9cf6af0e 100644 --- a/vespa-feed-client-cli/src/main/java/ai/vespa/feed/client/impl/CliClient.java +++ b/vespa-feed-client-cli/src/main/java/ai/vespa/feed/client/impl/CliClient.java @@ -138,6 +138,7 @@ public class CliClient { builder.setDryrun(cliArgs.dryrunEnabled()); cliArgs.doomSeconds().ifPresent(doom -> builder.setCircuitBreaker(new GracePeriodCircuitBreaker(Duration.ofSeconds(10), Duration.ofSeconds(doom)))); + cliArgs.proxy().ifPresent(builder::setProxy); return builder.build(); } diff --git a/vespa-feed-client-cli/src/test/java/ai/vespa/feed/client/impl/CliArgumentsTest.java b/vespa-feed-client-cli/src/test/java/ai/vespa/feed/client/impl/CliArgumentsTest.java index fe3dc465814..201a85ed09d 100644 --- a/vespa-feed-client-cli/src/test/java/ai/vespa/feed/client/impl/CliArgumentsTest.java +++ b/vespa-feed-client-cli/src/test/java/ai/vespa/feed/client/impl/CliArgumentsTest.java @@ -1,7 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.feed.client.impl; -import ai.vespa.feed.client.impl.CliArguments; import org.junit.jupiter.api.Test; import java.io.ByteArrayOutputStream; @@ -11,7 +10,10 @@ import java.nio.file.Files; import java.nio.file.Paths; import java.time.Duration; -import static org.junit.jupiter.api.Assertions.*; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; /** * @author bjorncs @@ -26,7 +28,7 @@ class CliArgumentsTest { "--ca-certificates=ca-certs.pem", "--disable-ssl-hostname-verification", "--header=\"My-Header: my-value\"", "--header", "Another-Header: another-value", "--benchmark", "--route=myroute", "--timeout=0.125", "--trace=9", "--verbose", "--silent", - "--show-errors", "--show-all", "--max-failure-seconds=30"}); + "--show-errors", "--show-all", "--max-failure-seconds=30", "--proxy", "https://myproxy:1234"}); assertEquals(URI.create("https://vespa.ai:4443/"), args.endpoint()); assertEquals(Paths.get("feed.json"), args.inputFile().get()); assertEquals(10, args.connections().getAsInt()); @@ -49,6 +51,7 @@ class CliArgumentsTest { assertTrue(args.showErrors()); assertTrue(args.showSuccesses()); assertFalse(args.showProgress()); + assertEquals(URI.create("https://myproxy:1234"), args.proxy().orElse(null)); } @Test diff --git a/vespa-feed-client-cli/src/test/resources/help.txt b/vespa-feed-client-cli/src/test/resources/help.txt index 323206ab128..66d7c3521c2 100644 --- a/vespa-feed-client-cli/src/test/resources/help.txt +++ b/vespa-feed-client-cli/src/test/resources/help.txt @@ -25,6 +25,7 @@ Vespa feed client streams per HTTP/2 connection --private-key <arg> Path to PEM/PKCS#8 encoded private key file + --proxy <arg> URI to proxy endpoint --route <arg> Target Vespa route for feed operations --show-all Print the result of every feed diff --git a/vespa-feed-client/src/main/java/ai/vespa/feed/client/impl/ApacheCluster.java b/vespa-feed-client/src/main/java/ai/vespa/feed/client/impl/ApacheCluster.java index b51210d22ea..62cd56f21ce 100644 --- a/vespa-feed-client/src/main/java/ai/vespa/feed/client/impl/ApacheCluster.java +++ b/vespa-feed-client/src/main/java/ai/vespa/feed/client/impl/ApacheCluster.java @@ -10,15 +10,13 @@ import org.apache.hc.client5.http.impl.async.HttpAsyncClients; import org.apache.hc.client5.http.ssl.ClientTlsStrategyBuilder; import org.apache.hc.core5.concurrent.FutureCallback; import org.apache.hc.core5.http.ContentType; +import org.apache.hc.core5.http.HttpHost; import org.apache.hc.core5.http.message.BasicHeader; import org.apache.hc.core5.http2.config.H2Config; import org.apache.hc.core5.net.URIAuthority; import org.apache.hc.core5.reactor.IOReactorConfig; -import org.apache.hc.core5.util.Timeout; - -import org.apache.hc.core5.function.Factory; import org.apache.hc.core5.reactor.ssl.TlsDetails; -import javax.net.ssl.SSLEngine; +import org.apache.hc.core5.util.Timeout; import javax.net.ssl.SSLContext; import java.io.IOException; @@ -45,11 +43,7 @@ class ApacheCluster implements Cluster { private final List<Endpoint> endpoints = new ArrayList<>(); private final List<BasicHeader> defaultHeaders = Arrays.asList(new BasicHeader("User-Agent", String.format("vespa-feed-client/%s", Vespa.VERSION)), new BasicHeader("Vespa-Client-Version", Vespa.VERSION)); - private final RequestConfig defaultConfig = RequestConfig.custom() - .setConnectTimeout(Timeout.ofSeconds(10)) - .setConnectionRequestTimeout(Timeout.DISABLED) - .setResponseTimeout(Timeout.ofSeconds(190)) - .build(); + private final RequestConfig requestConfig; private final ScheduledExecutorService executor = Executors.newSingleThreadScheduledExecutor(t -> new Thread(t, "request-timeout-thread")); @@ -57,6 +51,7 @@ class ApacheCluster implements Cluster { for (URI endpoint : builder.endpoints) for (int i = 0; i < builder.connectionsPerEndpoint; i++) endpoints.add(new Endpoint(createHttpClient(builder), endpoint)); + this.requestConfig = createRequestConfig(builder); } @Override @@ -75,7 +70,7 @@ class ApacheCluster implements Cluster { SimpleHttpRequest request = new SimpleHttpRequest(wrapped.method(), wrapped.path()); request.setScheme(endpoint.url.getScheme()); request.setAuthority(new URIAuthority(endpoint.url.getHost(), portOf(endpoint.url))); - request.setConfig(defaultConfig); + request.setConfig(requestConfig); defaultHeaders.forEach(request::setHeader); wrapped.headers().forEach((name, value) -> request.setHeader(name, value.get())); if (wrapped.body() != null) @@ -162,6 +157,15 @@ class ApacheCluster implements Cluster { : url.getPort(); } + private static RequestConfig createRequestConfig(FeedClientBuilderImpl b) { + RequestConfig.Builder builder = RequestConfig.custom() + .setConnectTimeout(Timeout.ofSeconds(10)) + .setConnectionRequestTimeout(Timeout.DISABLED) + .setResponseTimeout(Timeout.ofSeconds(190)); + if (b.proxy != null) builder.setProxy(new HttpHost(b.proxy.getScheme(), b.proxy.getHost(), b.proxy.getPort())); + return builder.build(); + } + private static class ApacheHttpResponse implements HttpResponse { private final SimpleHttpResponse wrapped; diff --git a/vespa-feed-client/src/main/java/ai/vespa/feed/client/impl/FeedClientBuilderImpl.java b/vespa-feed-client/src/main/java/ai/vespa/feed/client/impl/FeedClientBuilderImpl.java index 7dafeb0b541..134ad464618 100644 --- a/vespa-feed-client/src/main/java/ai/vespa/feed/client/impl/FeedClientBuilderImpl.java +++ b/vespa-feed-client/src/main/java/ai/vespa/feed/client/impl/FeedClientBuilderImpl.java @@ -19,7 +19,6 @@ import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Objects; import java.util.function.Supplier; import static java.util.Objects.requireNonNull; @@ -50,7 +49,7 @@ public class FeedClientBuilderImpl implements FeedClientBuilder { Collection<X509Certificate> caCertificates; boolean benchmark = true; boolean dryrun = false; - + URI proxy; public FeedClientBuilderImpl() { @@ -188,6 +187,8 @@ public class FeedClientBuilderImpl implements FeedClientBuilder { return this; } + @Override public FeedClientBuilder setProxy(URI uri) { this.proxy = uri; return this; } + /** Constructs instance of {@link ai.vespa.feed.client.FeedClient} from builder configuration */ @Override public FeedClient build() { diff --git a/vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/mapreduce/VespaRecordWriter.java b/vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/mapreduce/VespaRecordWriter.java index c381ec87492..6d6c3789835 100644 --- a/vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/mapreduce/VespaRecordWriter.java +++ b/vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/mapreduce/VespaRecordWriter.java @@ -130,6 +130,12 @@ public class VespaRecordWriter extends RecordWriter<Object, Object> { .setMaxStreamPerConnection(streamsPerConnection) .setDryrun(config.dryrun()) .setRetryStrategy(retryStrategy(config)); + if (config.proxyHost() != null) { + URI proxyUri = URI.create(String.format( + "%s://%s:%d", config.proxyScheme(), config.proxyHost(), config.proxyPort())); + log.info("Using proxy " + proxyUri); + feedClientBuilder.setProxy(proxyUri); + } onFeedClientInitialization(feedClientBuilder); return feedClientBuilder.build(); diff --git a/vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/mapreduce/util/VespaConfiguration.java b/vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/mapreduce/util/VespaConfiguration.java index 1421a3fcd43..715546fe6fe 100644 --- a/vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/mapreduce/util/VespaConfiguration.java +++ b/vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/mapreduce/util/VespaConfiguration.java @@ -16,6 +16,7 @@ public class VespaConfiguration { public static final String USE_SSL = "vespa.feed.ssl"; public static final String PROXY_HOST = "vespa.feed.proxy.host"; public static final String PROXY_PORT = "vespa.feed.proxy.port"; + public static final String PROXY_SCHEME = "vespa.feed.proxy.scheme"; public static final String DRYRUN = "vespa.feed.dryrun"; public static final String USE_COMPRESSION = "vespa.feed.usecompression"; public static final String DATA_FORMAT = "vespa.feed.data.format"; @@ -71,6 +72,13 @@ public class VespaConfiguration { } + public String proxyScheme() { + String raw = getString(PROXY_SCHEME); + if (raw == null) return "http"; + return raw; + } + + public boolean dryrun() { return getBoolean(DRYRUN, false); } @@ -186,6 +194,7 @@ public class VespaConfiguration { sb.append(USE_SSL + ": " + useSSL().map(Object::toString).orElse("<empty>") + "\n"); sb.append(PROXY_HOST + ": " + proxyHost() + "\n"); sb.append(PROXY_PORT + ": " + proxyPort() + "\n"); + sb.append(PROXY_SCHEME + ": " + proxyScheme() + "\n"); sb.append(DRYRUN + ": " + dryrun() +"\n"); sb.append(USE_COMPRESSION + ": " + useCompression() +"\n"); sb.append(DATA_FORMAT + ": " + dataFormat() +"\n"); diff --git a/vespalib/src/vespa/vespalib/net/server_socket.h b/vespalib/src/vespa/vespalib/net/server_socket.h index 5dd59bf6559..09dd61a4c5b 100644 --- a/vespalib/src/vespa/vespalib/net/server_socket.h +++ b/vespalib/src/vespa/vespalib/net/server_socket.h @@ -20,7 +20,7 @@ private: void cleanup(); public: - ServerSocket() : _handle(), _path() {} + ServerSocket() : _handle(), _path(), _blocking(false), _shutdown(false) {} explicit ServerSocket(const SocketSpec &spec); explicit ServerSocket(const vespalib::string &spec); explicit ServerSocket(int port); |