diff options
71 files changed, 515 insertions, 323 deletions
diff --git a/build_settings.cmake b/build_settings.cmake index 272242ecc5c..1dfd55e7d0e 100644 --- a/build_settings.cmake +++ b/build_settings.cmake @@ -138,6 +138,14 @@ if(VALGRIND_EXECUTABLE) set(VALGRIND_OPTIONS "--leak-check=yes --error-exitcode=1 --run-libc-freeres=no --track-origins=yes --suppressions=${VALGRIND_SUPPRESSIONS_FILE}") set(VALGRIND_COMMAND "${VALGRIND_EXECUTABLE} ${VALGRIND_OPTIONS}") endif() +# Automatically set sanitizer suppressions file and arguments for unit tests +if(VESPA_USE_SANITIZER) + if(VESPA_USE_SANITIZER STREQUAL "thread") + set(VESPA_SANITIZER_SUPPRESSIONS_FILE "${PROJECT_SOURCE_DIR}/tsan-suppressions.txt") + # Maximize the amount of history we can track, including mutex order inversion histories + set(VESPA_SANITIZER_ENV "TSAN_OPTIONS=suppressions=${VESPA_SANITIZER_SUPPRESSIONS_FILE} history_size=7 detect_deadlocks=1 second_deadlock_stack=1") + endif() +endif() if(VESPA_LLVM_VERSION) else() diff --git a/configserver-client/src/main/java/ai/vespa/hosted/client/AbstractConfigServerClient.java b/configserver-client/src/main/java/ai/vespa/hosted/client/AbstractConfigServerClient.java index b96d41648da..865280def6c 100644 --- a/configserver-client/src/main/java/ai/vespa/hosted/client/AbstractConfigServerClient.java +++ b/configserver-client/src/main/java/ai/vespa/hosted/client/AbstractConfigServerClient.java @@ -1,6 +1,9 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.hosted.client; +import ai.vespa.http.HttpURL; +import ai.vespa.http.HttpURL.Path; +import ai.vespa.http.HttpURL.Query; import org.apache.hc.client5.http.config.RequestConfig; import org.apache.hc.client5.http.protocol.HttpClientContext; import org.apache.hc.core5.http.ClassicHttpRequest; @@ -12,14 +15,11 @@ import org.apache.hc.core5.http.ParseException; import org.apache.hc.core5.http.io.entity.EntityUtils; import org.apache.hc.core5.http.io.entity.HttpEntities; import org.apache.hc.core5.http.io.support.ClassicRequestBuilder; -import org.apache.hc.core5.net.URIBuilder; import java.io.IOException; import java.io.UncheckedIOException; import java.net.URI; -import java.net.URISyntaxException; import java.time.Duration; -import java.util.ArrayList; import java.util.List; import java.util.concurrent.TimeUnit; import java.util.function.BiFunction; @@ -50,8 +50,11 @@ public abstract class AbstractConfigServerClient implements ConfigServerClient { Throwable thrown = null; for (URI host : builder.hosts) { ClassicHttpRequest request = ClassicRequestBuilder.create(builder.method.name()) - .setUri(concat(host, builder.uriBuilder)) - .build(); + .setUri(HttpURL.from(host) + .appendPath(builder.path) + .mergeQuery(builder.query) + .asURI()) + .build(); request.setEntity(builder.entity); try { try { @@ -87,23 +90,6 @@ public abstract class AbstractConfigServerClient implements ConfigServerClient { throw new IllegalStateException("No hosts to perform the request against"); } - /** Append path to the given host, which may already contain a root path. */ - static URI concat(URI host, URIBuilder pathAndQuery) { - URIBuilder builder = new URIBuilder(host); - List<String> pathSegments = new ArrayList<>(builder.getPathSegments()); - if ( ! pathSegments.isEmpty() && pathSegments.get(pathSegments.size() - 1).isEmpty()) - pathSegments.remove(pathSegments.size() - 1); - pathSegments.addAll(pathAndQuery.getPathSegments()); - try { - return builder.setPathSegments(pathSegments) - .setParameters(pathAndQuery.getQueryParams()) - .build(); - } - catch (URISyntaxException e) { - throw new IllegalArgumentException("URISyntaxException should not be possible here", e); - } - } - @Override public ConfigServerClient.RequestBuilder send(HostStrategy hosts, Method method) { return new RequestBuilder(hosts, method); @@ -114,8 +100,8 @@ public abstract class AbstractConfigServerClient implements ConfigServerClient { private final Method method; private final HostStrategy hosts; - private final URIBuilder uriBuilder = new URIBuilder(); - private final List<String> pathSegments = new ArrayList<>(); + private HttpURL.Path path = Path.empty(); + private HttpURL.Query query = Query.empty(); private HttpEntity entity; private RequestConfig config = ConfigServerClient.defaultRequestConfig; private ResponseVerifier verifier = ConfigServerClient.throwOnError; @@ -130,8 +116,8 @@ public abstract class AbstractConfigServerClient implements ConfigServerClient { } @Override - public RequestBuilder at(List<String> pathSegments) { - this.pathSegments.addAll(pathSegments); + public RequestBuilder at(Path subPath) { + path = path.append(subPath); return this; } @@ -149,7 +135,7 @@ public abstract class AbstractConfigServerClient implements ConfigServerClient { @Override public ConfigServerClient.RequestBuilder emptyParameters(List<String> keys) { for (String key : keys) - uriBuilder.setParameter(key, null); + query = query.add(key); return this; } @@ -162,7 +148,7 @@ public abstract class AbstractConfigServerClient implements ConfigServerClient { for (int i = 0; i < pairs.size(); ) { String key = pairs.get(i++), value = pairs.get(i++); if (value != null) - uriBuilder.setParameter(key, value); + query = query.put(key, value); } return this; @@ -230,7 +216,6 @@ public abstract class AbstractConfigServerClient implements ConfigServerClient { @Override public <T> T handle(ResponseHandler<T> handler) { - uriBuilder.setPathSegments(pathSegments); return execute(this, (response, request) -> { try { diff --git a/configserver-client/src/main/java/ai/vespa/hosted/client/ConfigServerClient.java b/configserver-client/src/main/java/ai/vespa/hosted/client/ConfigServerClient.java index bfed26779ec..9cbe59c6bec 100644 --- a/configserver-client/src/main/java/ai/vespa/hosted/client/ConfigServerClient.java +++ b/configserver-client/src/main/java/ai/vespa/hosted/client/ConfigServerClient.java @@ -1,6 +1,8 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.hosted.client; +import ai.vespa.http.HttpURL; +import ai.vespa.http.HttpURL.Path; import org.apache.hc.client5.http.config.RequestConfig; import org.apache.hc.core5.http.ClassicHttpRequest; import org.apache.hc.core5.http.ClassicHttpResponse; @@ -60,11 +62,14 @@ public interface ConfigServerClient extends Closeable { /** Builder for a request against a given set of hosts, using this config server client. */ interface RequestBuilder { - /** Appends to the request path. */ + /** Appends to the request path, with no trailing slash. */ default RequestBuilder at(String... pathSegments) { return at(List.of(pathSegments)); } + /** Appends to the request path, with no trailing slash. */ + default RequestBuilder at(List<String> pathSegments) { return at(Path.from(pathSegments).withoutTrailingSlash()); } + /** Appends to the request path. */ - RequestBuilder at(List<String> pathSegments); + RequestBuilder at(HttpURL.Path path); /** Sets the request body as UTF-8 application/json. */ RequestBuilder body(byte[] json); @@ -85,7 +90,7 @@ public interface ConfigServerClient extends Closeable { return parameters(Arrays.asList(pairs)); } - /** Sets the parameter key/values for the request. Number of arguments must be even. Null values are omitted. */ + /** Sets the parameter key/values for the request. Number of arguments must be even. Pairs with {@code null} values are omitted. */ RequestBuilder parameters(List<String> pairs); /** Overrides the default socket read timeout of the request. {@code Duration.ZERO} gives infinite timeout. */ diff --git a/configserver-client/src/test/java/ai/vespa/hosted/client/HttpConfigServerClientTest.java b/configserver-client/src/test/java/ai/vespa/hosted/client/HttpConfigServerClientTest.java index bcdfaa9efa9..e969201605e 100644 --- a/configserver-client/src/test/java/ai/vespa/hosted/client/HttpConfigServerClientTest.java +++ b/configserver-client/src/test/java/ai/vespa/hosted/client/HttpConfigServerClientTest.java @@ -55,7 +55,7 @@ class HttpConfigServerClientTest { server.resetRequests(); // Two attempts on a different IOException. - server.stubFor(post("/prefix/%2Froot")) + server.stubFor(post("/prefix/root")) .setResponse(okJson("{}").withFault(Fault.EMPTY_RESPONSE) .build()); assertThrows(UncheckedIOException.class, @@ -63,9 +63,9 @@ class HttpConfigServerClientTest { URI.create("http://localhost:" + server.port() + "/prefix/"))), Method.POST) .body("hello".getBytes(UTF_8)) - .at("/root") + .at("root") .stream()); - server.verify(2, postRequestedFor(urlEqualTo("/prefix/%2Froot")).withRequestBody(equalTo("hello"))); + server.verify(2, postRequestedFor(urlEqualTo("/prefix/root")).withRequestBody(equalTo("hello"))); server.verify(2, anyRequestedFor(anyUrl())); server.resetRequests(); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java index 76d7ff2fc7f..6028c6db5ea 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java @@ -1,7 +1,8 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server; -import com.yahoo.net.DomainName; +import ai.vespa.http.DomainName; +import ai.vespa.http.HttpURL; import com.google.inject.Inject; import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.component.Version; @@ -31,7 +32,6 @@ import com.yahoo.docproc.jdisc.metric.NullMetric; import com.yahoo.io.IOUtils; import com.yahoo.jdisc.Metric; import com.yahoo.path.Path; -import com.yahoo.restapi.HttpURL; import com.yahoo.slime.Slime; import com.yahoo.transaction.NestedTransaction; import com.yahoo.transaction.Transaction; @@ -559,24 +559,8 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye } } - public HttpResponse serviceStatusPage(ApplicationId applicationId, String hostName, String serviceName, HttpURL.Path pathSuffix) { - // WARNING: pathSuffix may be given by the external user. Make sure no security issues arise... - // We should be OK here, because at most, pathSuffix may change the parent path, but cannot otherwise - // change the hostname and port. Exposing other paths on the cluster controller should be fine. - // TODO: It would be nice to have a simple check to verify pathSuffix doesn't contain /../ components. - HttpURL.Path pathPrefix = HttpURL.Path.empty(); - switch (serviceName) { - case "container-clustercontroller": - pathPrefix = pathPrefix.append("clustercontroller-status").append("v1"); - break; - case "distributor": - case "storagenode": - break; - default: - throw new NotFoundException("No status page for service: " + serviceName); - } - - return httpProxy.get(getApplication(applicationId), hostName, serviceName, pathPrefix.append(pathSuffix)); + public HttpResponse proxyServiceHostnameRequest(ApplicationId applicationId, String hostName, String serviceName, HttpURL.Path path) { + return httpProxy.get(getApplication(applicationId), hostName, serviceName, path); } public Map<String, ClusterReindexing> getClusterReindexingStatus(ApplicationId applicationId) { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/HttpProxy.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/HttpProxy.java index 14acd1b5630..afb72026de6 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/application/HttpProxy.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/HttpProxy.java @@ -1,20 +1,15 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.application; +import ai.vespa.http.DomainName; +import ai.vespa.http.HttpURL; +import ai.vespa.http.HttpURL.Path; +import ai.vespa.http.HttpURL.Scheme; import com.google.inject.Inject; import com.yahoo.config.model.api.HostInfo; import com.yahoo.config.model.api.PortInfo; import com.yahoo.config.model.api.ServiceInfo; import com.yahoo.container.jdisc.HttpResponse; - -import java.util.HashSet; -import java.util.List; -import java.util.logging.Level; - -import com.yahoo.net.DomainName; -import com.yahoo.restapi.HttpURL; -import com.yahoo.restapi.HttpURL.Path; -import com.yahoo.restapi.HttpURL.Scheme; import com.yahoo.vespa.config.server.http.HttpErrorResponse; import com.yahoo.vespa.config.server.http.HttpFetcher; import com.yahoo.vespa.config.server.http.HttpFetcher.Params; @@ -22,10 +17,9 @@ import com.yahoo.vespa.config.server.http.NotFoundException; import com.yahoo.vespa.config.server.http.SimpleHttpFetcher; import java.net.MalformedURLException; -import java.net.URL; +import java.util.List; +import java.util.logging.Level; import java.util.logging.Logger; -import java.util.stream.Collectors; -import java.util.stream.Stream; public class HttpProxy { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/ContentHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/ContentHandler.java index 0ceb459233b..2989eee0b55 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/ContentHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/ContentHandler.java @@ -3,13 +3,12 @@ package com.yahoo.vespa.config.server.http; import com.yahoo.config.application.api.ApplicationFile; import com.yahoo.container.jdisc.HttpResponse; -import com.yahoo.restapi.HttpURL.Path; +import ai.vespa.http.HttpURL.Path; import com.yahoo.restapi.SlimeJsonResponse; import com.yahoo.slime.Cursor; import com.yahoo.slime.Slime; import java.io.InputStreamReader; -import java.util.Arrays; import java.util.Collections; import java.util.List; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/ContentRequest.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/ContentRequest.java index f06e1dabf8c..4536bbad9f6 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/ContentRequest.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/ContentRequest.java @@ -3,7 +3,7 @@ package com.yahoo.vespa.config.server.http; import com.yahoo.config.application.api.ApplicationFile; import com.yahoo.container.jdisc.HttpRequest; -import com.yahoo.restapi.HttpURL.Path; +import ai.vespa.http.HttpURL.Path; import java.io.InputStream; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/SimpleHttpFetcher.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/SimpleHttpFetcher.java index 4fb14dacd9a..6add1f7a9fc 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/SimpleHttpFetcher.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/SimpleHttpFetcher.java @@ -8,16 +8,13 @@ import org.apache.hc.client5.http.config.RequestConfig; import org.apache.hc.client5.http.impl.classic.CloseableHttpClient; import org.apache.hc.client5.http.impl.classic.CloseableHttpResponse; import org.apache.hc.core5.http.HttpEntity; -import org.apache.hc.core5.http.ParseException; -import org.apache.hc.core5.http.io.entity.EntityUtils; import org.apache.hc.core5.util.Timeout; -import java.util.logging.Level; - import java.io.IOException; import java.net.SocketTimeoutException; import java.net.URISyntaxException; import java.net.URL; +import java.util.logging.Level; import java.util.logging.Logger; public class SimpleHttpFetcher implements HttpFetcher { @@ -40,16 +37,12 @@ public class SimpleHttpFetcher implements HttpFetcher { return new StaticResponse( response.getCode(), entity.getContentType(), - EntityUtils.toString(entity)); + entity.getContent().readAllBytes()); } } catch (SocketTimeoutException e) { String message = "Timed out after " + params.readTimeoutMs + " ms reading response from " + url; logger.log(Level.WARNING, message, e); throw new RequestTimeoutException(message); - } catch (ParseException e) { - String message = "Parse error in response from " + url; - logger.log(Level.WARNING, message, e); - throw new InternalServerException(message); } catch (IOException e) { String message = "Failed to get response from " + url; logger.log(Level.WARNING, message, e); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/StaticResponse.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/StaticResponse.java index 3b0f29be765..a64a5551095 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/StaticResponse.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/StaticResponse.java @@ -3,33 +3,27 @@ package com.yahoo.vespa.config.server.http; import com.yahoo.container.jdisc.HttpResponse; -import java.io.ByteArrayInputStream; import java.io.IOException; -import java.io.InputStream; import java.io.OutputStream; import java.nio.charset.StandardCharsets; public class StaticResponse extends HttpResponse { private final String contentType; - private final InputStream body; + private final byte[] body; - /** - * @param body Ownership is passed to StaticResponse (is responsible for closing it) - */ - public StaticResponse(int status, String contentType, InputStream body) { + public StaticResponse(int status, String contentType, byte[] body) { super(status); this.contentType = contentType; this.body = body; } public StaticResponse(int status, String contentType, String body) { - this(status, contentType, new ByteArrayInputStream(body.getBytes(StandardCharsets.UTF_8))); + this(status, contentType, body.getBytes(StandardCharsets.UTF_8)); } @Override public void render(OutputStream outputStream) throws IOException { - body.transferTo(outputStream); - body.close(); + outputStream.write(body); } @Override diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java index 885456ff69c..c788f9a0968 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java @@ -1,7 +1,8 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.http.v2; -import com.yahoo.net.DomainName; +import ai.vespa.http.DomainName; +import ai.vespa.http.HttpURL; import com.google.inject.Inject; import com.yahoo.component.Version; import com.yahoo.config.application.api.ApplicationFile; @@ -16,7 +17,6 @@ import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.io.IOUtils; import com.yahoo.jdisc.Response; import com.yahoo.restapi.ErrorResponse; -import com.yahoo.restapi.HttpURL; import com.yahoo.restapi.MessageResponse; import com.yahoo.restapi.Path; import com.yahoo.slime.Cursor; @@ -86,6 +86,7 @@ public class ApplicationHandler extends HttpHandler { if (path.matches("/application/v2/tenant/{tenant}/application/{application}/environment/{ignore}/region/{ignore}/instance/{instance}/metrics/proton")) return protonMetrics(applicationId(path)); if (path.matches("/application/v2/tenant/{tenant}/application/{application}/environment/{ignore}/region/{ignore}/instance/{instance}/reindexing")) return getReindexingStatus(applicationId(path)); if (path.matches("/application/v2/tenant/{tenant}/application/{application}/environment/{ignore}/region/{ignore}/instance/{instance}/service/{service}/{hostname}/status/{*}")) return serviceStatusPage(applicationId(path), path.get("service"), path.get("hostname"), path.getRest()); + if (path.matches("/application/v2/tenant/{tenant}/application/{application}/environment/{ignore}/region/{ignore}/instance/{instance}/service/{service}/{hostname}/state/v1/metrics")) return serviceStateV1metrics(applicationId(path), path.get("service"), path.get("hostname")); if (path.matches("/application/v2/tenant/{tenant}/application/{application}/environment/{ignore}/region/{ignore}/instance/{instance}/serviceconverge")) return listServiceConverge(applicationId(path), request); if (path.matches("/application/v2/tenant/{tenant}/application/{application}/environment/{ignore}/region/{ignore}/instance/{instance}/serviceconverge/{hostAndPort}")) return checkServiceConverge(applicationId(path), path.get("hostAndPort"), request); if (path.matches("/application/v2/tenant/{tenant}/application/{application}/environment/{ignore}/region/{ignore}/instance/{instance}/suspended")) return isSuspended(applicationId(path)); @@ -134,7 +135,22 @@ public class ApplicationHandler extends HttpHandler { } private HttpResponse serviceStatusPage(ApplicationId applicationId, String service, String hostname, HttpURL.Path pathSuffix) { - return applicationRepository.serviceStatusPage(applicationId, hostname, service, pathSuffix); + HttpURL.Path pathPrefix = HttpURL.Path.empty(); + switch (service) { + case "container-clustercontroller": + pathPrefix = pathPrefix.append("clustercontroller-status").append("v1"); + break; + case "distributor": + case "storagenode": + break; + default: + throw new com.yahoo.vespa.config.server.NotFoundException("No status page for service: " + service); + } + return applicationRepository.proxyServiceHostnameRequest(applicationId, hostname, service, pathPrefix.append(pathSuffix)); + } + + private HttpResponse serviceStateV1metrics(ApplicationId applicationId, String service, String hostname) { + return applicationRepository.proxyServiceHostnameRequest(applicationId, hostname, service, HttpURL.Path.parse("/state/v1/metrics")); } private HttpResponse content(ApplicationId applicationId, HttpURL.Path contentPath, HttpRequest request) { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionContentHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionContentHandler.java index f6af9e616a9..8c69c52e17d 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionContentHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionContentHandler.java @@ -6,7 +6,7 @@ import com.yahoo.config.application.api.ApplicationFile; import com.yahoo.config.provision.TenantName; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; -import com.yahoo.restapi.HttpURL.Path; +import ai.vespa.http.HttpURL.Path; import com.yahoo.vespa.config.server.ApplicationRepository; import com.yahoo.vespa.config.server.http.ContentRequest; import com.yahoo.vespa.config.server.http.ContentHandler; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/request/ApplicationContentRequest.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/request/ApplicationContentRequest.java index 15d6c5c18ff..fe59119a088 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/request/ApplicationContentRequest.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/request/ApplicationContentRequest.java @@ -5,7 +5,7 @@ import com.yahoo.config.application.api.ApplicationFile; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Zone; import com.yahoo.container.jdisc.HttpRequest; -import com.yahoo.restapi.HttpURL.Path; +import ai.vespa.http.HttpURL.Path; import com.yahoo.vespa.config.server.http.ContentRequest; /** diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/request/SessionContentRequestV2.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/request/SessionContentRequestV2.java index 449058eb911..ec402f263db 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/request/SessionContentRequestV2.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/request/SessionContentRequestV2.java @@ -4,11 +4,9 @@ package com.yahoo.vespa.config.server.http.v2.request; import com.yahoo.config.application.api.ApplicationFile; import com.yahoo.config.provision.TenantName; import com.yahoo.container.jdisc.HttpRequest; -import com.yahoo.jdisc.application.BindingMatch; -import com.yahoo.restapi.HttpURL; +import ai.vespa.http.HttpURL; import com.yahoo.restapi.Path; import com.yahoo.vespa.config.server.http.ContentRequest; -import com.yahoo.vespa.config.server.http.Utils; /** * Requests for content and content status (v2) diff --git a/configserver/src/main/java/com/yahoo/vespa/serviceview/StateRequestHandler.java b/configserver/src/main/java/com/yahoo/vespa/serviceview/StateRequestHandler.java index 8b9714c3bfb..1eea88e9caf 100644 --- a/configserver/src/main/java/com/yahoo/vespa/serviceview/StateRequestHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/serviceview/StateRequestHandler.java @@ -5,11 +5,11 @@ import com.fasterxml.jackson.jaxrs.json.JacksonJsonProvider; import com.google.inject.Inject; import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.container.jdisc.ThreadedHttpRequestHandler; -import com.yahoo.net.DomainName; -import com.yahoo.restapi.HttpURL; -import com.yahoo.restapi.HttpURL.Path; -import com.yahoo.restapi.HttpURL.Query; -import com.yahoo.restapi.HttpURL.Scheme; +import ai.vespa.http.DomainName; +import ai.vespa.http.HttpURL; +import ai.vespa.http.HttpURL.Path; +import ai.vespa.http.HttpURL.Query; +import ai.vespa.http.HttpURL.Scheme; import com.yahoo.restapi.RestApi; import com.yahoo.restapi.RestApiRequestHandler; import com.yahoo.restapi.UriBuilder; diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java index e3fc2345c68..ce44fade2f0 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server; -import com.yahoo.net.DomainName; +import ai.vespa.http.DomainName; import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.component.Version; import com.yahoo.config.ConfigInstance; diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/application/HttpProxyTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/application/HttpProxyTest.java index 83cae04cbfd..a2ed5ef6656 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/application/HttpProxyTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/application/HttpProxyTest.java @@ -6,7 +6,7 @@ import com.yahoo.config.model.api.Model; import com.yahoo.config.model.api.ServiceInfo; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.container.jdisc.HttpResponse; -import com.yahoo.restapi.HttpURL.Path; +import ai.vespa.http.HttpURL.Path; import com.yahoo.vespa.config.server.http.HttpFetcher; import com.yahoo.vespa.config.server.http.RequestTimeoutException; import com.yahoo.vespa.config.server.http.StaticResponse; diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java index 9f7e539a2e3..a4de818cb50 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java @@ -15,7 +15,7 @@ import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.jdisc.Response; import com.yahoo.jdisc.http.HttpRequest.Method; -import com.yahoo.restapi.HttpURL; +import ai.vespa.http.HttpURL; import com.yahoo.test.ManualClock; import com.yahoo.vespa.config.server.ApplicationRepository; import com.yahoo.vespa.config.server.MockLogRetriever; diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionCreateHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionCreateHandlerTest.java index 58a3593ae14..2e86f5e0538 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionCreateHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionCreateHandlerTest.java @@ -7,7 +7,7 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.TenantName; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; -import com.yahoo.restapi.HttpURL.Path; +import ai.vespa.http.HttpURL.Path; import com.yahoo.vespa.config.server.ApplicationRepository; import com.yahoo.vespa.config.server.MockProvisioner; import com.yahoo.vespa.config.server.application.CompressedApplicationInputStreamTest; diff --git a/configserver/src/test/java/com/yahoo/vespa/serviceview/StateRequestHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/serviceview/StateRequestHandlerTest.java index 35e63c46bf8..0e5bd085743 100644 --- a/configserver/src/test/java/com/yahoo/vespa/serviceview/StateRequestHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/serviceview/StateRequestHandlerTest.java @@ -3,7 +3,7 @@ package com.yahoo.vespa.serviceview; import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.jdisc.test.MockMetric; -import com.yahoo.restapi.HttpURL.Path; +import ai.vespa.http.HttpURL.Path; import com.yahoo.restapi.UriBuilder; import com.yahoo.vespa.serviceview.bindings.ApplicationView; import com.yahoo.vespa.serviceview.bindings.HealthClient; diff --git a/container-core/src/main/java/com/yahoo/restapi/Path.java b/container-core/src/main/java/com/yahoo/restapi/Path.java index c639432db89..f11c1b3189d 100644 --- a/container-core/src/main/java/com/yahoo/restapi/Path.java +++ b/container-core/src/main/java/com/yahoo/restapi/Path.java @@ -1,6 +1,8 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.restapi; +import ai.vespa.http.HttpURL; + import java.net.URI; import java.util.HashMap; import java.util.List; diff --git a/container-core/src/main/java/com/yahoo/restapi/RestApi.java b/container-core/src/main/java/com/yahoo/restapi/RestApi.java index 353ac3eb5cc..05528bc79e2 100644 --- a/container-core/src/main/java/com/yahoo/restapi/RestApi.java +++ b/container-core/src/main/java/com/yahoo/restapi/RestApi.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.restapi; +import ai.vespa.http.HttpURL; import com.fasterxml.jackson.databind.ObjectMapper; import com.yahoo.container.jdisc.AclMapping; import com.yahoo.container.jdisc.HttpRequest; diff --git a/container-core/src/main/java/com/yahoo/restapi/RestApiImpl.java b/container-core/src/main/java/com/yahoo/restapi/RestApiImpl.java index cc243a3e92b..ccc3dd49795 100644 --- a/container-core/src/main/java/com/yahoo/restapi/RestApiImpl.java +++ b/container-core/src/main/java/com/yahoo/restapi/RestApiImpl.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.restapi; +import ai.vespa.http.HttpURL; import com.fasterxml.jackson.databind.ObjectMapper; import com.yahoo.container.jdisc.AclMapping; import com.yahoo.container.jdisc.HttpRequest; diff --git a/container-core/src/test/java/com/yahoo/restapi/PathTest.java b/container-core/src/test/java/com/yahoo/restapi/PathTest.java index 4786eb9775c..17b35a6343c 100644 --- a/container-core/src/test/java/com/yahoo/restapi/PathTest.java +++ b/container-core/src/test/java/com/yahoo/restapi/PathTest.java @@ -4,6 +4,7 @@ package com.yahoo.restapi; import org.junit.Test; import java.net.URI; +import java.util.List; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertThrows; @@ -35,7 +36,7 @@ public class PathTest { assertTrue(path.matches("/a/{foo}/bar/{b}/{*}")); assertEquals("1", path.get("foo")); assertEquals("fuz", path.get("b")); - assertEquals("/", path.getRest().raw()); + assertEquals(List.of(), path.getRest().segments()); } { @@ -43,7 +44,7 @@ public class PathTest { assertTrue(path.matches("/a/{foo}/bar/{b}/{*}")); assertEquals("1", path.get("foo")); assertEquals("fuz", path.get("b")); - assertEquals("/kanoo", path.getRest().raw()); + assertEquals(List.of("kanoo"), path.getRest().segments()); } { @@ -51,7 +52,7 @@ public class PathTest { assertTrue(path.matches("/a/{foo}/bar/{b}/{*}")); assertEquals("1", path.get("foo")); assertEquals("fuz", path.get("b")); - assertEquals("/kanoo/trips", path.getRest().raw()); + assertEquals(List.of("kanoo", "trips"), path.getRest().segments()); } { @@ -59,7 +60,7 @@ public class PathTest { assertTrue(path.matches("/a/{foo}/bar/{b}/{*}")); assertEquals("1", path.get("foo")); assertEquals("fuz", path.get("b")); - assertEquals("/kanoo/trips/", path.getRest().raw()); + assertEquals(List.of("kanoo", "trips"), path.getRest().segments()); } } diff --git a/container-search/src/main/java/com/yahoo/prelude/query/parser/StructuredParser.java b/container-search/src/main/java/com/yahoo/prelude/query/parser/StructuredParser.java index a35ad91acbd..54a6e5b6e90 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/parser/StructuredParser.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/parser/StructuredParser.java @@ -160,8 +160,8 @@ abstract class StructuredParser extends AbstractParser { firstWord.add(tokens.next()); } - if (tokens.currentIsNoIgnore(DOT)) { - tokens.skip(); + while (tokens.currentIsNoIgnore(DOT)) { + secondWord.add(tokens.next()); if (tokens.currentIsNoIgnore(WORD) || tokens.currentIsNoIgnore(NUMBER)) { secondWord.add(tokens.next()); } else { @@ -177,11 +177,7 @@ abstract class StructuredParser extends AbstractParser { if ( ! tokens.skipNoIgnore(COLON)) return null; - if (secondWord.size() == 0) { - item = concatenate(firstWord); - } else { - item = concatenate(firstWord) + "." + concatenate(secondWord); - } + item = concatenate(firstWord) + concatenate(secondWord); item = indexFacts.getCanonicName(item); @@ -395,7 +391,7 @@ abstract class StructuredParser extends AbstractParser { if ( ! tokens.currentIs(NUMBER)) return null; item = new IntItem(">" + (negative ? "-" : "") + tokens.next() + decimalPart(), true); - item.setOrigin(new Substring(initial.substring.start, tokens.currentNoIgnore().substring.start, + item.setOrigin(new Substring(initial.substring.start, tokens.currentNoIgnore().substring.start, initial.getSubstring().getSuperstring())); // XXX: Unsafe end? return item; } finally { diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServer.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServer.java index 42368fa358d..37acc042c34 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServer.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServer.java @@ -4,8 +4,8 @@ package com.yahoo.vespa.hosted.controller.api.integration.configserver; import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.zone.ZoneId; -import com.yahoo.net.DomainName; -import com.yahoo.restapi.HttpURL.Path; +import ai.vespa.http.DomainName; +import ai.vespa.http.HttpURL.Path; import com.yahoo.vespa.flags.json.FlagData; import com.yahoo.vespa.hosted.controller.api.application.v4.model.ClusterMetrics; import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeploymentData; @@ -56,7 +56,8 @@ public interface ConfigServer { Map<?,?> getServiceApiResponse(DeploymentId deployment, String serviceName, Path restPath); - String getServiceStatusPage(DeploymentId deployment, String serviceName, DomainName node, Path subPath); + /** Returns a proxied response from a given path running on a given service and node */ + ProxyResponse getServiceNodePage(DeploymentId deployment, String serviceName, DomainName node, Path subPath); /** * Gets the Vespa logs of the given deployment. diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java index 6eaad63251c..84ff3d5d8c3 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java @@ -14,6 +14,10 @@ import com.yahoo.slime.Slime; import com.yahoo.slime.SlimeUtils; import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.curator.Lock; +import com.yahoo.vespa.curator.MultiplePathsLock; +import com.yahoo.vespa.flags.FlagSource; +import com.yahoo.vespa.flags.Flags; +import com.yahoo.vespa.flags.StringFlag; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.integration.archive.ArchiveBucket; @@ -111,6 +115,7 @@ public class CuratorDb { private final Curator curator; private final Duration tryLockTimeout; + private final StringFlag lockScheme; // For each application id (path), store the ZK node version and its deserialised data - update when version changes. // This will grow to keep all applications in memory, but this should be OK @@ -120,13 +125,14 @@ public class CuratorDb { private final Map<Path, Pair<Integer, NavigableMap<RunId, Run>>> cachedHistoricRuns = new ConcurrentHashMap<>(); @Inject - public CuratorDb(Curator curator) { - this(curator, defaultTryLockTimeout); + public CuratorDb(Curator curator, FlagSource flagSource) { + this(curator, defaultTryLockTimeout, flagSource); } - CuratorDb(Curator curator, Duration tryLockTimeout) { + CuratorDb(Curator curator, Duration tryLockTimeout, FlagSource flagSource) { this.curator = curator; this.tryLockTimeout = tryLockTimeout; + this.lockScheme = Flags.CONTROLLER_LOCK_SCHEME.bindTo(flagSource); } /** Returns all hostnames configured to be part of this ZooKeeper cluster */ @@ -144,19 +150,55 @@ public class CuratorDb { } public Lock lock(TenantAndApplicationId id) { - return curator.lock(lockPath(id), defaultLockTimeout.multipliedBy(2)); + switch (lockScheme.value()) { + case "BOTH": + return new MultiplePathsLock(lockPath(id), legacyLockPath(id), defaultLockTimeout.multipliedBy(2), curator); + case "OLD": + return curator.lock(legacyLockPath(id), defaultLockTimeout.multipliedBy(2)); + case "NEW": + return curator.lock(lockPath(id), defaultLockTimeout.multipliedBy(2)); + default: + throw new IllegalArgumentException("Unknown lock scheme " + lockScheme.value()); + } } public Lock lockForDeployment(ApplicationId id, ZoneId zone) { - return curator.lock(lockPath(id, zone), deployLockTimeout); + switch (lockScheme.value()) { + case "BOTH": + return new MultiplePathsLock(lockPath(id, zone), legacyLockPath(id, zone), deployLockTimeout, curator); + case "OLD": + return curator.lock(legacyLockPath(id, zone), deployLockTimeout); + case "NEW": + return curator.lock(lockPath(id, zone), deployLockTimeout); + default: + throw new IllegalArgumentException("Unknown lock scheme " + lockScheme.value()); + } } public Lock lock(ApplicationId id, JobType type) { - return curator.lock(lockPath(id, type), defaultLockTimeout); + switch (lockScheme.value()) { + case "BOTH": + return new MultiplePathsLock(lockPath(id, type), legacyLockPath(id, type), defaultLockTimeout, curator); + case "OLD": + return curator.lock(legacyLockPath(id, type), defaultLockTimeout); + case "NEW": + return curator.lock(lockPath(id, type), defaultLockTimeout); + default: + throw new IllegalArgumentException("Unknown lock scheme " + lockScheme.value()); + } } public Lock lock(ApplicationId id, JobType type, Step step) throws TimeoutException { - return tryLock(lockPath(id, type, step)); + switch (lockScheme.value()) { + case "BOTH": + return tryLock(lockPath(id, type, step), legacyLockPath(id, type, step)); + case "OLD": + return tryLock(legacyLockPath(id, type, step)); + case "NEW": + return tryLock(lockPath(id, type, step)); + default: + throw new IllegalArgumentException("Unknown lock scheme " + lockScheme.value()); + } } public Lock lockRotations() { @@ -239,6 +281,19 @@ public class CuratorDb { } } + /** Try locking with a low timeout, meaning it is OK to fail lock acquisition. + * + * Useful for maintenance jobs, where there is no point in running the jobs back to back. + */ + private Lock tryLock(Path path, Path path2) throws TimeoutException { + try { + return new MultiplePathsLock(path, path2, tryLockTimeout, curator); + } + catch (UncheckedTimeoutException e) { + throw new TimeoutException(e.getMessage()); + } + } + private <T> Optional<T> read(Path path, Function<byte[], T> mapper) { return curator.getData(path).filter(data -> data.length > 0).map(mapper); } @@ -667,32 +722,48 @@ public class CuratorDb { .append(tenant.value()); } - private Path lockPath(TenantAndApplicationId application) { + private Path legacyLockPath(TenantAndApplicationId application) { return lockPath(application.tenant()) .append(application.application().value()); } - private Path lockPath(ApplicationId instance) { - return lockPath(TenantAndApplicationId.from(instance)) + private Path legacyLockPath(ApplicationId instance) { + return legacyLockPath(TenantAndApplicationId.from(instance)) .append(instance.instance().value()); } - private Path lockPath(ApplicationId instance, ZoneId zone) { - return lockPath(instance) + private Path legacyLockPath(ApplicationId instance, ZoneId zone) { + return legacyLockPath(instance) .append(zone.environment().value()) .append(zone.region().value()); } - private Path lockPath(ApplicationId instance, JobType type) { - return lockPath(instance) + private Path legacyLockPath(ApplicationId instance, JobType type) { + return legacyLockPath(instance) .append(type.jobName()); } - private Path lockPath(ApplicationId instance, JobType type, Step step) { - return lockPath(instance, type) + private Path legacyLockPath(ApplicationId instance, JobType type, Step step) { + return legacyLockPath(instance, type) .append(step.name()); } + private Path lockPath(TenantAndApplicationId application) { + return lockRoot.append(application.tenant().value() + ":" + application.application().value()); + } + + private Path lockPath(ApplicationId instance, ZoneId zone) { + return lockRoot.append(instance.serializedForm() + ":" + zone.environment().value() + ":" + zone.region().value()); + } + + private Path lockPath(ApplicationId instance, JobType type) { + return lockRoot.append(instance.serializedForm() + ":" + type.jobName()); + } + + private Path lockPath(ApplicationId instance, JobType type, Step step) { + return lockRoot.append(instance.serializedForm() + ":" + type.jobName() + ":" + step.name()); + } + private Path lockPath(String provisionId) { return lockRoot .append(provisionStatePath()) diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/MockCuratorDb.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/MockCuratorDb.java index f98ecb80dd5..65c9859ad72 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/MockCuratorDb.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/MockCuratorDb.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.hosted.controller.persistence; import com.google.inject.Inject; import com.yahoo.vespa.curator.mock.MockCurator; +import com.yahoo.vespa.flags.InMemoryFlagSource; import java.time.Duration; @@ -31,7 +32,7 @@ public class MockCuratorDb extends CuratorDb { } public MockCuratorDb(MockCurator curator) { - super(curator, Duration.ofMillis(100)); + super(curator, Duration.ofMillis(100), new InMemoryFlagSource()); this.curator = curator; } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ProxyRequest.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ProxyRequest.java index 7e1c9c8884f..8b89c2300e4 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ProxyRequest.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ProxyRequest.java @@ -3,13 +3,12 @@ package com.yahoo.vespa.hosted.controller.proxy; import com.yahoo.container.jdisc.HttpRequest; -import com.yahoo.restapi.HttpURL; -import com.yahoo.restapi.HttpURL.Path; -import com.yahoo.restapi.HttpURL.Query; +import ai.vespa.http.HttpURL; +import ai.vespa.http.HttpURL.Path; +import ai.vespa.http.HttpURL.Query; import com.yahoo.text.Text; import java.io.InputStream; import java.net.URI; -import java.net.URISyntaxException; import java.util.List; import java.util.Map; import java.util.Objects; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ProxyResponse.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ProxyResponse.java index 9ac30898f8b..05cdc0d0565 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ProxyResponse.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ProxyResponse.java @@ -2,14 +2,12 @@ package com.yahoo.vespa.hosted.controller.proxy; import com.yahoo.container.jdisc.HttpResponse; -import com.yahoo.restapi.HttpURL; -import com.yahoo.restapi.HttpURL.Path; -import org.apache.http.client.utils.URIBuilder; +import ai.vespa.http.HttpURL; +import ai.vespa.http.HttpURL.Path; import java.io.IOException; import java.io.OutputStream; import java.net.URI; -import java.net.URISyntaxException; import java.nio.charset.StandardCharsets; /** diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index 0f5322af176..77ce51b838a 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -2,7 +2,6 @@ package com.yahoo.vespa.hosted.controller.restapi.application; import ai.vespa.hosted.api.Signatures; -import ai.vespa.validation.Validation; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Joiner; @@ -26,10 +25,10 @@ import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.container.jdisc.ThreadedHttpRequestHandler; import com.yahoo.io.IOUtils; -import com.yahoo.net.DomainName; +import ai.vespa.http.DomainName; import com.yahoo.restapi.ByteArrayResponse; import com.yahoo.restapi.ErrorResponse; -import com.yahoo.restapi.HttpURL; +import ai.vespa.http.HttpURL; import com.yahoo.restapi.MessageResponse; import com.yahoo.restapi.Path; import com.yahoo.restapi.ResourceResponse; @@ -70,9 +69,6 @@ import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.SourceRevision; import com.yahoo.vespa.hosted.controller.api.integration.noderepository.RestartFilter; -import com.yahoo.vespa.hosted.controller.api.integration.resource.MeteringData; -import com.yahoo.vespa.hosted.controller.api.integration.resource.ResourceAllocation; -import com.yahoo.vespa.hosted.controller.api.integration.resource.ResourceSnapshot; import com.yahoo.vespa.hosted.controller.api.integration.secrets.TenantSecretStore; import com.yahoo.vespa.hosted.controller.api.integration.user.User; import com.yahoo.vespa.hosted.controller.api.role.Role; @@ -158,7 +154,6 @@ import java.util.logging.Level; import java.util.stream.Collectors; import java.util.stream.Stream; -import static ai.vespa.validation.Validation.requireAtLeast; import static com.yahoo.jdisc.Response.Status.BAD_REQUEST; import static com.yahoo.jdisc.Response.Status.CONFLICT; import static com.yahoo.yolean.Exceptions.uncheck; @@ -274,7 +269,8 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/suspended")) return suspended(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/service")) return services(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/service/{service}/state/v1/{*}")) return service(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), path.get("service"), path.getRest(), request); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/service/{service}/{host}/status/{*}")) return status(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), path.get("service"), path.get("host"), path.getRest(), request); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/service/{service}/{host}/status/{*}")) return status(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), path.get("service"), path.get("host"), path.getRest()); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/service/{service}/{host}/state/v1/metrics")) return stateV1Metrics(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), path.get("service"), path.get("host")); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/nodes")) return nodes(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region")); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/clusters")) return clusters(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region")); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/content/{*}")) return content(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), path.getRest(), request); @@ -289,7 +285,7 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/suspended")) return suspended(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/service")) return services(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/service/{service}/state/v1/{*}")) return service(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), path.get("service"), path.getRest(), request); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/service/{service}/{host}/status/{*}")) return status(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), path.get("service"), path.get("host"), path.getRest(), request); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/service/{service}/{host}/status/{*}")) return status(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), path.get("service"), path.get("host"), path.getRest()); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/nodes")) return nodes(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region")); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/clusters")) return clusters(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region")); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/logs")) return logs(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request.propertyMap()); @@ -1800,13 +1796,16 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { return response; } - private HttpResponse status(String tenantName, String applicationName, String instanceName, String environment, String region, String serviceName, String host, HttpURL.Path restPath, HttpRequest request) { + private HttpResponse status(String tenantName, String applicationName, String instanceName, String environment, String region, String serviceName, String host, HttpURL.Path restPath) { DeploymentId deploymentId = new DeploymentId(ApplicationId.from(tenantName, applicationName, instanceName), requireZone(environment, region)); - String result = controller.serviceRegistry().configServer().getServiceStatusPage(deploymentId, - serviceName, - DomainName.of(host), - restPath); // TODO: add query - return new HtmlResponse(result); + return controller.serviceRegistry().configServer().getServiceNodePage( + deploymentId, serviceName, DomainName.of(host), HttpURL.Path.parse("/status").append(restPath)); // TODO: add query + } + + private HttpResponse stateV1Metrics(String tenantName, String applicationName, String instanceName, String environment, String region, String serviceName, String host) { + DeploymentId deploymentId = new DeploymentId(ApplicationId.from(tenantName, applicationName, instanceName), requireZone(environment, region)); + return controller.serviceRegistry().configServer().getServiceNodePage( + deploymentId, serviceName, DomainName.of(host), HttpURL.Path.parse("/state/v1/metrics")); } private HttpResponse service(String tenantName, String applicationName, String instanceName, String environment, String region, String serviceName, HttpURL.Path restPath, HttpRequest request) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ServiceApiResponse.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ServiceApiResponse.java index a21f93bdaea..41d891a0987 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ServiceApiResponse.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ServiceApiResponse.java @@ -4,13 +4,12 @@ package com.yahoo.vespa.hosted.controller.restapi.application; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.container.jdisc.HttpResponse; -import com.yahoo.restapi.HttpURL; -import com.yahoo.restapi.HttpURL.Path; -import com.yahoo.restapi.HttpURL.Query; +import ai.vespa.http.HttpURL; +import ai.vespa.http.HttpURL.Path; +import ai.vespa.http.HttpURL.Query; import com.yahoo.slime.Cursor; import com.yahoo.slime.JsonFormat; import com.yahoo.slime.Slime; -import com.yahoo.restapi.UriBuilder; import com.yahoo.vespa.serviceview.bindings.ApplicationView; import com.yahoo.vespa.serviceview.bindings.ClusterView; import com.yahoo.vespa.serviceview.bindings.ServiceView; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/configserver/ConfigServerApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/configserver/ConfigServerApiHandler.java index 27a8cbeaf3e..99632203645 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/configserver/ConfigServerApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/configserver/ConfigServerApiHandler.java @@ -6,7 +6,7 @@ import com.yahoo.config.provision.zone.ZoneList; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.restapi.ErrorResponse; -import com.yahoo.restapi.HttpURL; +import ai.vespa.http.HttpURL; import com.yahoo.restapi.Path; import com.yahoo.restapi.SlimeJsonResponse; import com.yahoo.slime.Cursor; @@ -25,7 +25,7 @@ import java.util.logging.Level; import java.util.stream.Collectors; import java.util.stream.Stream; -import static com.yahoo.restapi.HttpURL.Path.parse; +import static ai.vespa.http.HttpURL.Path.parse; /** * REST API for proxying operator APIs to config servers in a given zone. diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/ZoneApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/ZoneApiHandler.java index 40a402f209b..800588fdf8c 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/ZoneApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/ZoneApiHandler.java @@ -7,7 +7,7 @@ import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.container.jdisc.ThreadedHttpRequestHandler; import com.yahoo.restapi.ErrorResponse; -import com.yahoo.restapi.HttpURL; +import ai.vespa.http.HttpURL; import com.yahoo.restapi.Path; import com.yahoo.restapi.SlimeJsonResponse; import com.yahoo.slime.Cursor; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java index 1643f1f818b..4a451890bda 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java @@ -13,8 +13,8 @@ import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.zone.ZoneId; -import com.yahoo.net.DomainName; -import com.yahoo.restapi.HttpURL.Path; +import ai.vespa.http.DomainName; +import ai.vespa.http.HttpURL.Path; import com.yahoo.text.Text; import com.yahoo.vespa.flags.json.FlagData; import com.yahoo.vespa.hosted.controller.api.application.v4.model.ClusterMetrics; @@ -534,8 +534,8 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer } @Override - public String getServiceStatusPage(DeploymentId deployment, String serviceName, DomainName node, Path subPath) { - return "<h1>OK</h1>"; + public ProxyResponse getServiceNodePage(DeploymentId deployment, String serviceName, DomainName node, Path subPath) { + return new ProxyResponse("<h1>OK</h1>".getBytes(StandardCharsets.UTF_8), "text/html", 200); } @Override diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutorImplTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutorImplTest.java index e10a41cbae3..dd43f419624 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutorImplTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutorImplTest.java @@ -6,7 +6,7 @@ import com.github.tomakehurst.wiremock.stubbing.Scenario; import com.yahoo.config.provision.SystemName; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; -import com.yahoo.restapi.HttpURL.Path; +import ai.vespa.http.HttpURL.Path; import com.yahoo.vespa.hosted.controller.integration.ZoneRegistryMock; import com.yahoo.yolean.concurrent.Sleeper; import org.apache.http.protocol.HttpContext; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/proxy/ProxyRequestTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/proxy/ProxyRequestTest.java index e8b5df7efa1..5827ef676d7 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/proxy/ProxyRequestTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/proxy/ProxyRequestTest.java @@ -2,10 +2,8 @@ package com.yahoo.vespa.hosted.controller.proxy; import com.yahoo.jdisc.http.HttpRequest; -import com.yahoo.restapi.HttpURL.Path; -import org.junit.Rule; +import ai.vespa.http.HttpURL.Path; import org.junit.Test; -import org.junit.rules.ExpectedException; import java.net.URI; import java.util.List; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/proxy/ProxyResponseTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/proxy/ProxyResponseTest.java index 1ba0200eec3..599827f2d03 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/proxy/ProxyResponseTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/proxy/ProxyResponseTest.java @@ -2,7 +2,7 @@ package com.yahoo.vespa.hosted.controller.proxy; import com.yahoo.jdisc.http.HttpRequest; -import com.yahoo.restapi.HttpURL.Path; +import ai.vespa.http.HttpURL.Path; import org.junit.Test; import java.io.ByteArrayOutputStream; diff --git a/document/src/vespa/document/fieldvalue/iteratorhandler.h b/document/src/vespa/document/fieldvalue/iteratorhandler.h index 2f7b5d522e9..bbf24b77fb2 100644 --- a/document/src/vespa/document/fieldvalue/iteratorhandler.h +++ b/document/src/vespa/document/fieldvalue/iteratorhandler.h @@ -77,6 +77,7 @@ public: void setArrayIndex(uint32_t index) { _arrayIndexStack.back() = index; } ModificationStatus modify(FieldValue &fv) { return doModify(fv); } fieldvalue::VariableMap &getVariables() { return _variables; } + fieldvalue::VariableMap && stealVariables() { return std::move(_variables); } void setVariables(fieldvalue::VariableMap vars) { _variables = std::move(vars); } virtual bool createMissingPath() const { return false; } private: diff --git a/document/src/vespa/document/select/resultlist.cpp b/document/src/vespa/document/select/resultlist.cpp index e72eb36cbdb..e6ce79fe117 100644 --- a/document/src/vespa/document/select/resultlist.cpp +++ b/document/src/vespa/document/select/resultlist.cpp @@ -7,7 +7,8 @@ namespace document::select { ResultList::ResultList() = default; - +ResultList::ResultList(ResultList &&) noexcept = default; +ResultList & ResultList::operator = (ResultList &&) noexcept = default; ResultList::~ResultList() = default; ResultList::ResultList(const Result& result) { @@ -15,9 +16,9 @@ ResultList::ResultList(const Result& result) { } void -ResultList::add(const fieldvalue::VariableMap& variables, const Result& result) +ResultList::add(fieldvalue::VariableMap variables, const Result& result) { - _results.push_back(ResultPair(variables, &result)); + _results.emplace_back(std::move(variables), &result); } void @@ -109,7 +110,7 @@ ResultList::operator&&(const ResultList& other) const if (vars.empty()) { resultForNoVariables.set(result.toEnum()); } else { - results.add(vars, result); + results.add(std::move(vars), result); } } } @@ -137,7 +138,7 @@ ResultList::operator||(const ResultList& other) const if (vars.empty()) { resultForNoVariables.set(result.toEnum()); } else { - results.add(vars, result); + results.add(std::move(vars), result); } } } @@ -152,11 +153,11 @@ ResultList::operator||(const ResultList& other) const } ResultList -ResultList::operator!() const { +ResultList::operator!() && { ResultList result; - for (const auto & it : _results) { - result.add(it.first, !*it.second); + for (auto & it : _results) { + result.add(std::move(it.first), !*it.second); } return result; diff --git a/document/src/vespa/document/select/resultlist.h b/document/src/vespa/document/select/resultlist.h index b59a4f1a904..4099c1e39ba 100644 --- a/document/src/vespa/document/select/resultlist.h +++ b/document/src/vespa/document/select/resultlist.h @@ -14,11 +14,13 @@ public: typedef std::vector<ResultPair> Results; typedef Results::iterator iterator; typedef Results::const_iterator const_iterator; - using const_reverse_iterator = Results::const_reverse_iterator; + using reverse_iterator = Results::reverse_iterator; ResultList(); - ResultList(ResultList &&) = default; - ResultList & operator = (ResultList &&) = default; + ResultList(ResultList &&) noexcept; + ResultList & operator = (ResultList &&) noexcept; + ResultList(const ResultList &) = delete; + ResultList & operator = (const ResultList &) = delete; ~ResultList(); /** @@ -26,11 +28,11 @@ public: */ explicit ResultList(const Result& result); - void add(const VariableMap& variables, const Result& result); + void add(VariableMap variables, const Result& result); ResultList operator&&(const ResultList& other) const; ResultList operator||(const ResultList& other) const; - ResultList operator!() const; + ResultList operator!() &&; void print(std::ostream& out, bool verbose, const std::string& indent) const override; @@ -43,8 +45,8 @@ public: const Results& getResults() const { return _results; } const_iterator begin() const { return _results.begin(); } const_iterator end() const { return _results.end(); } - const_reverse_iterator rbegin() const { return _results.rbegin(); } - const_reverse_iterator rend() const { return _results.rend(); } + reverse_iterator rbegin() { return _results.rbegin(); } + reverse_iterator rend() { return _results.rend(); } private: Results _results; diff --git a/document/src/vespa/document/select/value.cpp b/document/src/vespa/document/select/value.cpp index aabd09b2558..a36a25103da 100644 --- a/document/src/vespa/document/select/value.cpp +++ b/document/src/vespa/document/select/value.cpp @@ -380,13 +380,29 @@ StructValue::operator==(const Value& value) const } void -StructValue::print(std::ostream& out, bool verbose, - const std::string& indent) const +StructValue::print(std::ostream& out, bool verbose, const std::string& indent) const { (void) verbose; (void) indent; out << "<no struct representation in language yet>"; } +namespace { + +fieldvalue::VariableMap +cloneMap(const fieldvalue::VariableMap &map) { + fieldvalue::VariableMap m; + for (const auto & item : map) { + if (item.second.key) { + m.emplace(item.first, fieldvalue::IndexValue(*item.second.key)); + } else { + m.emplace(item.first, fieldvalue::IndexValue(item.second.index)); + } + } + return m; +} + +} + template <typename Predicate> ResultList ArrayValue::doCompare(const Value& value, const Predicate& cmp) const @@ -415,7 +431,7 @@ ArrayValue::doCompare(const Value& value, const Predicate& cmp) const if (item.first.empty()) { resultForNoVariables.set(result.toEnum()); } else { - results.add(item.first, result); + results.add(cloneMap(item.first), result); } } for (uint32_t i(0); i < resultForNoVariables.size(); i++) { diff --git a/document/src/vespa/document/select/valuenodes.cpp b/document/src/vespa/document/select/valuenodes.cpp index 2a9604e7f43..1c7d47d0591 100644 --- a/document/src/vespa/document/select/valuenodes.cpp +++ b/document/src/vespa/document/select/valuenodes.cpp @@ -263,7 +263,7 @@ IteratorHandler::onPrimitive(uint32_t fid, const Content& fv) { if (!_firstValue && getVariables().empty()) { _firstValue = getInternalValue(fv.getValue()); } else { - _values.emplace_back(getVariables(), Value::SP(getInternalValue(fv.getValue()).release())); + _values.emplace_back(stealVariables(), Value::SP(getInternalValue(fv.getValue()).release())); } } diff --git a/functions.cmake b/functions.cmake index 65c02885ddb..b8ac3497ff2 100644 --- a/functions.cmake +++ b/functions.cmake @@ -428,6 +428,9 @@ function(vespa_add_test) if(NOT VALGRIND_EXECUTABLE) message(FATAL_ERROR "Requested valgrind tests, but could not find valgrind executable.") endif() + if(VESPA_USE_SANITIZER) + message(FATAL_ERROR "Cannot run sanitizer-instrumented unit tests under Valgrind") + endif() if(IS_SCRIPT) # For shell scripts, export a VALGRIND environment variable @@ -445,6 +448,9 @@ function(vespa_add_test) set(ARG_COMMAND "${VALGRIND_COMMAND} ${COMMAND_FIRST} ${COMMAND_REST}") endif() endif() + if(VESPA_USE_SANITIZER AND VESPA_SANITIZER_ENV) + list(APPEND ARG_ENVIRONMENT "${VESPA_SANITIZER_ENV}") + endif() separate_arguments(ARG_COMMAND) add_test(NAME ${ARG_NAME} COMMAND ${ARG_COMMAND} WORKING_DIRECTORY ${ARG_WORKING_DIRECTORY}) diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/ContainerData.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/ContainerData.java index 7e30ae8d649..10c62695f70 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/ContainerData.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/ContainerData.java @@ -34,6 +34,9 @@ public interface ContainerData { * @param symlink The path to the symlink inside the container * @param target The path to the target file for the symbolic link inside the container */ - void createSymlink(ContainerPath symlink, Path target); + void addSymlink(ContainerPath symlink, Path target); + + /** Writes all the files, directories and symlinks that were previously added */ + void converge(NodeAgentContext context); } diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/ContainerEngineMock.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/ContainerEngineMock.java index de3ae4cff98..2d3a4976fe5 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/ContainerEngineMock.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/ContainerEngineMock.java @@ -87,8 +87,13 @@ public class ContainerEngineMock implements ContainerEngine { } @Override - public void createSymlink(ContainerPath symlink, Path target) { - throw new UnsupportedOperationException("createSymlink not implemented"); + public void addSymlink(ContainerPath symlink, Path target) { + throw new UnsupportedOperationException("addSymlink not implemented"); + } + + @Override + public void converge(NodeAgentContext context) { + throw new UnsupportedOperationException("converge not implemented"); } }; } diff --git a/searchcore/src/tests/proton/common/cachedselect_test.cpp b/searchcore/src/tests/proton/common/cachedselect_test.cpp index d197d94fda4..2fd798cd8fd 100644 --- a/searchcore/src/tests/proton/common/cachedselect_test.cpp +++ b/searchcore/src/tests/proton/common/cachedselect_test.cpp @@ -61,8 +61,6 @@ using search::attribute::IAttributeContext; using search::attribute::test::MockAttributeManager; using vespalib::string; -using namespace search::index; - using IATint32 = IntegerAttributeTemplate<int32_t>; using IntEnumAttribute = EnumAttribute<IATint32>; using NodeUP = Node::UP; diff --git a/searchlib/src/tests/attribute/searchable/attributeblueprint_test.cpp b/searchlib/src/tests/attribute/searchable/attributeblueprint_test.cpp index 605a59b8a6a..192e137ddff 100644 --- a/searchlib/src/tests/attribute/searchable/attributeblueprint_test.cpp +++ b/searchlib/src/tests/attribute/searchable/attributeblueprint_test.cpp @@ -152,6 +152,16 @@ struct StringAttributeFiller { } }; +struct WsetStringAttributeFiller { + using ValueType = vespalib::string; + static void add(AttributeVector& attr, const vespalib::string& value) { + auto& real = downcast<StringAttribute>(attr); + uint32_t docid = attr.getNumDocs() - 1; + real.append(docid, value, 1); + real.commit(); + } +}; + struct IntegerAttributeFiller { using ValueType = int64_t; static void add(AttributeVector& attr, int64_t value) { @@ -183,6 +193,17 @@ make_string_attribute(const std::string& value) } AttributeVector::SP +make_wset_string_attribute(const std::string& value) +{ + Config cfg(BasicType::STRING, CollectionType::WSET); + // fast-search is needed to trigger use of DirectAttributeBlueprint. + cfg.setFastSearch(true); + auto attr = AttributeFactory::createAttribute(field, cfg); + fill<WsetStringAttributeFiller>(*attr, value); + return attr; +} + +AttributeVector::SP make_int_attribute(int64_t value) { Config cfg(BasicType::INT32, CollectionType::SINGLE); @@ -413,4 +434,17 @@ TEST(AttributeBlueprintTest, attribute_field_blueprint_wraps_filter_search_itera EXPECT_TRUE(wrapper.seek(2)); } +TEST(AttributeBlueprintTest, direct_attribute_blueprint_wraps_filter_search_iterator) +{ + BlueprintFactoryFixture f(make_wset_string_attribute("foo")); + SimpleStringTerm term("foo", field, 0, Weight(0)); + auto blueprint = f.create_blueprint(term); + + auto itr = blueprint->createFilterSearch(true, Blueprint::FilterConstraint::UPPER_BOUND); + auto& wrapper = downcast<FilterWrapper>(*itr); + wrapper.initRange(1, 3); + EXPECT_FALSE(wrapper.seek(1)); + EXPECT_TRUE(wrapper.seek(2)); +} + GTEST_MAIN_RUN_ALL_TESTS() diff --git a/searchlib/src/vespa/searchlib/attribute/CMakeLists.txt b/searchlib/src/vespa/searchlib/attribute/CMakeLists.txt index b3aa3bd958b..2e562b4a54f 100644 --- a/searchlib/src/vespa/searchlib/attribute/CMakeLists.txt +++ b/searchlib/src/vespa/searchlib/attribute/CMakeLists.txt @@ -65,6 +65,7 @@ vespa_add_library(searchlib_attribute OBJECT loadednumericvalue.cpp loadedvalue.cpp multi_numeric_enum_search_context.cpp + multi_numeric_flag_search_context.cpp multi_numeric_search_context.cpp multi_string_enum_search_context.cpp multi_string_enum_hint_search_context.cpp diff --git a/searchlib/src/vespa/searchlib/attribute/attribute_blueprint_factory.cpp b/searchlib/src/vespa/searchlib/attribute/attribute_blueprint_factory.cpp index f14966dbfc8..bde73faf466 100644 --- a/searchlib/src/vespa/searchlib/attribute/attribute_blueprint_factory.cpp +++ b/searchlib/src/vespa/searchlib/attribute/attribute_blueprint_factory.cpp @@ -562,6 +562,13 @@ public: return std::make_unique<queryeval::DocumentWeightSearchIterator>(*tfmda[0], _attr, _dict_entry); } + SearchIteratorUP createFilterSearch(bool strict, FilterConstraint constraint) const override { + (void) constraint; // We provide an iterator with exact results, so no need to take constraint into consideration. + auto wrapper = std::make_unique<FilterWrapper>(getState().numFields()); + wrapper->wrap(createLeafSearch(wrapper->tfmda(), strict)); + return wrapper; + } + void visitMembers(vespalib::ObjectVisitor &visitor) const override { LeafBlueprint::visitMembers(visitor); visit(visitor, "attribute", _attrName); diff --git a/searchlib/src/vespa/searchlib/attribute/attributeiterators.h b/searchlib/src/vespa/searchlib/attribute/attributeiterators.h index 8765e52d38a..d12c5a7eadc 100644 --- a/searchlib/src/vespa/searchlib/attribute/attributeiterators.h +++ b/searchlib/src/vespa/searchlib/attribute/attributeiterators.h @@ -333,7 +333,6 @@ template <typename SC> class FlagAttributeIteratorT : public FlagAttributeIterator { private: - using Attribute = typename SC::Attribute; void doSeek(uint32_t docId) override; protected: @@ -366,7 +365,6 @@ private: using FlagAttributeIteratorT<SC>::setDocId; using FlagAttributeIteratorT<SC>::setAtEnd; using FlagAttributeIteratorT<SC>::isAtEnd; - using Attribute = typename SC::Attribute; using Trinary=vespalib::Trinary; void doSeek(uint32_t docId) override; Trinary is_strict() const override { return Trinary::True; } diff --git a/searchlib/src/vespa/searchlib/attribute/attributeiterators.hpp b/searchlib/src/vespa/searchlib/attribute/attributeiterators.hpp index 8cde2862645..16b0c0da143 100644 --- a/searchlib/src/vespa/searchlib/attribute/attributeiterators.hpp +++ b/searchlib/src/vespa/searchlib/attribute/attributeiterators.hpp @@ -307,9 +307,8 @@ void FlagAttributeIteratorStrict<SC>::doSeek(uint32_t docId) { const SC & sc(_concreteSearchCtx); - const Attribute &attr = static_cast<const Attribute &>(sc.attribute()); for (int i = sc._low; (i <= sc._high); ++i) { - const BitVector * bv = attr.getBitVector(i); + const BitVector * bv = sc.get_bit_vector(i); if ((bv != nullptr) && !isAtEnd(docId) && bv->testBit(docId)) { setDocId(docId); return; @@ -318,7 +317,7 @@ FlagAttributeIteratorStrict<SC>::doSeek(uint32_t docId) uint32_t minNextBit(search::endDocId); for (int i = sc._low; (i <= sc._high); ++i) { - const BitVector * bv = attr.getBitVector(i); + const BitVector * bv = sc.get_bit_vector(i); if (bv != nullptr && !isAtEnd(docId)) { uint32_t nextBit = bv->getNextTrueBit(docId); minNextBit = std::min(nextBit, minNextBit); @@ -336,9 +335,8 @@ void FlagAttributeIteratorT<SC>::doSeek(uint32_t docId) { const SC & sc(_concreteSearchCtx); - const Attribute &attr = static_cast<const Attribute &>(sc.attribute()); for (int i = sc._low; (i <= sc._high); ++i) { - const BitVector * bv = attr.getBitVector(i); + const BitVector * bv = sc.get_bit_vector(i); if ((bv != nullptr) && !isAtEnd(docId) && bv->testBit(docId)) { setDocId(docId); return; @@ -351,9 +349,8 @@ void FlagAttributeIteratorT<SC>::or_hits_into(BitVector &result, uint32_t begin_id) { (void) begin_id; const SC & sc(_concreteSearchCtx); - const Attribute &attr = static_cast<const Attribute &>(sc.attribute()); for (int i = sc._low; (i <= sc._high); ++i) { - const BitVector * bv = attr.getBitVector(i); + const BitVector * bv = sc.get_bit_vector(i); if (bv != nullptr) { result.orWith(*bv); } @@ -364,9 +361,8 @@ template <typename SC> void FlagAttributeIteratorT<SC>::and_hits_into(BitVector &result, uint32_t begin_id) { const SC & sc(_concreteSearchCtx); - const Attribute &attr = static_cast<const Attribute &>(sc.attribute()); if (sc._low == sc._high) { - const BitVector * bv = attr.getBitVector(sc._low); + const BitVector * bv = sc.get_bit_vector(sc._low); if (bv != nullptr) { result.andWith(*bv); } else { @@ -383,18 +379,17 @@ template <typename SC> std::unique_ptr<BitVector> FlagAttributeIteratorT<SC>::get_hits(uint32_t begin_id) { const SC & sc(_concreteSearchCtx); - const Attribute &attr = static_cast<const Attribute &>(sc.attribute()); int i = sc._low; BitVector::UP result; for (;!result && i < sc._high; ++i) { - const BitVector * bv = attr.getBitVector(i); + const BitVector * bv = sc.get_bit_vector(i); if (bv != nullptr) { result = BitVector::create(*bv, begin_id, getEndId()); } } for (; i <= sc._high; ++i) { - const BitVector * bv = attr.getBitVector(i); + const BitVector * bv = sc.get_bit_vector(i); if (bv != nullptr) { result->orWith(*bv); } diff --git a/searchlib/src/vespa/searchlib/attribute/flagattribute.cpp b/searchlib/src/vespa/searchlib/attribute/flagattribute.cpp index ed72e54946e..17a898765d3 100644 --- a/searchlib/src/vespa/searchlib/attribute/flagattribute.cpp +++ b/searchlib/src/vespa/searchlib/attribute/flagattribute.cpp @@ -2,11 +2,9 @@ #include "flagattribute.h" #include "load_utils.hpp" -#include "attributeiterators.h" #include "multinumericattribute.hpp" - -#include <vespa/searchlib/queryeval/emptysearch.h> -#include <vespa/searchlib/common/bitvectoriterator.h> +#include "multi_numeric_flag_search_context.h" +#include <vespa/searchlib/common/bitvector.h> #include <vespa/log/log.h> LOG_SETUP(".searchlib.attribute.flag_attribute"); @@ -56,7 +54,7 @@ template <typename B> std::unique_ptr<attribute::SearchContext> FlagAttributeT<B>::getSearch(QueryTermSimple::UP qTerm, const attribute::SearchContextParams &) const { - return std::make_unique<SearchContext>(std::move(qTerm), *this, this->_mvMapping.make_read_view(this->getCommittedDocIdLimit())); + return std::make_unique<attribute::MultiNumericFlagSearchContext<typename B::BaseType, typename B::WType>>(std::move(qTerm), *this, this->_mvMapping.make_read_view(this->getCommittedDocIdLimit()), _bitVectors); } template <typename B> @@ -232,38 +230,6 @@ FlagAttributeT<B>::removeOldGenerations(vespalib::GenerationHandler::generation_ _bitVectorHolder.trimHoldLists(firstUsed); } -template <typename B> -FlagAttributeT<B>::SearchContext::SearchContext(QueryTermSimple::UP qTerm, const FlagAttributeT<B> & toBeSearched, MvMappingReadView mv_mapping_read_view) - : BaseSC(std::move(qTerm), toBeSearched, mv_mapping_read_view), - _zeroHits(false) -{ -} - -template <typename B> -SearchIterator::UP -FlagAttributeT<B>::SearchContext::createIterator(fef::TermFieldMatchData * matchData, bool strict) -{ - if (this->valid()) { - if (this->_low == this->_high) { - const Attribute & attr(static_cast<const Attribute &>(this->attribute())); - const BitVector * bv(attr.getBitVector(this->_low)); - if (bv != nullptr) { - return BitVectorIterator::create(bv, attr.getCommittedDocIdLimit(), *matchData, strict); - } else { - return std::make_unique<queryeval::EmptySearch>(); - } - } else { - SearchIterator::UP flagIterator( - strict - ? new FlagAttributeIteratorStrict<typename FlagAttributeT<B>::SearchContext>(*this, matchData) - : new FlagAttributeIteratorT<typename FlagAttributeT<B>::SearchContext>(*this, matchData)); - return flagIterator; - } - } else { - return std::make_unique<queryeval::EmptySearch>(); - } -} - template class FlagAttributeT<FlagBaseImpl>; } diff --git a/searchlib/src/vespa/searchlib/attribute/flagattribute.h b/searchlib/src/vespa/searchlib/attribute/flagattribute.h index 46fcf1592a9..24bec517eb0 100644 --- a/searchlib/src/vespa/searchlib/attribute/flagattribute.h +++ b/searchlib/src/vespa/searchlib/attribute/flagattribute.h @@ -7,7 +7,6 @@ namespace search { typedef MultiValueNumericAttribute< IntegerAttributeTemplate<int8_t>, multivalue::Value<int8_t> > FlagBaseImpl; -typedef MultiValueNumericAttribute< IntegerAttributeTemplate<int8_t>, multivalue::Value<int8_t> > HugeFlagBaseImpl; template <typename B> class FlagAttributeT : public B { @@ -15,22 +14,6 @@ public: FlagAttributeT(const vespalib::string & baseFileName, const AttributeVector::Config & cfg); private: typedef AttributeVector::DocId DocId; - using BaseSC = attribute::MultiNumericSearchContext<typename B::BaseType, typename B::WType>; - class SearchContext : public BaseSC { - public: - typedef FlagAttributeT<B> Attribute; - using MvMappingReadView = attribute::MultiValueMappingReadView<typename B::WType>; - SearchContext(std::unique_ptr<QueryTermSimple> qTerm, const FlagAttributeT<B> & toBeSearched, MvMappingReadView mv_mapping_read_view); - - std::unique_ptr<queryeval::SearchIterator> - createIterator(fef::TermFieldMatchData * matchData, bool strict) override; - - private: - bool _zeroHits; - - template <class SC> friend class FlagAttributeIteratorT; - template <class SC> friend class FlagAttributeIteratorStrict; - }; bool onLoad(vespalib::Executor *executor) override; bool onLoadEnumerated(ReaderBase &attrReader) override; std::unique_ptr<attribute::SearchContext> @@ -50,20 +33,14 @@ private: void resizeBitVectors(uint32_t neededSize); void removeOldGenerations(vespalib::GenerationHandler::generation_t firstUsed) override; uint32_t getOffset(int8_t value) const { return value + 128; } - BitVector * getBitVector(typename B::BaseType value) const { - return _bitVectors[value + 128]; - } vespalib::GenerationHolder _bitVectorHolder; std::vector<std::shared_ptr<BitVector> > _bitVectorStore; std::vector<BitVector *> _bitVectors; uint32_t _bitVectorSize; - template <class SC> friend class FlagAttributeIteratorT; - template <class SC> friend class FlagAttributeIteratorStrict; }; typedef FlagAttributeT<FlagBaseImpl> FlagAttribute; -typedef FlagAttributeT<HugeFlagBaseImpl> HugeFlagAttribute; } // namespace search diff --git a/searchlib/src/vespa/searchlib/attribute/multi_numeric_flag_search_context.cpp b/searchlib/src/vespa/searchlib/attribute/multi_numeric_flag_search_context.cpp new file mode 100644 index 00000000000..1c187c5dbd6 --- /dev/null +++ b/searchlib/src/vespa/searchlib/attribute/multi_numeric_flag_search_context.cpp @@ -0,0 +1,50 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "multi_numeric_flag_search_context.h" +#include "attributeiterators.hpp" +#include "attributevector.h" +#include <vespa/searchcommon/attribute/multivalue.h> +#include <vespa/searchlib/common/bitvectoriterator.h> +#include <vespa/searchlib/queryeval/emptysearch.h> +#include <vespa/searchlib/query/query_term_simple.h> + +namespace search::attribute { + +using queryeval::SearchIterator; + +template <typename T, typename M> +MultiNumericFlagSearchContext<T, M>::MultiNumericFlagSearchContext(std::unique_ptr<QueryTermSimple> qTerm, const AttributeVector& toBeSearched, MultiValueMappingReadView<M> mv_mapping_read_view, vespalib::ConstArrayRef<BitVector *> bit_vectors) + : MultiNumericSearchContext<T, M>(std::move(qTerm), toBeSearched, mv_mapping_read_view), + _bit_vectors(bit_vectors), + _zeroHits(false) +{ +} + +template <typename T, typename M> +std::unique_ptr<SearchIterator> +MultiNumericFlagSearchContext<T, M>::createIterator(fef::TermFieldMatchData* matchData, bool strict) +{ + if (this->valid()) { + if (this->_low == this->_high) { + const AttributeVector & attr = this->attribute(); + const BitVector * bv(get_bit_vector(this->_low)); + if (bv != nullptr) { + return BitVectorIterator::create(bv, attr.getCommittedDocIdLimit(), *matchData, strict); + } else { + return std::make_unique<queryeval::EmptySearch>(); + } + } else { + SearchIterator::UP flagIterator( + strict + ? new FlagAttributeIteratorStrict<MultiNumericFlagSearchContext>(*this, matchData) + : new FlagAttributeIteratorT<MultiNumericFlagSearchContext>(*this, matchData)); + return flagIterator; + } + } else { + return std::make_unique<queryeval::EmptySearch>(); + } +} + +template class MultiNumericFlagSearchContext<int8_t, multivalue::Value<int8_t>>; + +} diff --git a/searchlib/src/vespa/searchlib/attribute/multi_numeric_flag_search_context.h b/searchlib/src/vespa/searchlib/attribute/multi_numeric_flag_search_context.h new file mode 100644 index 00000000000..00f7077b2d0 --- /dev/null +++ b/searchlib/src/vespa/searchlib/attribute/multi_numeric_flag_search_context.h @@ -0,0 +1,39 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "multi_numeric_search_context.h" + +namespace search { +class BitVector; +template <class SC> class FlagAttributeIteratorT; +template <class SC> class FlagAttributeIteratorStrict; +} + +namespace search::attribute { + +/* + * MultiNumericFlagSearchContext handles the creation of search iterators for + * a query term on a multi value numeric flag attribute vector. + */ +template <typename T, typename M> +class MultiNumericFlagSearchContext : public MultiNumericSearchContext<T, M> +{ +public: + MultiNumericFlagSearchContext(std::unique_ptr<QueryTermSimple> qTerm, const AttributeVector& toBeSearched, MultiValueMappingReadView<M> mv_mapping_read_view, vespalib::ConstArrayRef<BitVector *> bit_vectors); + + std::unique_ptr<queryeval::SearchIterator> + createIterator(fef::TermFieldMatchData * matchData, bool strict) override; +private: + vespalib::ConstArrayRef<BitVector *> _bit_vectors; + bool _zeroHits; + const BitVector* get_bit_vector(T value) const { + static_assert(std::is_same_v<T, int8_t>, "Flag attribute search context is only supported for int8_t data type"); + return _bit_vectors[value + 128]; + } + + template <class SC> friend class ::search::FlagAttributeIteratorT; + template <class SC> friend class ::search::FlagAttributeIteratorStrict; +}; + +} diff --git a/searchlib/src/vespa/searchlib/attribute/multinumericattribute.hpp b/searchlib/src/vespa/searchlib/attribute/multinumericattribute.hpp index 99963094366..39e4f6866fb 100644 --- a/searchlib/src/vespa/searchlib/attribute/multinumericattribute.hpp +++ b/searchlib/src/vespa/searchlib/attribute/multinumericattribute.hpp @@ -4,13 +4,11 @@ #include "multinumericattribute.h" #include "multivalueattribute.hpp" #include "attributevector.hpp" -#include "attributeiterators.hpp" #include "multinumericattributesaver.h" #include "multi_numeric_search_context.h" #include "load_utils.h" #include "primitivereader.h" #include <vespa/searchlib/query/query_term_simple.h> -#include <vespa/searchlib/queryeval/emptysearch.h> #include <vespa/searchlib/util/fileutil.h> namespace search { diff --git a/searchlib/src/vespa/searchlib/attribute/multinumericenumattribute.hpp b/searchlib/src/vespa/searchlib/attribute/multinumericenumattribute.hpp index c35a2e55ec3..3323440dd0d 100644 --- a/searchlib/src/vespa/searchlib/attribute/multinumericenumattribute.hpp +++ b/searchlib/src/vespa/searchlib/attribute/multinumericenumattribute.hpp @@ -2,13 +2,11 @@ #pragma once -#include "attributeiterators.hpp" #include "load_utils.h" #include "loadednumericvalue.h" #include "multinumericenumattribute.h" #include "multi_numeric_enum_search_context.h" #include <vespa/searchlib/query/query_term_simple.h> -#include <vespa/searchlib/queryeval/emptysearch.h> #include <vespa/searchlib/util/fileutil.hpp> namespace search { diff --git a/searchlib/src/vespa/searchlib/attribute/singlenumericattribute.hpp b/searchlib/src/vespa/searchlib/attribute/singlenumericattribute.hpp index cf60183e2ca..917f0f55894 100644 --- a/searchlib/src/vespa/searchlib/attribute/singlenumericattribute.hpp +++ b/searchlib/src/vespa/searchlib/attribute/singlenumericattribute.hpp @@ -1,7 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include "attributeiterators.hpp" #include "attributevector.hpp" #include "load_utils.h" #include "numeric_matcher.h" @@ -11,7 +10,6 @@ #include "singlenumericattributesaver.h" #include "single_numeric_search_context.h" #include <vespa/searchlib/query/query_term_simple.h> -#include <vespa/searchlib/queryeval/emptysearch.h> namespace search { diff --git a/searchlib/src/vespa/searchlib/attribute/singlenumericenumattribute.hpp b/searchlib/src/vespa/searchlib/attribute/singlenumericenumattribute.hpp index f5219e2e8c7..aefc3c1cba3 100644 --- a/searchlib/src/vespa/searchlib/attribute/singlenumericenumattribute.hpp +++ b/searchlib/src/vespa/searchlib/attribute/singlenumericenumattribute.hpp @@ -2,7 +2,6 @@ #pragma once -#include "attributeiterators.hpp" #include "load_utils.h" #include "loadednumericvalue.h" #include "primitivereader.h" @@ -10,7 +9,6 @@ #include "singlenumericenumattribute.h" #include "single_numeric_enum_search_context.h" #include <vespa/searchlib/query/query_term_simple.h> -#include <vespa/searchlib/queryeval/emptysearch.h> #include <vespa/searchlib/util/fileutil.hpp> namespace search { diff --git a/vespajlib/abi-spec.json b/vespajlib/abi-spec.json index 642fdd5c16f..ae69564b671 100644 --- a/vespajlib/abi-spec.json +++ b/vespajlib/abi-spec.json @@ -349,20 +349,6 @@ ], "fields": [] }, - "com.yahoo.net.DomainName": { - "superClass": "ai.vespa.validation.PatternedStringWrapper", - "interfaces": [], - "attributes": [ - "public" - ], - "methods": [ - "public static com.yahoo.net.DomainName of(java.lang.String)", - "public static java.lang.String requireLabel(java.lang.String)" - ], - "fields": [ - "public static final com.yahoo.net.DomainName localhost" - ] - }, "com.yahoo.net.HostName": { "superClass": "ai.vespa.validation.PatternedStringWrapper", "interfaces": [], diff --git a/vespajlib/src/main/java/com/yahoo/net/DomainName.java b/vespajlib/src/main/java/ai/vespa/http/DomainName.java index ff8ba204674..a566f5b95be 100644 --- a/vespajlib/src/main/java/com/yahoo/net/DomainName.java +++ b/vespajlib/src/main/java/ai/vespa/http/DomainName.java @@ -1,5 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.net; +package ai.vespa.http; import ai.vespa.validation.PatternedStringWrapper; diff --git a/container-core/src/main/java/com/yahoo/restapi/HttpURL.java b/vespajlib/src/main/java/ai/vespa/http/HttpURL.java index e890b0fe71a..607d72f2550 100644 --- a/container-core/src/main/java/com/yahoo/restapi/HttpURL.java +++ b/vespajlib/src/main/java/ai/vespa/http/HttpURL.java @@ -1,8 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.restapi; +package ai.vespa.http; import ai.vespa.validation.StringWrapper; -import com.yahoo.net.DomainName; import java.net.URI; import java.net.URISyntaxException; @@ -116,10 +115,18 @@ public class HttpURL { return create(scheme, domain, port, path, query); } + public HttpURL appendPath(Path path) { + return create(scheme, domain, port, this.path.append(path), query); + } + public HttpURL withQuery(Query query) { return create(scheme, domain, port, path, query); } + public HttpURL mergeQuery(Query query) { + return create(scheme, domain, port, path, this.query.merge(query)); + } + public Scheme scheme() { return scheme; } @@ -285,7 +292,7 @@ public class HttpURL { } /** A raw path string which parses to this, by splitting on {@code "/"}, and then URL decoding. */ - String raw() { + private String raw() { StringJoiner joiner = new StringJoiner("/", "/", trailingSlash ? "/" : "").setEmptyValue(trailingSlash ? "/" : ""); for (String segment : segments) joiner.add(encode(segment, UTF_8)); return joiner.toString(); diff --git a/vespajlib/src/main/java/ai/vespa/http/package-info.java b/vespajlib/src/main/java/ai/vespa/http/package-info.java new file mode 100644 index 00000000000..e5600c6f82d --- /dev/null +++ b/vespajlib/src/main/java/ai/vespa/http/package-info.java @@ -0,0 +1,5 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package ai.vespa.http; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/vespajlib/src/main/java/ai/vespa/validation/StringWrapper.java b/vespajlib/src/main/java/ai/vespa/validation/StringWrapper.java index 258be824ef5..c3c44c0ff4c 100644 --- a/vespajlib/src/main/java/ai/vespa/validation/StringWrapper.java +++ b/vespajlib/src/main/java/ai/vespa/validation/StringWrapper.java @@ -5,7 +5,19 @@ import static java.util.Objects.requireNonNull; /** * Abstract wrapper for glorified strings, to ease adding new such wrappers. - * + * <p> + * <br> + * What's in a name?<br> + * That which we call a String<br> + * by any other name would smell as foul.<br> + * No? 'Tis not soft?<br> + * No ... I see it now!<br> + * Baptiz'd a-new, the String—<br> + * no more a String,<br> + * no less a {@link #value}—<br> + * it bringeth counsel<br> + * and is proof against their enmity.<br> + * </p> * @param <T> child type * * @author jonmv diff --git a/vespajlib/src/main/java/com/yahoo/net/HostName.java b/vespajlib/src/main/java/com/yahoo/net/HostName.java index 47bd8246bb3..20f1008055e 100644 --- a/vespajlib/src/main/java/com/yahoo/net/HostName.java +++ b/vespajlib/src/main/java/com/yahoo/net/HostName.java @@ -1,14 +1,16 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.net; +import ai.vespa.http.DomainName; import ai.vespa.validation.PatternedStringWrapper; import java.util.Optional; +import java.util.regex.Pattern; import static ai.vespa.validation.Validation.requireLength; /** - * Hostnames match {@link DomainName#domainNamePattern}, but are restricted to 64 characters in length. + * Hostnames match {@link #hostNamePattern}, and are restricted to 64 characters in length. * * This class also has utilities for getting the hostname of the system running the JVM. * Detection of the hostname is now done before starting any Vespa @@ -20,10 +22,13 @@ import static ai.vespa.validation.Validation.requireLength; */ public class HostName extends PatternedStringWrapper<HostName> { + static final Pattern labelPattern = Pattern.compile("([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9-]{0,61}[A-Za-z0-9])"); + static final Pattern hostNamePattern = Pattern.compile("(" + labelPattern + "\\.)*" + labelPattern); + private static HostName preferredHostName = null; private HostName(String value) { - super(requireLength(value, "hostname length", 1, 64), DomainName.domainNamePattern, "hostname"); + super(requireLength(value, "hostname length", 1, 64), hostNamePattern, "hostname"); } public static HostName of(String value) { diff --git a/vespajlib/src/main/java/com/yahoo/net/UriTools.java b/vespajlib/src/main/java/com/yahoo/net/UriTools.java index 55a94884a57..58fa2825a14 100644 --- a/vespajlib/src/main/java/com/yahoo/net/UriTools.java +++ b/vespajlib/src/main/java/com/yahoo/net/UriTools.java @@ -6,7 +6,7 @@ import java.net.URI; /** * Utility methods for working with URIs. * - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + * @author Steinar Knutsen */ public final class UriTools { private UriTools() { diff --git a/vespajlib/src/test/java/com/yahoo/net/DomainNameTest.java b/vespajlib/src/test/java/ai/vespa/http/DomainNameTest.java index d8e76b71d7e..aa19e8667f6 100644 --- a/vespajlib/src/test/java/com/yahoo/net/DomainNameTest.java +++ b/vespajlib/src/test/java/ai/vespa/http/DomainNameTest.java @@ -1,7 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.net; +package ai.vespa.http; -import com.yahoo.net.DomainName; import org.junit.jupiter.api.Test; import static org.junit.jupiter.api.Assertions.assertThrows; diff --git a/container-core/src/test/java/com/yahoo/restapi/HttpURLTest.java b/vespajlib/src/test/java/ai/vespa/http/HttpURLTest.java index 858513c2a69..181e02999b8 100644 --- a/container-core/src/test/java/com/yahoo/restapi/HttpURLTest.java +++ b/vespajlib/src/test/java/ai/vespa/http/HttpURLTest.java @@ -1,9 +1,9 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.restapi; +package ai.vespa.http; +import ai.vespa.http.HttpURL.Path; import ai.vespa.validation.Name; -import com.yahoo.net.DomainName; -import com.yahoo.restapi.HttpURL.Query; +import ai.vespa.http.HttpURL.Query; import org.junit.jupiter.api.Test; import java.net.URI; @@ -14,9 +14,9 @@ import java.util.Map; import java.util.OptionalInt; import java.util.function.Consumer; -import static com.yahoo.net.DomainName.localhost; -import static com.yahoo.restapi.HttpURL.Scheme.http; -import static com.yahoo.restapi.HttpURL.Scheme.https; +import static ai.vespa.http.DomainName.localhost; +import static ai.vespa.http.HttpURL.Scheme.http; +import static ai.vespa.http.HttpURL.Scheme.https; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; @@ -43,11 +43,11 @@ class HttpURLTest { @Test void testModification() { - HttpURL url = HttpURL.create(http, localhost).withPath(HttpURL.Path.empty(Name::of)); + HttpURL url = HttpURL.create(http, localhost).withPath(Path.empty(Name::of)); assertEquals(http, url.scheme()); assertEquals(localhost, url.domain()); assertEquals(OptionalInt.empty(), url.port()); - assertEquals(HttpURL.Path.empty(Name::of), url.path()); + assertEquals(Path.empty(Name::of), url.path()); assertEquals(HttpURL.Query.empty(Name::of), url.query()); url = url.withScheme(https) @@ -58,7 +58,7 @@ class HttpURLTest { assertEquals(https, url.scheme()); assertEquals(DomainName.of("domain"), url.domain()); assertEquals(OptionalInt.of(0), url.port()); - assertEquals(HttpURL.Path.parse("/foo", Name::of), url.path()); + assertEquals(Path.parse("/foo", Name::of), url.path()); assertEquals(HttpURL.Query.parse("boo=bar&baz", Name::of), url.query()); } @@ -107,7 +107,7 @@ class HttpURLTest { @Test void testPath() { - HttpURL.Path path = HttpURL.Path.parse("foo/bar/baz", Name::of); + Path path = Path.parse("foo/bar/baz", Name::of); List<String> expected = List.of("foo", "bar", "baz"); assertEquals(expected, path.segments()); @@ -119,7 +119,7 @@ class HttpURLTest { assertEquals(path, path.withoutTrailingSlash().withoutTrailingSlash()); assertEquals(List.of("one", "foo", "bar", "baz", "two"), - HttpURL.Path.from(List.of("one")).append(path).append("two").segments()); + Path.from(List.of("one")).append(path).append("two").segments()); assertEquals(List.of(expected.get(2), expected.get(0)), path.append(path).cut(2).skip(2).segments()); @@ -147,19 +147,19 @@ class HttpURLTest { assertEquals("path segment decoded cannot contain '/', but got: '/'", assertThrows(IllegalArgumentException.class, - () -> HttpURL.Path.empty().append("%2525252525252525%2525252525253%25252532%252525%252534%36")).getMessage()); + () -> Path.empty().append("%2525252525252525%2525252525253%25252532%252525%252534%36")).getMessage()); assertEquals("path segment decoded cannot contain '?', but got: '?'", assertThrows(IllegalArgumentException.class, - () -> HttpURL.Path.empty().append("?")).getMessage()); + () -> Path.empty().append("?")).getMessage()); assertEquals("path segment decoded cannot contain '#', but got: '#'", assertThrows(IllegalArgumentException.class, - () -> HttpURL.Path.empty().append("#")).getMessage()); + () -> Path.empty().append("#")).getMessage()); assertEquals("path segments cannot be \"\", \".\", or \"..\", but got: '..'", assertThrows(IllegalArgumentException.class, - () -> HttpURL.Path.empty().append("%2E%25252E")).getMessage()); + () -> Path.empty().append("%2E%25252E")).getMessage()); } @Test diff --git a/zkfacade/src/main/java/com/yahoo/vespa/curator/Lock.java b/zkfacade/src/main/java/com/yahoo/vespa/curator/Lock.java index 56ae65bb317..3f9c52594a3 100644 --- a/zkfacade/src/main/java/com/yahoo/vespa/curator/Lock.java +++ b/zkfacade/src/main/java/com/yahoo/vespa/curator/Lock.java @@ -69,6 +69,9 @@ public class Lock implements Mutex { throw new RuntimeException("Exception releasing lock '" + lockPath + "'"); } } + + protected String lockPath() { return lockPath; } + } diff --git a/zkfacade/src/main/java/com/yahoo/vespa/curator/MultiplePathsLock.java b/zkfacade/src/main/java/com/yahoo/vespa/curator/MultiplePathsLock.java new file mode 100644 index 00000000000..9576f5c0385 --- /dev/null +++ b/zkfacade/src/main/java/com/yahoo/vespa/curator/MultiplePathsLock.java @@ -0,0 +1,41 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.curator; + +import com.yahoo.path.Path; + +import java.time.Duration; +import java.util.logging.Level; +import java.util.logging.Logger; + +/** + * Class that holds two locks, originally used for transitioning from one lock to + * another, where you need to hold both the old lock and the new lock in the + * transition period. Locks are acquired in constructor. + * + * @author hmusum + */ +public class MultiplePathsLock extends Lock { + + private static final Logger log = Logger.getLogger(MultiplePathsLock.class.getName()); + + private final Lock oldLock; + + public MultiplePathsLock(Path newLockPath, Path oldLockPath, Duration timeout, Curator curator) { + super(newLockPath.getAbsolute(), curator); + log.log(Level.INFO, "Acquiring lock " + oldLockPath); + this.oldLock = curator.lock(oldLockPath, timeout);; + log.log(Level.INFO, "Acquiring lock " + lockPath()); + super.acquire(timeout); + } + + @Override + public void close() { + log.log(Level.INFO, "Closing lock " + oldLock.lockPath()); + oldLock.close(); + log.log(Level.INFO, "Closing lock " + lockPath()); + super.close(); + } + +} + + |