diff options
Diffstat (limited to 'vespaclient-container-plugin')
24 files changed, 334 insertions, 103 deletions
diff --git a/vespaclient-container-plugin/pom.xml b/vespaclient-container-plugin/pom.xml index c960c2cca44..5c3a04beeb8 100644 --- a/vespaclient-container-plugin/pom.xml +++ b/vespaclient-container-plugin/pom.xml @@ -8,11 +8,11 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>vespaclient-container-plugin</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <packaging>container-plugin</packaging> <dependencies> <dependency> @@ -38,42 +38,6 @@ <scope>provided</scope> </dependency> <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>vespa-http-client</artifactId> - <version>${project.version}</version> - <exclusions> - <!-- Exclude artifacts that are provided by Jdisc container --> - <exclusion> - <groupId>com.fasterxml.jackson.core</groupId> - <artifactId>*</artifactId> - </exclusion> - <exclusion> - <groupId>com.fasterxml.jackson.datatype</groupId> - <artifactId>*</artifactId> - </exclusion> - <exclusion> - <groupId>com.fasterxml.jackson.module</groupId> - <artifactId>*</artifactId> - </exclusion> - <exclusion> - <groupId>com.google.guava</groupId> - <artifactId>guava</artifactId> - </exclusion> - <exclusion> - <groupId>com.yahoo.vespa</groupId> - <artifactId>security-utils</artifactId> - </exclusion> - <exclusion> - <groupId>org.apache.httpcomponents</groupId> - <artifactId>*</artifactId> - </exclusion> - <exclusion> - <groupId>org.bouncycastle</groupId> - <artifactId>*</artifactId> - </exclusion> - </exclusions> - </dependency> - <dependency> <groupId>org.mockito</groupId> <artifactId>mockito-core</artifactId> <scope>test</scope> diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/DocumentV1ApiHandler.java b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/DocumentV1ApiHandler.java index d3f8bb40b04..7f940e5695a 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/DocumentV1ApiHandler.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/DocumentV1ApiHandler.java @@ -22,6 +22,7 @@ import com.yahoo.document.TestAndSetCondition; import com.yahoo.document.config.DocumentmanagerConfig; import com.yahoo.document.fieldset.AllFields; import com.yahoo.document.fieldset.DocIdOnly; +import com.yahoo.document.fieldset.DocumentOnly; import com.yahoo.document.idstring.IdIdString; import com.yahoo.document.json.DocumentOperationType; import com.yahoo.document.json.JsonReader; @@ -377,8 +378,7 @@ public class DocumentV1ApiHandler extends AbstractRequestHandler { StorageCluster destination = resolveCluster(Optional.of(requireProperty(request, DESTINATION_CLUSTER)), clusters); VisitorParameters parameters = parseParameters(request, path); parameters.setRemoteDataHandler("[Content:cluster=" + destination.name() + "]"); // Bypass indexing. - // TODO Vespa 8: change to DocumentOnly.NAME - parameters.setFieldSet(AllFields.NAME); + parameters.setFieldSet(DocumentOnly.NAME); return () -> { visitWithRemote(request, parameters, handler); return true; // VisitorSession has its own throttle handling. @@ -627,7 +627,7 @@ public class DocumentV1ApiHandler extends AbstractRequestHandler { /** Creates a new JsonResponse with path field written. */ static JsonResponse create(HttpRequest request, ResponseHandler handler) throws IOException { - JsonResponse response = new JsonResponse(handler); + JsonResponse response = new JsonResponse(handler, request); response.writePathId(request.getUri().getRawPath()); return response; } @@ -713,12 +713,17 @@ public class DocumentV1ApiHandler extends AbstractRequestHandler { } } - synchronized void writeSingleDocument(Document document) throws IOException { - boolean tensorShortForm = false; - if (request != null && request.parameters().containsKey("format.tensors")) { - tensorShortForm = request.parameters().get("format.tensors").contains("short"); + private boolean tensorShortForm() { + if (request != null && + request.parameters().containsKey("format.tensors") && + request.parameters().get("format.tensors").contains("long")) { + return false; } - new JsonWriter(json, tensorShortForm).writeFields(document); + return true; // default + } + + synchronized void writeSingleDocument(Document document) throws IOException { + new JsonWriter(json, tensorShortForm()).writeFields(document); } synchronized void writeDocumentsArrayStart() throws IOException { @@ -737,7 +742,7 @@ public class DocumentV1ApiHandler extends AbstractRequestHandler { ByteArrayOutputStream myOut = new ByteArrayOutputStream(1); myOut.write(','); // Prepend rather than append, to avoid double memory copying. try (JsonGenerator myJson = jsonFactory.createGenerator(myOut)) { - new JsonWriter(myJson).write(document); + new JsonWriter(myJson, tensorShortForm()).write(document); } docs.add(myOut); @@ -1102,8 +1107,8 @@ public class DocumentV1ApiHandler extends AbstractRequestHandler { throw new IllegalArgumentException("Must set 'cluster' parameter to a valid content cluster id when visiting at a root /document/v1/ level"); VisitorParameters parameters = parseCommonParameters(request, path, cluster); - // TODO Vespa 8: change to DocumentOnly.NAME - parameters.setFieldSet(getProperty(request, FIELD_SET).orElse(path.documentType().map(type -> type + ":[document]").orElse(AllFields.NAME))); + // TODO can the else-case be safely reduced to always be DocumentOnly.NAME? + parameters.setFieldSet(getProperty(request, FIELD_SET).orElse(path.documentType().map(type -> type + ":[document]").orElse(DocumentOnly.NAME))); parameters.setMaxTotalHits(wantedDocumentCount); parameters.visitInconsistentBuckets(true); long timeoutMs = Math.max(1, request.getTimeout(MILLISECONDS) - handlerTimeout.toMillis()); @@ -1402,11 +1407,14 @@ public class DocumentV1ApiHandler extends AbstractRequestHandler { @Override public ContentChannel handleResponse(Response response) { - switch (response.getStatus() / 100) { - case 2: metrics.reportSuccessful(type, start); break; - case 4: metrics.reportFailure(type, DocumentOperationStatus.REQUEST_ERROR); break; - case 5: metrics.reportFailure(type, DocumentOperationStatus.SERVER_ERROR); break; - } + var statusCodeGroup = response.getStatus() / 100; + // Status code 412 - condition not met - is considered OK + if (statusCodeGroup == 2 || response.getStatus() == 412) + metrics.reportSuccessful(type, start); + else if (statusCodeGroup == 4) + metrics.reportFailure(type, DocumentOperationStatus.REQUEST_ERROR); + else if (statusCodeGroup == 5) + metrics.reportFailure(type, DocumentOperationStatus.SERVER_ERROR); return delegate.handleResponse(response); } diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/ClientFeederV3.java b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/ClientFeederV3.java index 784575cd001..8ea9234009d 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/ClientFeederV3.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/ClientFeederV3.java @@ -4,7 +4,6 @@ package com.yahoo.vespa.http.server; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.document.DocumentTypeManager; -import com.yahoo.documentapi.messagebus.protocol.DocumentMessage; import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol; import com.yahoo.jdisc.Metric; import com.yahoo.jdisc.ReferencedResource; @@ -14,9 +13,6 @@ import com.yahoo.messagebus.ReplyHandler; import com.yahoo.messagebus.Result; import com.yahoo.messagebus.shared.SharedSourceSession; import com.yahoo.net.HostName; -import com.yahoo.vespa.http.client.core.ErrorCode; -import com.yahoo.vespa.http.client.core.Headers; -import com.yahoo.vespa.http.client.core.OperationStatus; import com.yahoo.vespaxmlparser.FeedOperation; import com.yahoo.yolean.Exceptions; @@ -240,7 +236,6 @@ class ClientFeederV3 { return message; } - @SuppressWarnings("removal") // TODO: Remove on Vespa 8 private void setMessageParameters(DocumentOperationMessageV3 msg, FeederSettings settings) { msg.getMessage().setContext(new ReplyContext(msg.getOperationId(), feedReplies)); if (settings.traceLevel != null) { @@ -249,9 +244,6 @@ class ClientFeederV3 { if (settings.priority != null) { try { DocumentProtocol.Priority priority = DocumentProtocol.Priority.valueOf(settings.priority); - if (msg.getMessage() instanceof DocumentMessage) { - ((DocumentMessage) msg.getMessage()).setPriority(priority); // TODO: Remove on Vespa 8 - } } catch (IllegalArgumentException i) { log.severe(i.getMessage()); diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/ClientState.java b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/ClientState.java index 13a12f707d9..973c154b336 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/ClientState.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/ClientState.java @@ -4,14 +4,13 @@ package com.yahoo.vespa.http.server; import com.yahoo.jdisc.Metric; import com.yahoo.jdisc.ReferencedResource; import com.yahoo.messagebus.shared.SharedSourceSession; -import com.yahoo.vespa.http.client.core.OperationStatus; import java.util.concurrent.BlockingQueue; /** * The state of a client session, used to save replies when client disconnects. * - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + * @author Steinar Knutsen */ public class ClientState { diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/Encoder.java b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/Encoder.java new file mode 100644 index 00000000000..65fb1223c7d --- /dev/null +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/Encoder.java @@ -0,0 +1,100 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.http.server; + +/** + * Simple encoding scheme to remove space, linefeed, control characters and + * anything outside ISO 646.irv:1991 from strings. The scheme is supposed to be + * human readable and debugging friendly. Opening and closing curly braces are + * used as quoting characters, the output is by definition US-ASCII only + * characters. + * + * @author Steinar Knutsen + */ +final class Encoder { + + /** + * ISO 646.irv:1991 safe quoting into a StringBuilder instance. + * + * @param input the string to encode + * @param output the destination buffer + * @return the destination buffer given as input + */ + public static StringBuilder encode(String input, StringBuilder output) { + for (int i = 0; i < input.length(); i = input.offsetByCodePoints(i, 1)) { + int c = input.codePointAt(i); + if (c <= '~') { + if (c <= ' ') { + encode(c, output); + } else { + switch (c) { + case '{': + case '}': + encode(c, output); + break; + default: + output.append((char) c); + } + } + } else { + encode(c, output); + } + } + return output; + } + + /** + * ISO 646.irv:1991 safe unquoting into a StringBuilder instance. + * + * @param input the string to decode + * @param output the destination buffer + * @return the destination buffer given as input + * @throws IllegalArgumentException if the input string contains unexpected or invalid data + */ + public static StringBuilder decode(String input, StringBuilder output) { + for (int i = 0; i < input.length(); i = input.offsetByCodePoints(i, 1)) { + int c = input.codePointAt(i); + if (c > '~') + throw new IllegalArgumentException("Input contained character above printable ASCII at position " + i); + if (c == '{') + i = decode(input, i, output); + else + output.append((char) c); + } + return output; + } + + private static int decode(String input, int offset, StringBuilder output) { + char c = 0; + int end = offset; + int start = offset + 1; + int codePoint; + + while ('}' != c) { + if (++end >= input.length()) { + throw new IllegalArgumentException("Unterminated quoted character or empty quoting."); + } + c = input.charAt(end); + } + try { + codePoint = Integer.parseInt(input.substring(start, end), 16); + } catch (NumberFormatException e) { + throw new IllegalArgumentException("Unexpected quoted data: [" + input.substring(start, end) + "]", e); + } + if (Character.charCount(codePoint) > 1) { + try { + output.append(Character.toChars(codePoint)); + } catch (IllegalArgumentException e) { + throw new IllegalArgumentException("Unexpected quoted data: [" + input.substring(start, end) + "]", e); + } + } else { + output.append((char) codePoint); + } + return end; + + } + + private static void encode(int c, StringBuilder output) { + output.append("{").append(Integer.toHexString(c)).append("}"); + } + +} diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/ErrorCode.java b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/ErrorCode.java new file mode 100644 index 00000000000..f819ecccbb1 --- /dev/null +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/ErrorCode.java @@ -0,0 +1,33 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.http.server; + +/** + * Return types for the server. + * + * @author Einar M R Rosenvinge + * @author Steinar Knutsen + */ +enum ErrorCode { + + OK(true, true), + ERROR(false, false), + TRANSIENT_ERROR(false, true), + END_OF_FEED(true, true); + + private final boolean success; + private final boolean _transient; + + ErrorCode(boolean success, boolean _transient) { + this.success = success; + this._transient = _transient; + } + + public boolean isSuccess() { + return success; + } + + public boolean isTransient() { + return _transient; + } + +} diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedHandler.java b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedHandler.java index f99274d3f2b..74665d60a04 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedHandler.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedHandler.java @@ -15,7 +15,6 @@ import com.yahoo.jdisc.Response; import com.yahoo.jdisc.handler.ResponseHandler; import com.yahoo.messagebus.ReplyHandler; import com.yahoo.metrics.simple.MetricReceiver; -import com.yahoo.vespa.http.client.core.Headers; import javax.inject.Inject; import java.io.IOException; diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedHandlerV3.java b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedHandlerV3.java index c8828df6d54..f9ae04623e6 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedHandlerV3.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedHandlerV3.java @@ -13,7 +13,6 @@ import com.yahoo.jdisc.ReferencedResource; import com.yahoo.messagebus.ReplyHandler; import com.yahoo.messagebus.SourceSessionParams; import com.yahoo.messagebus.shared.SharedSourceSession; -import com.yahoo.vespa.http.client.core.Headers; import com.yahoo.yolean.Exceptions; import java.util.HashMap; diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedParams.java b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedParams.java new file mode 100644 index 00000000000..6ce20cdec53 --- /dev/null +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedParams.java @@ -0,0 +1,23 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.http.server; + +import java.util.concurrent.TimeUnit; + +/** + * Feed level parameters. + * + * @author Einar M R Rosenvinge + */ +public final class FeedParams { + + /** + * Enumeration of data formats that are acceptable by the + * FeedClient methods. + */ + public enum DataFormat { + /** UTF-8-encoded XML. Preamble is not necessary. */ + XML_UTF8, + JSON_UTF8 + } + +} diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedReaderFactory.java b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedReaderFactory.java index 069ccfd84f0..f96b650748d 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedReaderFactory.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedReaderFactory.java @@ -4,7 +4,6 @@ package com.yahoo.vespa.http.server; import com.yahoo.document.DocumentTypeManager; import com.yahoo.document.json.JsonFeedReader; import com.yahoo.text.Utf8; -import com.yahoo.vespa.http.client.config.FeedParams; import com.yahoo.vespaxmlparser.FeedReader; import com.yahoo.vespaxmlparser.VespaXMLFeedReader; diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedReplyReader.java b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedReplyReader.java index 2fbb80d9fcc..1422ec10b08 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedReplyReader.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedReplyReader.java @@ -11,17 +11,11 @@ import com.yahoo.jdisc.Metric; import com.yahoo.messagebus.Reply; import com.yahoo.messagebus.ReplyHandler; import com.yahoo.messagebus.Trace; -import com.yahoo.vespa.http.client.core.ErrorCode; -import com.yahoo.vespa.http.client.core.OperationStatus; import java.util.Map; -import java.util.Optional; -import java.util.function.Predicate; import java.util.logging.Level; import java.util.logging.Logger; -import static java.util.function.Predicate.not; - /** * Catch message bus replies and make the available to a given session. * diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedResponse.java b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedResponse.java index 3e2a4a8795f..1da8aded27b 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedResponse.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedResponse.java @@ -2,9 +2,6 @@ package com.yahoo.vespa.http.server; import com.yahoo.container.jdisc.HttpResponse; -import com.yahoo.vespa.http.client.core.Headers; -import com.yahoo.vespa.http.client.core.ErrorCode; -import com.yahoo.vespa.http.client.core.OperationStatus; import java.io.IOException; import java.io.OutputStream; @@ -15,8 +12,7 @@ import java.util.concurrent.BlockingQueue; * Reads feed responses from a queue and renders them continuously to the * feeder. * - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> - * @since 5.1 + * @author Steinar Knutsen */ public class FeedResponse extends HttpResponse { diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeederSettings.java b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeederSettings.java index 37ccaf0d228..9bb8a58d6f6 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeederSettings.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeederSettings.java @@ -3,8 +3,6 @@ package com.yahoo.vespa.http.server; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.messagebus.routing.Route; -import com.yahoo.vespa.http.client.config.FeedParams.DataFormat; -import com.yahoo.vespa.http.client.core.Headers; import java.util.Optional; @@ -18,14 +16,14 @@ public class FeederSettings { private static final Route DEFAULT_ROUTE = Route.parse("default"); public final boolean drain; // TODO: Implement drain=true public final Route route; - public final DataFormat dataFormat; + public final FeedParams.DataFormat dataFormat; public final String priority; public final Integer traceLevel; public FeederSettings(HttpRequest request) { this.drain = Optional.ofNullable(request.getHeader(Headers.DRAIN)).map(Boolean::parseBoolean).orElse(false); this.route = Optional.ofNullable(request.getHeader(Headers.ROUTE)).map(Route::parse).orElse(DEFAULT_ROUTE); - this.dataFormat = Optional.ofNullable(request.getHeader(Headers.DATA_FORMAT)).map(DataFormat::valueOf).orElse(DataFormat.XML_UTF8); + this.dataFormat = Optional.ofNullable(request.getHeader(Headers.DATA_FORMAT)).map(FeedParams.DataFormat::valueOf).orElse(FeedParams.DataFormat.JSON_UTF8); this.priority = request.getHeader(Headers.PRIORITY); this.traceLevel = Optional.ofNullable(request.getHeader(Headers.TRACE_LEVEL)).map(Integer::valueOf).orElse(null); } diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/Headers.java b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/Headers.java new file mode 100644 index 00000000000..16bff38af4b --- /dev/null +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/Headers.java @@ -0,0 +1,37 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.http.server; + +/** + * HTTP headers. + * + * @author Steinar Knutsen + */ +final class Headers { + + private Headers() { + } + + public static final String CLIENT_VERSION = "Vespa-Client-Version"; + + public static final String TIMEOUT = "X-Yahoo-Feed-Timeout"; + public static final String DRAIN = "X-Yahoo-Feed-Drain"; + public static final String ROUTE = "X-Yahoo-Feed-Route"; + public static final String VERSION = "X-Yahoo-Feed-Protocol-Version"; + public static final String SESSION_ID = "X-Yahoo-Feed-Session-Id"; + public static final String DENY_IF_BUSY = "X-Yahoo-Feed-Deny-If-Busy"; + public static final String DATA_FORMAT = "X-Yahoo-Feed-Data-Format"; + // This value can be used to route the request to a specific server when using + // several servers. It is a random value that is the same for the whole session. + public static final String SHARDING_KEY = "X-Yahoo-Feed-Sharding-Key"; + public static final String PRIORITY = "X-Yahoo-Feed-Priority"; + public static final String TRACE_LEVEL = "X-Yahoo-Feed-Trace-Level"; + + public static final int HTTP_NOT_ACCEPTABLE = 406; + + // For version 3 of the API + public static final String CLIENT_ID = "X-Yahoo-Client-Id"; + public static final String OUTSTANDING_REQUESTS = "X-Yahoo-Outstanding-Requests"; + public static final String HOSTNAME = "X-Yahoo-Hostname"; + public static final String SILENTUPGRADE = "X-Yahoo-Silent-Upgrade"; + +} diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/OperationStatus.java b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/OperationStatus.java new file mode 100644 index 00000000000..e771128c2e5 --- /dev/null +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/OperationStatus.java @@ -0,0 +1,90 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.http.server; + +import com.google.common.base.Splitter; +import java.util.Iterator; + +/** + * Serialization/deserialization class for the result of a single document operation against Vespa. + * + * @author Steinar Knutsen + */ +final class OperationStatus { + + public static final String IS_CONDITION_NOT_MET = "IS-CONDITION-NOT-MET"; + public final String message; + public final String operationId; + public final ErrorCode errorCode; + public final String traceMessage; + public final boolean isConditionNotMet; + + private static final char EOL = '\n'; + private static final char SEPARATOR = ' '; + private static final Splitter spaceSep = Splitter.on(SEPARATOR); + + /** + * Constructor + * @param message some human readable information what happened + * @param operationId the doc ID for the operation + * @param errorCode if it is success, transitive, or fatal + * @param isConditionNotMet if error is due to condition not met + * @param traceMessage any tracemessage + */ + public OperationStatus(String message, String operationId, ErrorCode errorCode, boolean isConditionNotMet, String traceMessage) { + this.isConditionNotMet = isConditionNotMet; + this.message = message; + this.operationId = operationId; + this.errorCode = errorCode; + this.traceMessage = traceMessage; + } + + /** + * Parse a single rendered OperationStatus string. White space may be padded after + * and before the given status. + * + * @param singleLine + * a rendered OperationStatus + * @return an OperationStatus instance reflecting the input + * @throws IllegalArgumentException + * if there are illegal input data characters or the status + * element has no corresponding value in the ErrorCode + * enumeration + */ + public static OperationStatus parse(String singleLine) { + // Do note there is specifically left room for more arguments after + // the first in the serialized form. + Iterator<String> input = spaceSep.split(singleLine.trim()).iterator(); + String operationId; + ErrorCode errorCode; + String message; + String traceMessage = ""; + + operationId = Encoder.decode(input.next(), new StringBuilder()) + .toString(); + errorCode = ErrorCode.valueOf(Encoder.decode(input.next(), + new StringBuilder()).toString()); + + message = Encoder.decode(input.next(), new StringBuilder()).toString(); + // We are backwards compatible, meaning it is ok not to supply the last argument. + boolean isConditionNotMet = false; + if (message.startsWith(IS_CONDITION_NOT_MET)) { + message = message.replaceFirst(IS_CONDITION_NOT_MET, ""); + isConditionNotMet = true; + } + if (input.hasNext()) { + traceMessage = Encoder.decode(input.next(), new StringBuilder()).toString(); + } + return new OperationStatus(message, operationId, errorCode, isConditionNotMet, traceMessage); + } + + /** Returns a string representing the status. */ + public String render() { + StringBuilder s = new StringBuilder(); + Encoder.encode(operationId, s).append(SEPARATOR); + Encoder.encode(errorCode.toString(), s).append(SEPARATOR); + Encoder.encode(isConditionNotMet ? IS_CONDITION_NOT_MET + message : message, s).append(SEPARATOR); + Encoder.encode(traceMessage, s).append(EOL); + return s.toString(); + } + +} diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/ReplyContext.java b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/ReplyContext.java index aa2651595ef..6dc7f4ab516 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/ReplyContext.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/ReplyContext.java @@ -1,8 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.http.server; -import com.yahoo.vespa.http.client.core.OperationStatus; - import java.util.concurrent.BlockingQueue; /** diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/StreamReaderV3.java b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/StreamReaderV3.java index 4ddc430b35f..c2c6d00fa25 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/StreamReaderV3.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/StreamReaderV3.java @@ -3,7 +3,6 @@ package com.yahoo.vespa.http.server; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.document.DocumentTypeManager; -import com.yahoo.vespa.http.client.core.Encoder; import com.yahoo.vespa.http.server.util.ByteLimitedInputStream; import com.yahoo.vespaxmlparser.FeedOperation; import com.yahoo.vespaxmlparser.FeedReader; diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/UnknownClientException.java b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/UnknownClientException.java index 5324b86a98a..513b9355f3e 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/UnknownClientException.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/UnknownClientException.java @@ -2,8 +2,7 @@ package com.yahoo.vespa.http.server; /** - * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> - * @since 5.5.0 + * @author Einar M R Rosenvinge */ public class UnknownClientException extends RuntimeException { diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/util/ByteLimitedInputStream.java b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/util/ByteLimitedInputStream.java index 270ebe7796b..74489c774f0 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/util/ByteLimitedInputStream.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/util/ByteLimitedInputStream.java @@ -6,8 +6,6 @@ import java.io.InputStream; /** * @author Einar M R Rosenvinge - * - * @since 5.1.23 */ public class ByteLimitedInputStream extends InputStream { diff --git a/vespaclient-container-plugin/src/test/cfg/music.sd b/vespaclient-container-plugin/src/test/cfg/music.sd index 75a1918de5b..86597379dd3 100644 --- a/vespaclient-container-plugin/src/test/cfg/music.sd +++ b/vespaclient-container-plugin/src/test/cfg/music.sd @@ -2,5 +2,6 @@ search music { document music { field artist type string { } + field embedding type tensor(x[3]) {} } } diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/resource/DocumentV1ApiTest.java b/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/resource/DocumentV1ApiTest.java index 9f3c536a59a..74a86b6a7b7 100644 --- a/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/resource/DocumentV1ApiTest.java +++ b/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/resource/DocumentV1ApiTest.java @@ -14,6 +14,7 @@ import com.yahoo.document.FixedBucketSpaces; import com.yahoo.document.TestAndSetCondition; import com.yahoo.document.config.DocumentmanagerConfig; import com.yahoo.document.datatypes.StringFieldValue; +import com.yahoo.document.datatypes.TensorFieldValue; import com.yahoo.document.restapi.DocumentOperationExecutorConfig; import com.yahoo.document.restapi.resource.DocumentV1ApiHandler.StorageCluster; import com.yahoo.document.update.FieldUpdate; @@ -51,6 +52,7 @@ import com.yahoo.schema.derived.Deriver; import com.yahoo.slime.Inspector; import com.yahoo.slime.JsonFormat; import com.yahoo.slime.SlimeUtils; +import com.yahoo.tensor.Tensor; import com.yahoo.test.ManualClock; import com.yahoo.vdslib.VisitorStatistics; import com.yahoo.vespa.config.content.AllClustersBucketSpacesConfig; @@ -118,7 +120,9 @@ public class DocumentV1ApiTest { final Document doc3 = new Document(manager.getDocumentType("music"), "id:space:music:g=a:three"); { doc1.setFieldValue("artist", "Tom Waits"); + doc1.setFieldValue("embedding", new TensorFieldValue(Tensor.from("tensor(x[3]):[1,2,3]"))); doc2.setFieldValue("artist", "Asa-Chan & Jun-Ray"); + doc2.setFieldValue("embedding", new TensorFieldValue(Tensor.from("tensor(x[3]):[4,5,6]"))); } final Map<String, StorageCluster> clusters = Map.of("content", new StorageCluster("content", @@ -229,13 +233,15 @@ public class DocumentV1ApiTest { " {" + " \"id\": \"id:space:music::one\"," + " \"fields\": {" + - " \"artist\": \"Tom Waits\"" + + " \"artist\": \"Tom Waits\", " + + " \"embedding\": { \"type\": \"tensor(x[3])\", \"values\": [1.0,2.0,3.0] } " + " }" + " }," + " {" + " \"id\": \"id:space:music:n=1:two\"," + " \"fields\": {" + - " \"artist\": \"Asa-Chan & Jun-Ray\"" + + " \"artist\": \"Asa-Chan & Jun-Ray\", " + + " \"embedding\": { \"type\": \"tensor(x[3])\", \"values\": [4.0,5.0,6.0] } " + " }" + " }," + " {" + @@ -278,13 +284,15 @@ public class DocumentV1ApiTest { " {" + " \"id\": \"id:space:music::one\"," + " \"fields\": {" + - " \"artist\": \"Tom Waits\"" + + " \"artist\": \"Tom Waits\"," + + " \"embedding\": { \"type\": \"tensor(x[3])\", \"values\": [1.0,2.0,3.0] } " + " }" + " }," + " {" + " \"id\": \"id:space:music:n=1:two\"," + " \"fields\": {" + - " \"artist\": \"Asa-Chan & Jun-Ray\"" + + " \"artist\": \"Asa-Chan & Jun-Ray\"," + + " \"embedding\": { \"type\": \"tensor(x[3])\", \"values\": [4.0,5.0,6.0] } " + " }" + " }" + " ]," + @@ -337,7 +345,7 @@ public class DocumentV1ApiTest { // POST with namespace and document type is a restricted visit with a required destination cluster ("destinationCluster") access.expect(parameters -> { assertEquals("[Content:cluster=content]", parameters.getRemoteDataHandler()); - assertEquals("[all]", parameters.fieldSet()); + assertEquals("[document]", parameters.fieldSet()); assertEquals(60_000L, parameters.getSessionTimeoutMs()); parameters.getControlHandler().onDone(VisitorControlHandler.CompletionCode.SUCCESS, "We made it!"); }); @@ -491,12 +499,13 @@ public class DocumentV1ApiTest { parameters.responseHandler().get().handleResponse(new DocumentResponse(0, doc1)); return new Result(); }); - response = driver.sendRequest("http://localhost/document/v1/space/music/docid/one?"); + response = driver.sendRequest("http://localhost/document/v1/space/music/docid/one?format.tensors=long"); assertSameJson("{" + " \"pathId\": \"/document/v1/space/music/docid/one\"," + " \"id\": \"id:space:music::one\"," + " \"fields\": {" + - " \"artist\": \"Tom Waits\"" + + " \"artist\": \"Tom Waits\"," + + " \"embedding\": { \"cells\": [{\"address\":{\"x\":\"0\"},\"value\":1.0},{\"address\":{\"x\":\"1\"},\"value\": 2.0},{\"address\":{\"x\":\"2\"},\"value\": 3.0}]}" + " }" + "}", response.readAll()); assertEquals(200, response.getStatus()); @@ -532,7 +541,8 @@ public class DocumentV1ApiTest { response = driver.sendRequest("http://localhost/document/v1/space/music/number/1/two?condition=test%20it&tracelevel=9", POST, "{" + " \"fields\": {" + - " \"artist\": \"Asa-Chan & Jun-Ray\"" + + " \"artist\": \"Asa-Chan & Jun-Ray\"," + + " \"embedding\": { \"values\": [4.0,5.0,6.0] } " + " }" + "}"); assertSameJson("{" + diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/FeedHandlerV3Test.java b/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/FeedHandlerV3Test.java index a5a8f4cb5bd..dcabc1f338e 100644 --- a/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/FeedHandlerV3Test.java +++ b/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/FeedHandlerV3Test.java @@ -17,10 +17,6 @@ import com.yahoo.messagebus.SourceSessionParams; import com.yahoo.messagebus.shared.SharedSourceSession; import com.yahoo.metrics.simple.MetricReceiver; import com.yahoo.text.Utf8; -import com.yahoo.vespa.http.client.config.FeedParams; -import com.yahoo.vespa.http.client.core.ErrorCode; -import com.yahoo.vespa.http.client.core.Headers; -import com.yahoo.vespa.http.client.core.OperationStatus; import org.junit.Test; import org.mockito.stubbing.Answer; diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/FeedReaderFactoryTestCase.java b/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/FeedReaderFactoryTestCase.java index 6b0bd1c9518..08a7e82a158 100644 --- a/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/FeedReaderFactoryTestCase.java +++ b/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/FeedReaderFactoryTestCase.java @@ -3,7 +3,6 @@ package com.yahoo.vespa.http.server; import com.yahoo.document.DocumentTypeManager; import com.yahoo.text.Utf8; -import com.yahoo.vespa.http.client.config.FeedParams; import org.junit.Test; import java.io.ByteArrayInputStream; diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/vespaxmlparser/MockFeedReaderFactory.java b/vespaclient-container-plugin/src/test/java/com/yahoo/vespaxmlparser/MockFeedReaderFactory.java index 513892af213..a009e70fc30 100644 --- a/vespaclient-container-plugin/src/test/java/com/yahoo/vespaxmlparser/MockFeedReaderFactory.java +++ b/vespaclient-container-plugin/src/test/java/com/yahoo/vespaxmlparser/MockFeedReaderFactory.java @@ -2,7 +2,7 @@ package com.yahoo.vespaxmlparser; import com.yahoo.document.DocumentTypeManager; -import com.yahoo.vespa.http.client.config.FeedParams; +import com.yahoo.vespa.http.server.FeedParams; import com.yahoo.vespa.http.server.FeedReaderFactory; import java.io.InputStream; |