diff options
28 files changed, 291 insertions, 362 deletions
diff --git a/jdisc_core/abi-spec.json b/jdisc_core/abi-spec.json index d43c3e03e8f..5beb723465b 100644 --- a/jdisc_core/abi-spec.json +++ b/jdisc_core/abi-spec.json @@ -178,9 +178,7 @@ "public void <init>(com.yahoo.jdisc.Request, java.net.URI)", "public com.yahoo.jdisc.Container container()", "public java.net.URI getUri()", - "public com.yahoo.jdisc.Request setUri(java.net.URI)", "public boolean isServerRequest()", - "public com.yahoo.jdisc.Request setServerRequest(boolean)", "public com.yahoo.jdisc.application.BindingMatch getBindingMatch()", "public com.yahoo.jdisc.Request setBindingMatch(com.yahoo.jdisc.application.BindingMatch)", "public java.util.Map context()", diff --git a/vespa-hadoop/pom.xml b/vespa-hadoop/pom.xml index 0c724c95839..1dde2d58610 100644 --- a/vespa-hadoop/pom.xml +++ b/vespa-hadoop/pom.xml @@ -136,11 +136,6 @@ <!-- Vespa feeding dependencies --> <dependency> <groupId>com.yahoo.vespa</groupId> - <artifactId>vespa-http-client</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> <artifactId>vespa-feed-client</artifactId> <version>${project.version}</version> </dependency> diff --git a/vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/mapreduce/LegacyVespaRecordWriter.java b/vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/mapreduce/LegacyVespaRecordWriter.java deleted file mode 100644 index 6900c7dc82f..00000000000 --- a/vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/mapreduce/LegacyVespaRecordWriter.java +++ /dev/null @@ -1,233 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hadoop.mapreduce; - -import com.fasterxml.jackson.core.JsonFactory; -import com.fasterxml.jackson.core.JsonParseException; -import com.fasterxml.jackson.core.JsonParser; -import com.fasterxml.jackson.core.JsonToken; -import com.yahoo.vespa.hadoop.mapreduce.util.VespaConfiguration; -import com.yahoo.vespa.hadoop.mapreduce.util.VespaCounters; -import com.yahoo.vespa.hadoop.pig.VespaDocumentOperation; -import com.yahoo.vespa.http.client.config.Cluster; -import com.yahoo.vespa.http.client.config.ConnectionParams; -import com.yahoo.vespa.http.client.config.Endpoint; -import com.yahoo.vespa.http.client.config.FeedParams; -import com.yahoo.vespa.http.client.config.FeedParams.DataFormat; -import com.yahoo.vespa.http.client.config.SessionParams; -import org.apache.hadoop.mapreduce.RecordWriter; -import org.apache.hadoop.mapreduce.TaskAttemptContext; - -import javax.xml.namespace.QName; -import javax.xml.stream.FactoryConfigurationError; -import javax.xml.stream.XMLEventReader; -import javax.xml.stream.XMLInputFactory; -import javax.xml.stream.XMLStreamException; -import javax.xml.stream.events.StartElement; -import javax.xml.stream.events.XMLEvent; -import java.io.IOException; -import java.io.StringReader; -import java.time.Duration; -import java.util.List; -import java.util.StringTokenizer; -import java.util.concurrent.ThreadLocalRandom; -import java.util.logging.Logger; - -/** - * {@link LegacyVespaRecordWriter} sends the output <key, value> to one or more Vespa endpoints using vespa-http-client. - * - * @author lesters - * @deprecated Replaced by {@link VespaRecordWriter} - */ -@Deprecated -public class LegacyVespaRecordWriter extends RecordWriter<Object, Object> { - - private final Logger log = Logger.getLogger(getClass().getCanonicalName()); - - private boolean initialized = false; - private com.yahoo.vespa.http.client.FeedClient feedClient; - private final VespaCounters counters; - private final int progressInterval; - - final VespaConfiguration configuration; - - LegacyVespaRecordWriter(VespaConfiguration configuration, VespaCounters counters) { - this.counters = counters; - this.configuration = configuration; - this.progressInterval = configuration.progressInterval(); - } - - - @Override - public void write(Object key, Object data) throws IOException, InterruptedException { - if (!initialized) { - initialize(); - } - - String doc = data.toString().trim(); - - // Parse data to find document id - if none found, skip this write - String docId = DataFormat.JSON_UTF8.equals(configuration.dataFormat()) ? findDocId(doc) - : findDocIdFromXml(doc); - if (docId != null && docId.length() >= 0) { - feedClient.stream(docId, doc); - counters.incrementDocumentsSent(1); - } else { - counters.incrementDocumentsSkipped(1); - } - - if (counters.getDocumentsSent() % progressInterval == 0) { - String progress = String.format("Feed progress: %d / %d / %d / %d (sent, ok, failed, skipped)", - counters.getDocumentsSent(), - counters.getDocumentsOk(), - counters.getDocumentsFailed(), - counters.getDocumentsSkipped()); - log.info(progress); - } - - } - - - @Override - public void close(TaskAttemptContext taskAttemptContext) throws IOException, InterruptedException { - if (feedClient != null) { - feedClient.close(); - } - } - - protected ConnectionParams.Builder configureConnectionParams() { - ConnectionParams.Builder connParamsBuilder = new ConnectionParams.Builder(); - connParamsBuilder.setDryRun(configuration.dryrun()); - connParamsBuilder.setUseCompression(configuration.useCompression()); - connParamsBuilder.setNumPersistentConnectionsPerEndpoint(configuration.numConnections()); - connParamsBuilder.setMaxRetries(configuration.numRetries()); - if (configuration.proxyHost() != null) { - connParamsBuilder.setProxyHost(configuration.proxyHost()); - } - if (configuration.proxyPort() >= 0) { - connParamsBuilder.setProxyPort(configuration.proxyPort()); - } - return connParamsBuilder; - } - - protected FeedParams.Builder configureFeedParams() { - FeedParams.Builder feedParamsBuilder = new FeedParams.Builder(); - feedParamsBuilder.setDataFormat(configuration.dataFormat()); - feedParamsBuilder.setRoute(configuration.route()); - feedParamsBuilder.setMaxSleepTimeMs(configuration.maxSleepTimeMs()); - feedParamsBuilder.setMaxInFlightRequests(configuration.maxInFlightRequests()); - feedParamsBuilder.setLocalQueueTimeOut(Duration.ofMinutes(10).toMillis()); - return feedParamsBuilder; - } - - protected SessionParams.Builder configureSessionParams() { - SessionParams.Builder sessionParamsBuilder = new SessionParams.Builder(); - sessionParamsBuilder.setThrottlerMinSize(configuration.throttlerMinSize()); - sessionParamsBuilder.setClientQueueSize(configuration.maxInFlightRequests()*2); - return sessionParamsBuilder; - } - - private void initialize() { - if (!configuration.dryrun() && configuration.randomStartupSleepMs() > 0) { - int delay = ThreadLocalRandom.current().nextInt(configuration.randomStartupSleepMs()); - log.info("VespaStorage: Delaying startup by " + delay + " ms"); - try { - Thread.sleep(delay); - } catch (Exception e) {} - } - - ConnectionParams.Builder connParamsBuilder = configureConnectionParams(); - FeedParams.Builder feedParamsBuilder = configureFeedParams(); - SessionParams.Builder sessionParams = configureSessionParams(); - - sessionParams.setConnectionParams(connParamsBuilder.build()); - sessionParams.setFeedParams(feedParamsBuilder.build()); - - String endpoints = configuration.endpoint(); - StringTokenizer tokenizer = new StringTokenizer(endpoints, ","); - while (tokenizer.hasMoreTokens()) { - String endpoint = tokenizer.nextToken().trim(); - sessionParams.addCluster(new Cluster.Builder().addEndpoint( - Endpoint.create(endpoint, configuration.defaultPort(), configuration.useSSL().orElse(false)) - ).build()); - } - - ResultCallback resultCallback = new ResultCallback(counters); - feedClient = com.yahoo.vespa.http.client.FeedClientFactory.create(sessionParams.build(), resultCallback); - - initialized = true; - log.info("VespaStorage configuration:\n" + configuration.toString()); - log.info(feedClient.getStatsAsJson()); - } - - private String findDocIdFromXml(String xml) { - try { - XMLEventReader eventReader = XMLInputFactory.newInstance().createXMLEventReader(new StringReader(xml)); - while (eventReader.hasNext()) { - XMLEvent event = eventReader.nextEvent(); - if (event.getEventType() == XMLEvent.START_ELEMENT) { - StartElement element = event.asStartElement(); - String elementName = element.getName().getLocalPart(); - if (VespaDocumentOperation.Operation.valid(elementName)) { - return element.getAttributeByName(QName.valueOf("documentid")).getValue(); - } - } - } - } catch (XMLStreamException | FactoryConfigurationError e) { - // as json dude does - return null; - } - return null; - } - - private String findDocId(String json) throws IOException { - JsonFactory factory = new JsonFactory(); - try(JsonParser parser = factory.createParser(json)) { - if (parser.nextToken() != JsonToken.START_OBJECT) { - return null; - } - while (parser.nextToken() != JsonToken.END_OBJECT) { - String fieldName = parser.getCurrentName(); - parser.nextToken(); - if (VespaDocumentOperation.Operation.valid(fieldName)) { - String docId = parser.getText(); - return docId; - } else { - parser.skipChildren(); - } - } - } catch (JsonParseException ex) { - return null; - } - return null; - } - - - class ResultCallback implements com.yahoo.vespa.http.client.FeedClient.ResultCallback { - final VespaCounters counters; - - public ResultCallback(VespaCounters counters) { - this.counters = counters; - } - - @Override - public void onCompletion(String docId, com.yahoo.vespa.http.client.Result documentResult) { - if (!documentResult.isSuccess()) { - counters.incrementDocumentsFailed(1); - StringBuilder sb = new StringBuilder(); - sb.append("Problems with docid "); - sb.append(docId); - sb.append(": "); - List<com.yahoo.vespa.http.client.Result.Detail> details = documentResult.getDetails(); - for (com.yahoo.vespa.http.client.Result.Detail detail : details) { - sb.append(detail.toString()); - sb.append(" "); - } - log.warning(sb.toString()); - return; - } - counters.incrementDocumentsOk(1); - } - - } - -} diff --git a/vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/mapreduce/VespaOutputFormat.java b/vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/mapreduce/VespaOutputFormat.java index 66ab94574d9..e49a5e17970 100644 --- a/vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/mapreduce/VespaOutputFormat.java +++ b/vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/mapreduce/VespaOutputFormat.java @@ -10,15 +10,12 @@ import org.apache.hadoop.mapreduce.RecordWriter; import org.apache.hadoop.mapreduce.TaskAttemptContext; import java.io.IOException; -import java.util.Objects; import java.util.Properties; import java.util.logging.Logger; -import static com.yahoo.vespa.http.client.config.FeedParams.DataFormat.XML_UTF8; - /** * An output specification for writing to Vespa instances in a Map-Reduce job. - * Mainly returns an instance of a {@link LegacyVespaRecordWriter} that does the + * Mainly returns an instance of a {@link VespaRecordWriter} that does the * actual feeding to Vespa. * * @author lesters @@ -46,14 +43,7 @@ public class VespaOutputFormat extends OutputFormat { public RecordWriter getRecordWriter(TaskAttemptContext context) throws IOException { VespaCounters counters = VespaCounters.get(context); VespaConfiguration configuration = VespaConfiguration.get(context.getConfiguration(), configOverride); - Boolean useLegacyClient = configuration.useLegacyClient().orElse(null); - if (Objects.equals(useLegacyClient, Boolean.TRUE) || configuration.dataFormat() == XML_UTF8) { - log.warning("Feeding with legacy client or XML will no longer be supported on Vespa 8. " + - "See https://docs.vespa.ai/en/vespa8-release-notes.html"); - return new LegacyVespaRecordWriter(configuration, counters); - } else { - return new VespaRecordWriter(configuration, counters); - } + return new VespaRecordWriter(configuration, counters); } diff --git a/vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/mapreduce/VespaRecordWriter.java b/vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/mapreduce/VespaRecordWriter.java index 6d6c3789835..c450d7cdeef 100644 --- a/vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/mapreduce/VespaRecordWriter.java +++ b/vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/mapreduce/VespaRecordWriter.java @@ -7,7 +7,6 @@ import ai.vespa.feed.client.JsonFeeder; import ai.vespa.feed.client.OperationParseException; import com.yahoo.vespa.hadoop.mapreduce.util.VespaConfiguration; import com.yahoo.vespa.hadoop.mapreduce.util.VespaCounters; -import com.yahoo.vespa.http.client.config.FeedParams; import org.apache.hadoop.mapreduce.RecordWriter; import org.apache.hadoop.mapreduce.TaskAttemptContext; @@ -85,18 +84,11 @@ public class VespaRecordWriter extends RecordWriter<Object, Object> { private void initializeOnFirstWrite() { if (initialized) return; - validateConfig(); useRandomizedStartupDelayIfEnabled(); feeder = createJsonStreamFeeder(); initialized = true; } - private void validateConfig() { - if (config.dataFormat() != FeedParams.DataFormat.JSON_UTF8) { - throw new IllegalArgumentException("Only JSON is support by this feed client implementation"); - } - } - private void useRandomizedStartupDelayIfEnabled() { if (!config.dryrun() && config.randomStartupSleepMs() > 0) { int delay = ThreadLocalRandom.current().nextInt(config.randomStartupSleepMs()); diff --git a/vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/mapreduce/util/VespaConfiguration.java b/vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/mapreduce/util/VespaConfiguration.java index 715546fe6fe..ae0b6a58155 100644 --- a/vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/mapreduce/util/VespaConfiguration.java +++ b/vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/mapreduce/util/VespaConfiguration.java @@ -1,7 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hadoop.mapreduce.util; -import com.yahoo.vespa.http.client.config.FeedParams; import org.apache.hadoop.conf.Configuration; import java.io.IOException; @@ -19,7 +18,6 @@ public class VespaConfiguration { public static final String PROXY_SCHEME = "vespa.feed.proxy.scheme"; public static final String DRYRUN = "vespa.feed.dryrun"; public static final String USE_COMPRESSION = "vespa.feed.usecompression"; - public static final String DATA_FORMAT = "vespa.feed.data.format"; public static final String PROGRESS_REPORT = "vespa.feed.progress.interval"; public static final String CONNECTIONS = "vespa.feed.connections"; public static final String THROTTLER_MIN_SIZE = "vespa.feed.throttler.min.size"; @@ -29,7 +27,6 @@ public class VespaConfiguration { public static final String MAX_IN_FLIGHT_REQUESTS = "vespa.feed.max.in.flight.requests"; public static final String RANDOM_STARTUP_SLEEP = "vespa.feed.random.startup.sleep.ms"; public static final String NUM_RETRIES = "vespa.feed.num.retries"; - public static final String USE_LEGACY_CLIENT = "vespa.feed.uselegacyclient"; private final Configuration conf; private final Properties override; @@ -129,25 +126,10 @@ public class VespaConfiguration { } - public FeedParams.DataFormat dataFormat() { - String format = getString(DATA_FORMAT); - if ("xml".equalsIgnoreCase(format)) { - return FeedParams.DataFormat.XML_UTF8; - } - return FeedParams.DataFormat.JSON_UTF8; - } - - public int progressInterval() { return getInt(PROGRESS_REPORT, 1000); } - public Optional<Boolean> useLegacyClient() { - String raw = getString(USE_LEGACY_CLIENT); - if (raw == null || raw.trim().isEmpty()) return Optional.empty(); - return Optional.of(Boolean.parseBoolean(raw)); - } - public String getString(String name) { if (override != null && override.containsKey(name)) { return override.getProperty(name); @@ -197,7 +179,6 @@ public class VespaConfiguration { sb.append(PROXY_SCHEME + ": " + proxyScheme() + "\n"); sb.append(DRYRUN + ": " + dryrun() +"\n"); sb.append(USE_COMPRESSION + ": " + useCompression() +"\n"); - sb.append(DATA_FORMAT + ": " + dataFormat() +"\n"); sb.append(PROGRESS_REPORT + ": " + progressInterval() +"\n"); sb.append(CONNECTIONS + ": " + numConnections() +"\n"); sb.append(THROTTLER_MIN_SIZE + ": " + throttlerMinSize() +"\n"); @@ -207,7 +188,6 @@ public class VespaConfiguration { sb.append(MAX_IN_FLIGHT_REQUESTS + ": " + maxInFlightRequests() +"\n"); sb.append(RANDOM_STARTUP_SLEEP + ": " + randomStartupSleepMs() +"\n"); sb.append(NUM_RETRIES + ": " + numRetries() +"\n"); - sb.append(USE_LEGACY_CLIENT + ": " + useLegacyClient().map(Object::toString).orElse("<empty>") +"\n"); return sb.toString(); } diff --git a/vespa-hadoop/src/test/java/com/yahoo/vespa/hadoop/pig/VespaStorageTest.java b/vespa-hadoop/src/test/java/com/yahoo/vespa/hadoop/pig/VespaStorageTest.java index f690e767194..3183c770bc7 100644 --- a/vespa-hadoop/src/test/java/com/yahoo/vespa/hadoop/pig/VespaStorageTest.java +++ b/vespa-hadoop/src/test/java/com/yahoo/vespa/hadoop/pig/VespaStorageTest.java @@ -24,14 +24,6 @@ import static org.junit.jupiter.api.Assertions.assertNotNull; public class VespaStorageTest { @Test - public void requireThatPremadeXmlOperationsFeedSucceeds() throws Exception { - Configuration conf = new HdfsConfiguration(); - conf.set(VespaConfiguration.DATA_FORMAT, "xml"); - assertAllDocumentsOk("src/test/pig/feed_operations_xml.pig", conf); - } - - - @Test public void requireThatPremadeOperationsFeedSucceeds() throws Exception { assertAllDocumentsOk("src/test/pig/feed_operations.pig"); } @@ -52,7 +44,6 @@ public class VespaStorageTest { public void requireThatPremadeOperationsWithJsonLoaderFeedAndNonLegacyClientSucceeds() throws Exception { Configuration conf = new HdfsConfiguration(); conf.set(VespaConfiguration.USE_SSL, Boolean.TRUE.toString()); - conf.set(VespaConfiguration.USE_LEGACY_CLIENT, Boolean.FALSE.toString()); assertAllDocumentsOk("src/test/pig/feed_operations_with_json_loader.pig", conf); } diff --git a/vespaclient-container-plugin/pom.xml b/vespaclient-container-plugin/pom.xml index c960c2cca44..d7b36e39c94 100644 --- a/vespaclient-container-plugin/pom.xml +++ b/vespaclient-container-plugin/pom.xml @@ -38,42 +38,6 @@ <scope>provided</scope> </dependency> <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>vespa-http-client</artifactId> - <version>${project.version}</version> - <exclusions> - <!-- Exclude artifacts that are provided by Jdisc container --> - <exclusion> - <groupId>com.fasterxml.jackson.core</groupId> - <artifactId>*</artifactId> - </exclusion> - <exclusion> - <groupId>com.fasterxml.jackson.datatype</groupId> - <artifactId>*</artifactId> - </exclusion> - <exclusion> - <groupId>com.fasterxml.jackson.module</groupId> - <artifactId>*</artifactId> - </exclusion> - <exclusion> - <groupId>com.google.guava</groupId> - <artifactId>guava</artifactId> - </exclusion> - <exclusion> - <groupId>com.yahoo.vespa</groupId> - <artifactId>security-utils</artifactId> - </exclusion> - <exclusion> - <groupId>org.apache.httpcomponents</groupId> - <artifactId>*</artifactId> - </exclusion> - <exclusion> - <groupId>org.bouncycastle</groupId> - <artifactId>*</artifactId> - </exclusion> - </exclusions> - </dependency> - <dependency> <groupId>org.mockito</groupId> <artifactId>mockito-core</artifactId> <scope>test</scope> diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/ClientFeederV3.java b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/ClientFeederV3.java index 875ff3e5bf0..8ea9234009d 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/ClientFeederV3.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/ClientFeederV3.java @@ -4,7 +4,6 @@ package com.yahoo.vespa.http.server; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.document.DocumentTypeManager; -import com.yahoo.documentapi.messagebus.protocol.DocumentMessage; import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol; import com.yahoo.jdisc.Metric; import com.yahoo.jdisc.ReferencedResource; @@ -14,9 +13,6 @@ import com.yahoo.messagebus.ReplyHandler; import com.yahoo.messagebus.Result; import com.yahoo.messagebus.shared.SharedSourceSession; import com.yahoo.net.HostName; -import com.yahoo.vespa.http.client.core.ErrorCode; -import com.yahoo.vespa.http.client.core.Headers; -import com.yahoo.vespa.http.client.core.OperationStatus; import com.yahoo.vespaxmlparser.FeedOperation; import com.yahoo.yolean.Exceptions; diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/ClientState.java b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/ClientState.java index 13a12f707d9..973c154b336 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/ClientState.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/ClientState.java @@ -4,14 +4,13 @@ package com.yahoo.vespa.http.server; import com.yahoo.jdisc.Metric; import com.yahoo.jdisc.ReferencedResource; import com.yahoo.messagebus.shared.SharedSourceSession; -import com.yahoo.vespa.http.client.core.OperationStatus; import java.util.concurrent.BlockingQueue; /** * The state of a client session, used to save replies when client disconnects. * - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + * @author Steinar Knutsen */ public class ClientState { diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/Encoder.java b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/Encoder.java new file mode 100644 index 00000000000..65fb1223c7d --- /dev/null +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/Encoder.java @@ -0,0 +1,100 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.http.server; + +/** + * Simple encoding scheme to remove space, linefeed, control characters and + * anything outside ISO 646.irv:1991 from strings. The scheme is supposed to be + * human readable and debugging friendly. Opening and closing curly braces are + * used as quoting characters, the output is by definition US-ASCII only + * characters. + * + * @author Steinar Knutsen + */ +final class Encoder { + + /** + * ISO 646.irv:1991 safe quoting into a StringBuilder instance. + * + * @param input the string to encode + * @param output the destination buffer + * @return the destination buffer given as input + */ + public static StringBuilder encode(String input, StringBuilder output) { + for (int i = 0; i < input.length(); i = input.offsetByCodePoints(i, 1)) { + int c = input.codePointAt(i); + if (c <= '~') { + if (c <= ' ') { + encode(c, output); + } else { + switch (c) { + case '{': + case '}': + encode(c, output); + break; + default: + output.append((char) c); + } + } + } else { + encode(c, output); + } + } + return output; + } + + /** + * ISO 646.irv:1991 safe unquoting into a StringBuilder instance. + * + * @param input the string to decode + * @param output the destination buffer + * @return the destination buffer given as input + * @throws IllegalArgumentException if the input string contains unexpected or invalid data + */ + public static StringBuilder decode(String input, StringBuilder output) { + for (int i = 0; i < input.length(); i = input.offsetByCodePoints(i, 1)) { + int c = input.codePointAt(i); + if (c > '~') + throw new IllegalArgumentException("Input contained character above printable ASCII at position " + i); + if (c == '{') + i = decode(input, i, output); + else + output.append((char) c); + } + return output; + } + + private static int decode(String input, int offset, StringBuilder output) { + char c = 0; + int end = offset; + int start = offset + 1; + int codePoint; + + while ('}' != c) { + if (++end >= input.length()) { + throw new IllegalArgumentException("Unterminated quoted character or empty quoting."); + } + c = input.charAt(end); + } + try { + codePoint = Integer.parseInt(input.substring(start, end), 16); + } catch (NumberFormatException e) { + throw new IllegalArgumentException("Unexpected quoted data: [" + input.substring(start, end) + "]", e); + } + if (Character.charCount(codePoint) > 1) { + try { + output.append(Character.toChars(codePoint)); + } catch (IllegalArgumentException e) { + throw new IllegalArgumentException("Unexpected quoted data: [" + input.substring(start, end) + "]", e); + } + } else { + output.append((char) codePoint); + } + return end; + + } + + private static void encode(int c, StringBuilder output) { + output.append("{").append(Integer.toHexString(c)).append("}"); + } + +} diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/ErrorCode.java b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/ErrorCode.java new file mode 100644 index 00000000000..f819ecccbb1 --- /dev/null +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/ErrorCode.java @@ -0,0 +1,33 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.http.server; + +/** + * Return types for the server. + * + * @author Einar M R Rosenvinge + * @author Steinar Knutsen + */ +enum ErrorCode { + + OK(true, true), + ERROR(false, false), + TRANSIENT_ERROR(false, true), + END_OF_FEED(true, true); + + private final boolean success; + private final boolean _transient; + + ErrorCode(boolean success, boolean _transient) { + this.success = success; + this._transient = _transient; + } + + public boolean isSuccess() { + return success; + } + + public boolean isTransient() { + return _transient; + } + +} diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedHandler.java b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedHandler.java index f99274d3f2b..74665d60a04 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedHandler.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedHandler.java @@ -15,7 +15,6 @@ import com.yahoo.jdisc.Response; import com.yahoo.jdisc.handler.ResponseHandler; import com.yahoo.messagebus.ReplyHandler; import com.yahoo.metrics.simple.MetricReceiver; -import com.yahoo.vespa.http.client.core.Headers; import javax.inject.Inject; import java.io.IOException; diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedHandlerV3.java b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedHandlerV3.java index c8828df6d54..f9ae04623e6 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedHandlerV3.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedHandlerV3.java @@ -13,7 +13,6 @@ import com.yahoo.jdisc.ReferencedResource; import com.yahoo.messagebus.ReplyHandler; import com.yahoo.messagebus.SourceSessionParams; import com.yahoo.messagebus.shared.SharedSourceSession; -import com.yahoo.vespa.http.client.core.Headers; import com.yahoo.yolean.Exceptions; import java.util.HashMap; diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedParams.java b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedParams.java new file mode 100644 index 00000000000..6ce20cdec53 --- /dev/null +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedParams.java @@ -0,0 +1,23 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.http.server; + +import java.util.concurrent.TimeUnit; + +/** + * Feed level parameters. + * + * @author Einar M R Rosenvinge + */ +public final class FeedParams { + + /** + * Enumeration of data formats that are acceptable by the + * FeedClient methods. + */ + public enum DataFormat { + /** UTF-8-encoded XML. Preamble is not necessary. */ + XML_UTF8, + JSON_UTF8 + } + +} diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedReaderFactory.java b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedReaderFactory.java index 069ccfd84f0..f96b650748d 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedReaderFactory.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedReaderFactory.java @@ -4,7 +4,6 @@ package com.yahoo.vespa.http.server; import com.yahoo.document.DocumentTypeManager; import com.yahoo.document.json.JsonFeedReader; import com.yahoo.text.Utf8; -import com.yahoo.vespa.http.client.config.FeedParams; import com.yahoo.vespaxmlparser.FeedReader; import com.yahoo.vespaxmlparser.VespaXMLFeedReader; diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedReplyReader.java b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedReplyReader.java index 2fbb80d9fcc..1422ec10b08 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedReplyReader.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedReplyReader.java @@ -11,17 +11,11 @@ import com.yahoo.jdisc.Metric; import com.yahoo.messagebus.Reply; import com.yahoo.messagebus.ReplyHandler; import com.yahoo.messagebus.Trace; -import com.yahoo.vespa.http.client.core.ErrorCode; -import com.yahoo.vespa.http.client.core.OperationStatus; import java.util.Map; -import java.util.Optional; -import java.util.function.Predicate; import java.util.logging.Level; import java.util.logging.Logger; -import static java.util.function.Predicate.not; - /** * Catch message bus replies and make the available to a given session. * diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedResponse.java b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedResponse.java index 3e2a4a8795f..1da8aded27b 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedResponse.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedResponse.java @@ -2,9 +2,6 @@ package com.yahoo.vespa.http.server; import com.yahoo.container.jdisc.HttpResponse; -import com.yahoo.vespa.http.client.core.Headers; -import com.yahoo.vespa.http.client.core.ErrorCode; -import com.yahoo.vespa.http.client.core.OperationStatus; import java.io.IOException; import java.io.OutputStream; @@ -15,8 +12,7 @@ import java.util.concurrent.BlockingQueue; * Reads feed responses from a queue and renders them continuously to the * feeder. * - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> - * @since 5.1 + * @author Steinar Knutsen */ public class FeedResponse extends HttpResponse { diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeederSettings.java b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeederSettings.java index 725349f6ebe..9bb8a58d6f6 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeederSettings.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeederSettings.java @@ -3,8 +3,6 @@ package com.yahoo.vespa.http.server; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.messagebus.routing.Route; -import com.yahoo.vespa.http.client.config.FeedParams.DataFormat; -import com.yahoo.vespa.http.client.core.Headers; import java.util.Optional; @@ -18,14 +16,14 @@ public class FeederSettings { private static final Route DEFAULT_ROUTE = Route.parse("default"); public final boolean drain; // TODO: Implement drain=true public final Route route; - public final DataFormat dataFormat; + public final FeedParams.DataFormat dataFormat; public final String priority; public final Integer traceLevel; public FeederSettings(HttpRequest request) { this.drain = Optional.ofNullable(request.getHeader(Headers.DRAIN)).map(Boolean::parseBoolean).orElse(false); this.route = Optional.ofNullable(request.getHeader(Headers.ROUTE)).map(Route::parse).orElse(DEFAULT_ROUTE); - this.dataFormat = Optional.ofNullable(request.getHeader(Headers.DATA_FORMAT)).map(DataFormat::valueOf).orElse(DataFormat.JSON_UTF8); + this.dataFormat = Optional.ofNullable(request.getHeader(Headers.DATA_FORMAT)).map(FeedParams.DataFormat::valueOf).orElse(FeedParams.DataFormat.JSON_UTF8); this.priority = request.getHeader(Headers.PRIORITY); this.traceLevel = Optional.ofNullable(request.getHeader(Headers.TRACE_LEVEL)).map(Integer::valueOf).orElse(null); } diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/Headers.java b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/Headers.java new file mode 100644 index 00000000000..16bff38af4b --- /dev/null +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/Headers.java @@ -0,0 +1,37 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.http.server; + +/** + * HTTP headers. + * + * @author Steinar Knutsen + */ +final class Headers { + + private Headers() { + } + + public static final String CLIENT_VERSION = "Vespa-Client-Version"; + + public static final String TIMEOUT = "X-Yahoo-Feed-Timeout"; + public static final String DRAIN = "X-Yahoo-Feed-Drain"; + public static final String ROUTE = "X-Yahoo-Feed-Route"; + public static final String VERSION = "X-Yahoo-Feed-Protocol-Version"; + public static final String SESSION_ID = "X-Yahoo-Feed-Session-Id"; + public static final String DENY_IF_BUSY = "X-Yahoo-Feed-Deny-If-Busy"; + public static final String DATA_FORMAT = "X-Yahoo-Feed-Data-Format"; + // This value can be used to route the request to a specific server when using + // several servers. It is a random value that is the same for the whole session. + public static final String SHARDING_KEY = "X-Yahoo-Feed-Sharding-Key"; + public static final String PRIORITY = "X-Yahoo-Feed-Priority"; + public static final String TRACE_LEVEL = "X-Yahoo-Feed-Trace-Level"; + + public static final int HTTP_NOT_ACCEPTABLE = 406; + + // For version 3 of the API + public static final String CLIENT_ID = "X-Yahoo-Client-Id"; + public static final String OUTSTANDING_REQUESTS = "X-Yahoo-Outstanding-Requests"; + public static final String HOSTNAME = "X-Yahoo-Hostname"; + public static final String SILENTUPGRADE = "X-Yahoo-Silent-Upgrade"; + +} diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/OperationStatus.java b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/OperationStatus.java new file mode 100644 index 00000000000..e771128c2e5 --- /dev/null +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/OperationStatus.java @@ -0,0 +1,90 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.http.server; + +import com.google.common.base.Splitter; +import java.util.Iterator; + +/** + * Serialization/deserialization class for the result of a single document operation against Vespa. + * + * @author Steinar Knutsen + */ +final class OperationStatus { + + public static final String IS_CONDITION_NOT_MET = "IS-CONDITION-NOT-MET"; + public final String message; + public final String operationId; + public final ErrorCode errorCode; + public final String traceMessage; + public final boolean isConditionNotMet; + + private static final char EOL = '\n'; + private static final char SEPARATOR = ' '; + private static final Splitter spaceSep = Splitter.on(SEPARATOR); + + /** + * Constructor + * @param message some human readable information what happened + * @param operationId the doc ID for the operation + * @param errorCode if it is success, transitive, or fatal + * @param isConditionNotMet if error is due to condition not met + * @param traceMessage any tracemessage + */ + public OperationStatus(String message, String operationId, ErrorCode errorCode, boolean isConditionNotMet, String traceMessage) { + this.isConditionNotMet = isConditionNotMet; + this.message = message; + this.operationId = operationId; + this.errorCode = errorCode; + this.traceMessage = traceMessage; + } + + /** + * Parse a single rendered OperationStatus string. White space may be padded after + * and before the given status. + * + * @param singleLine + * a rendered OperationStatus + * @return an OperationStatus instance reflecting the input + * @throws IllegalArgumentException + * if there are illegal input data characters or the status + * element has no corresponding value in the ErrorCode + * enumeration + */ + public static OperationStatus parse(String singleLine) { + // Do note there is specifically left room for more arguments after + // the first in the serialized form. + Iterator<String> input = spaceSep.split(singleLine.trim()).iterator(); + String operationId; + ErrorCode errorCode; + String message; + String traceMessage = ""; + + operationId = Encoder.decode(input.next(), new StringBuilder()) + .toString(); + errorCode = ErrorCode.valueOf(Encoder.decode(input.next(), + new StringBuilder()).toString()); + + message = Encoder.decode(input.next(), new StringBuilder()).toString(); + // We are backwards compatible, meaning it is ok not to supply the last argument. + boolean isConditionNotMet = false; + if (message.startsWith(IS_CONDITION_NOT_MET)) { + message = message.replaceFirst(IS_CONDITION_NOT_MET, ""); + isConditionNotMet = true; + } + if (input.hasNext()) { + traceMessage = Encoder.decode(input.next(), new StringBuilder()).toString(); + } + return new OperationStatus(message, operationId, errorCode, isConditionNotMet, traceMessage); + } + + /** Returns a string representing the status. */ + public String render() { + StringBuilder s = new StringBuilder(); + Encoder.encode(operationId, s).append(SEPARATOR); + Encoder.encode(errorCode.toString(), s).append(SEPARATOR); + Encoder.encode(isConditionNotMet ? IS_CONDITION_NOT_MET + message : message, s).append(SEPARATOR); + Encoder.encode(traceMessage, s).append(EOL); + return s.toString(); + } + +} diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/ReplyContext.java b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/ReplyContext.java index aa2651595ef..6dc7f4ab516 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/ReplyContext.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/ReplyContext.java @@ -1,8 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.http.server; -import com.yahoo.vespa.http.client.core.OperationStatus; - import java.util.concurrent.BlockingQueue; /** diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/StreamReaderV3.java b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/StreamReaderV3.java index 4ddc430b35f..c2c6d00fa25 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/StreamReaderV3.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/StreamReaderV3.java @@ -3,7 +3,6 @@ package com.yahoo.vespa.http.server; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.document.DocumentTypeManager; -import com.yahoo.vespa.http.client.core.Encoder; import com.yahoo.vespa.http.server.util.ByteLimitedInputStream; import com.yahoo.vespaxmlparser.FeedOperation; import com.yahoo.vespaxmlparser.FeedReader; diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/UnknownClientException.java b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/UnknownClientException.java index 5324b86a98a..513b9355f3e 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/UnknownClientException.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/UnknownClientException.java @@ -2,8 +2,7 @@ package com.yahoo.vespa.http.server; /** - * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> - * @since 5.5.0 + * @author Einar M R Rosenvinge */ public class UnknownClientException extends RuntimeException { diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/util/ByteLimitedInputStream.java b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/util/ByteLimitedInputStream.java index 270ebe7796b..74489c774f0 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/util/ByteLimitedInputStream.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/util/ByteLimitedInputStream.java @@ -6,8 +6,6 @@ import java.io.InputStream; /** * @author Einar M R Rosenvinge - * - * @since 5.1.23 */ public class ByteLimitedInputStream extends InputStream { diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/FeedHandlerV3Test.java b/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/FeedHandlerV3Test.java index a5a8f4cb5bd..dcabc1f338e 100644 --- a/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/FeedHandlerV3Test.java +++ b/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/FeedHandlerV3Test.java @@ -17,10 +17,6 @@ import com.yahoo.messagebus.SourceSessionParams; import com.yahoo.messagebus.shared.SharedSourceSession; import com.yahoo.metrics.simple.MetricReceiver; import com.yahoo.text.Utf8; -import com.yahoo.vespa.http.client.config.FeedParams; -import com.yahoo.vespa.http.client.core.ErrorCode; -import com.yahoo.vespa.http.client.core.Headers; -import com.yahoo.vespa.http.client.core.OperationStatus; import org.junit.Test; import org.mockito.stubbing.Answer; diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/FeedReaderFactoryTestCase.java b/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/FeedReaderFactoryTestCase.java index 6b0bd1c9518..08a7e82a158 100644 --- a/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/FeedReaderFactoryTestCase.java +++ b/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/FeedReaderFactoryTestCase.java @@ -3,7 +3,6 @@ package com.yahoo.vespa.http.server; import com.yahoo.document.DocumentTypeManager; import com.yahoo.text.Utf8; -import com.yahoo.vespa.http.client.config.FeedParams; import org.junit.Test; import java.io.ByteArrayInputStream; diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/vespaxmlparser/MockFeedReaderFactory.java b/vespaclient-container-plugin/src/test/java/com/yahoo/vespaxmlparser/MockFeedReaderFactory.java index 513892af213..a009e70fc30 100644 --- a/vespaclient-container-plugin/src/test/java/com/yahoo/vespaxmlparser/MockFeedReaderFactory.java +++ b/vespaclient-container-plugin/src/test/java/com/yahoo/vespaxmlparser/MockFeedReaderFactory.java @@ -2,7 +2,7 @@ package com.yahoo.vespaxmlparser; import com.yahoo.document.DocumentTypeManager; -import com.yahoo.vespa.http.client.config.FeedParams; +import com.yahoo.vespa.http.server.FeedParams; import com.yahoo.vespa.http.server.FeedReaderFactory; import java.io.InputStream; |