diff options
61 files changed, 451 insertions, 428 deletions
diff --git a/application/src/main/java/com/yahoo/application/container/handler/Headers.java b/application/src/main/java/com/yahoo/application/container/handler/Headers.java index 03b31e38659..53ffe76c923 100644 --- a/application/src/main/java/com/yahoo/application/container/handler/Headers.java +++ b/application/src/main/java/com/yahoo/application/container/handler/Headers.java @@ -175,44 +175,44 @@ public class Headers implements Map<String, List<String>> { } /** - * <p>Removes the given value from the entry of the specified key.</p> + * Removes the given value from the entry of the specified key. * - * @param key The key of the entry to remove from. - * @param value The value to remove from the entry. - * @return True if the value was removed. + * @param key the key of the entry to remove from + * @param value the value to remove from the entry + * @return true if the value was removed */ public boolean remove(String key, String value) { return h.remove(key, value); } /** - * <p>Convenience method for retrieving the first value of a named header field. If the header is not set, or if the - * value list is empty, this method returns null.</p> + * Convenience method for retrieving the first value of a named header field. If the header is not set, or if the + * value list is empty, this method returns null. * - * @param key The key whose first value to return. - * @return The first value of the named header, or null. + * @param key the key whose first value to return. + * @return the first value of the named header, or null. */ public String getFirst(String key) { return h.getFirst(key); } /** - * <p>Convenience method for checking whether or not a named header field is <em>true</em>. To satisfy this, the + * Convenience method for checking whether a named header field is <em>true</em>. To satisfy this, the * header field needs to have at least 1 entry, and Boolean.valueOf() of all its values must parse as - * <em>true</em>.</p> + * <em>true</em>. * - * @param key The key whose values to parse as a boolean. - * @return The boolean value of the named header. + * @param key the key whose values to parse as a boolean + * @return the boolean value of the named header */ public boolean isTrue(String key) { return h.isTrue(key); } /** - * <p>Returns an unmodifiable list of all key-value pairs of this. This provides a flattened view on the content of - * this map.</p> + * Returns an unmodifiable list of all key-value pairs of this. This provides a flattened view on the content of + * this map. * - * @return The collection of entries. + * @return the collection of entries */ public List<Entry<String, String>> entries() { return h.entries(); diff --git a/clustercontroller-apps/src/main/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/StateRestApiV2Handler.java b/clustercontroller-apps/src/main/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/StateRestApiV2Handler.java index 5ef4c544bc8..346e58b652f 100644 --- a/clustercontroller-apps/src/main/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/StateRestApiV2Handler.java +++ b/clustercontroller-apps/src/main/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/StateRestApiV2Handler.java @@ -15,6 +15,7 @@ import java.util.TreeMap; import java.util.logging.Logger; public class StateRestApiV2Handler extends JDiscHttpRequestHandler { + private static final Logger log = Logger.getLogger(StateRestApiV2Handler.class.getName()); @Inject diff --git a/container-core/src/main/java/com/yahoo/container/handler/LogReader.java b/container-core/src/main/java/com/yahoo/container/handler/LogReader.java index 2890cbfb5ab..fe841bac68e 100644 --- a/container-core/src/main/java/com/yahoo/container/handler/LogReader.java +++ b/container-core/src/main/java/com/yahoo/container/handler/LogReader.java @@ -43,13 +43,13 @@ import static java.nio.charset.StandardCharsets.UTF_8; * @author jonmv */ class LogReader { + static final Pattern logArchivePathPattern = Pattern.compile("(\\d{4})/(\\d{2})/(\\d{2})/(\\d{2})-\\d+(\\.gz|\\.zst)?"); static final Pattern vespaLogPathPattern = Pattern.compile("vespa\\.log(?:-(\\d{4})-(\\d{2})-(\\d{2})\\.(\\d{2})-(\\d{2})-(\\d{2})(?:\\.gz|\\.zst)?)?"); private final Path logDirectory; private final Pattern logFilePattern; - LogReader(String logDirectory, String logFilePattern) { this(Paths.get(Defaults.getDefaults().underVespaHome(logDirectory)), Pattern.compile(logFilePattern)); } diff --git a/container-core/src/main/java/com/yahoo/container/jdisc/state/StateHandler.java b/container-core/src/main/java/com/yahoo/container/jdisc/state/StateHandler.java index cfd2244bd70..7a64e14e65e 100644 --- a/container-core/src/main/java/com/yahoo/container/jdisc/state/StateHandler.java +++ b/container-core/src/main/java/com/yahoo/container/jdisc/state/StateHandler.java @@ -274,15 +274,11 @@ public class StateHandler extends AbstractRequestHandler { } private static List<Tuple> collapseMetrics(MetricSnapshot snapshot, String consumer) { - switch (consumer) { - case HEALTH_PATH: - return collapseHealthMetrics(snapshot); - case "all": // deprecated name - case METRICS_PATH: - return flattenAllMetrics(snapshot); - default: - throw new IllegalArgumentException("Unknown consumer '" + consumer + "'."); - } + return switch (consumer) { + case HEALTH_PATH -> collapseHealthMetrics(snapshot); // deprecated name + case "all", METRICS_PATH -> flattenAllMetrics(snapshot); + default -> throw new IllegalArgumentException("Unknown consumer '" + consumer + "'."); + }; } private static List<Tuple> collapseHealthMetrics(MetricSnapshot snapshot) { diff --git a/container-core/src/main/java/com/yahoo/container/logging/AccessLogEntry.java b/container-core/src/main/java/com/yahoo/container/logging/AccessLogEntry.java index c469c90f6ab..2639239d23f 100644 --- a/container-core/src/main/java/com/yahoo/container/logging/AccessLogEntry.java +++ b/container-core/src/main/java/com/yahoo/container/logging/AccessLogEntry.java @@ -65,16 +65,14 @@ public class AccessLogEntry { return null; } - final Map<String, List<String>> newMapWithImmutableValues = mapValues( + Map<String, List<String>> newMapWithImmutableValues = mapValues( keyValues.entrySet(), valueList -> Collections.unmodifiableList(new ArrayList<>(valueList))); return Collections.unmodifiableMap(newMapWithImmutableValues); } } - private static <K, V1, V2> Map<K, V2> mapValues( - final Set<Map.Entry<K, V1>> entrySet, - final Function<V1, V2> valueConverter) { + private static <K, V1, V2> Map<K, V2> mapValues(Set<Map.Entry<K, V1>> entrySet, Function<V1, V2> valueConverter) { return entrySet.stream() .collect(toMap( entry -> entry.getKey(), diff --git a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/FormPostRequestHandler.java b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/FormPostRequestHandler.java index 72057563e36..d2dbfaa3514 100644 --- a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/FormPostRequestHandler.java +++ b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/FormPostRequestHandler.java @@ -54,17 +54,14 @@ class FormPostRequestHandler extends AbstractRequestHandler implements ContentCh * @param delegateHandler the "real" request handler that this handler wraps * @param contentCharsetName name of the charset to use when interpreting the content data */ - public FormPostRequestHandler( - final RequestHandler delegateHandler, - final String contentCharsetName, - final boolean removeBody) { + public FormPostRequestHandler(RequestHandler delegateHandler, String contentCharsetName, boolean removeBody) { this.delegateHandler = Objects.requireNonNull(delegateHandler); this.contentCharsetName = Objects.requireNonNull(contentCharsetName); this.removeBody = removeBody; } @Override - public ContentChannel handleRequest(final Request request, final ResponseHandler responseHandler) { + public ContentChannel handleRequest(Request request, ResponseHandler responseHandler) { Preconditions.checkArgument(request instanceof HttpRequest, "Expected HttpRequest, got " + request); Objects.requireNonNull(responseHandler, "responseHandler"); @@ -77,24 +74,24 @@ class FormPostRequestHandler extends AbstractRequestHandler implements ContentCh } @Override - public void write(final ByteBuffer buf, final CompletionHandler completionHandler) { + public void write(ByteBuffer buf, CompletionHandler completionHandler) { assert buf.hasArray(); accumulatedRequestContent.write(buf.array(), buf.arrayOffset() + buf.position(), buf.remaining()); completionHandler.completed(); } @Override - public void close(final CompletionHandler completionHandler) { - try (final ResourceReference ref = requestReference) { - final byte[] requestContentBytes = accumulatedRequestContent.toByteArray(); - final String content = new String(requestContentBytes, contentCharset); + public void close(CompletionHandler completionHandler) { + try (ResourceReference ref = requestReference) { + byte[] requestContentBytes = accumulatedRequestContent.toByteArray(); + String content = new String(requestContentBytes, contentCharset); completionHandler.completed(); - final Map<String, List<String>> parameterMap = parseFormParameters(content); + Map<String, List<String>> parameterMap = parseFormParameters(content); mergeParameters(parameterMap, request.parameters()); - final ContentChannel contentChannel = delegateHandler.handleRequest(request, responseHandler); + ContentChannel contentChannel = delegateHandler.handleRequest(request, responseHandler); if (contentChannel != null) { if (!removeBody) { - final ByteBuffer byteBuffer = ByteBuffer.wrap(requestContentBytes); + ByteBuffer byteBuffer = ByteBuffer.wrap(requestContentBytes); contentChannel.write(byteBuffer, NOOP_COMPLETION_HANDLER); } contentChannel.close(NOOP_COMPLETION_HANDLER); @@ -109,14 +106,10 @@ class FormPostRequestHandler extends AbstractRequestHandler implements ContentCh * @return a valid Charset for the charset name (never returns null) * @throws RequestException if the charset name is invalid or unsupported */ - private static Charset getCharsetByName(final String charsetName) throws RequestException { + private static Charset getCharsetByName(String charsetName) throws RequestException { try { - final Charset charset = Charset.forName(charsetName); - if (charset == null) { - throw new RequestException(UNSUPPORTED_MEDIA_TYPE, "Unsupported charset " + charsetName); - } - return charset; - } catch (final IllegalCharsetNameException |UnsupportedCharsetException e) { + return Charset.forName(charsetName); + } catch (IllegalCharsetNameException | UnsupportedCharsetException e) { throw new RequestException(UNSUPPORTED_MEDIA_TYPE, "Unsupported charset " + charsetName, e); } } @@ -127,17 +120,17 @@ class FormPostRequestHandler extends AbstractRequestHandler implements ContentCh * @param formContent raw form content data (body) * @return map of decoded parameters */ - private static Map<String, List<String>> parseFormParameters(final String formContent) { + private static Map<String, List<String>> parseFormParameters(String formContent) { if (formContent.isEmpty()) { return Collections.emptyMap(); } - final Map<String, List<String>> parameterMap = new HashMap<>(); - final String[] params = formContent.split("&"); - for (final String param : params) { - final String[] parts = param.split("="); - final String paramName = urlDecode(parts[0]); - final String paramValue = parts.length > 1 ? urlDecode(parts[1]) : ""; + Map<String, List<String>> parameterMap = new HashMap<>(); + String[] params = formContent.split("&"); + for (String param : params) { + String[] parts = param.split("="); + String paramName = urlDecode(parts[0]); + String paramValue = parts.length > 1 ? urlDecode(parts[1]) : ""; List<String> currentValues = parameterMap.get(paramName); if (currentValues == null) { currentValues = new LinkedList<>(); @@ -159,7 +152,7 @@ class FormPostRequestHandler extends AbstractRequestHandler implements ContentCh // Regardless of the charset used to transfer the request body, // all percent-escaping of non-ascii characters should use UTF-8 code points. return URLDecoder.decode(encoded, StandardCharsets.UTF_8.name()); - } catch (final UnsupportedEncodingException e) { + } catch (UnsupportedEncodingException e) { // Unfortunately, there is no URLDecoder.decode() method that takes a Charset, so we have to deal // with this exception. throw new IllegalStateException("Whoa, JVM doesn't support UTF-8 today.", e); @@ -172,11 +165,9 @@ class FormPostRequestHandler extends AbstractRequestHandler implements ContentCh * @param source containing the parameters to copy into the destination * @param destination receiver of parameters, possibly already containing data */ - private static void mergeParameters( - final Map<String,List<String>> source, - final Map<String,List<String>> destination) { + private static void mergeParameters(Map<String,List<String>> source, Map<String,List<String>> destination) { for (Map.Entry<String, List<String>> entry : source.entrySet()) { - final List<String> destinationValues = destination.get(entry.getKey()); + List<String> destinationValues = destination.get(entry.getKey()); if (destinationValues != null) { destinationValues.addAll(entry.getValue()); } else { @@ -189,4 +180,5 @@ class FormPostRequestHandler extends AbstractRequestHandler implements ContentCh public RequestHandler getDelegate() { return delegateHandler; } + } diff --git a/container-core/src/main/java/com/yahoo/processing/handler/AbstractProcessingHandler.java b/container-core/src/main/java/com/yahoo/processing/handler/AbstractProcessingHandler.java index 2c15c994bb7..f4db1c5085a 100644 --- a/container-core/src/main/java/com/yahoo/processing/handler/AbstractProcessingHandler.java +++ b/container-core/src/main/java/com/yahoo/processing/handler/AbstractProcessingHandler.java @@ -187,7 +187,7 @@ public abstract class AbstractProcessingHandler<COMPONENT extends Processor> ext private void populate(String prefixName,Map<String,?> parameters,Properties properties) { CompoundName prefix = new CompoundName(prefixName); for (Map.Entry<String,?> entry : parameters.entrySet()) - properties.set(prefix.append(entry.getKey()),entry.getValue()); + properties.set(prefix.append(entry.getKey()), entry.getValue()); } private static class FreezeListener implements Runnable, ResponseReceiver { diff --git a/container-core/src/test/java/com/yahoo/metrics/simple/BucketTest.java b/container-core/src/test/java/com/yahoo/metrics/simple/BucketTest.java index 11cc8b86a09..75d1c37c5c1 100644 --- a/container-core/src/test/java/com/yahoo/metrics/simple/BucketTest.java +++ b/container-core/src/test/java/com/yahoo/metrics/simple/BucketTest.java @@ -23,9 +23,10 @@ import com.yahoo.metrics.simple.UntypedMetric.AssumedType; * Functional tests for the value buckets, as implemented in the class Bucket, * and by extension the value store itself, UntypedValue. * - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + * @author Steinar Knutsen */ public class BucketTest { + private Bucket bucket; @BeforeEach @@ -55,23 +56,12 @@ public class BucketTest { for (Entry<Identifier, UntypedMetric> x : bucket.entrySet()) { String metricName = x.getKey().getName(); switch (metricName) { - case "nalle": - ++nalle; - break; - case "nalle_0": - ++nalle0; - break; - case "nalle_1": - ++nalle1; - break; - case "nalle_2": - ++nalle2; - break; - case "nalle_3": - ++nalle3; - break; - default: - throw new IllegalStateException(); + case "nalle" -> ++nalle; + case "nalle_0" -> ++nalle0; + case "nalle_1" -> ++nalle1; + case "nalle_2" -> ++nalle2; + case "nalle_3" -> ++nalle3; + default -> throw new IllegalStateException(); } } assertEquals(4, nalle); diff --git a/container-core/src/test/java/com/yahoo/metrics/simple/CounterTest.java b/container-core/src/test/java/com/yahoo/metrics/simple/CounterTest.java index 45a76078619..074c0c7b2e5 100644 --- a/container-core/src/test/java/com/yahoo/metrics/simple/CounterTest.java +++ b/container-core/src/test/java/com/yahoo/metrics/simple/CounterTest.java @@ -33,7 +33,7 @@ public class CounterTest { } @Test - final void testAdd() throws InterruptedException { + final void testAdd() { final String metricName = "unitTestCounter"; Counter c = receiver.declareCounter(metricName); c.add(); @@ -47,7 +47,7 @@ public class CounterTest { } @Test - final void testAddLong() throws InterruptedException { + final void testAddLong() { final String metricName = "unitTestCounter"; Counter c = receiver.declareCounter(metricName); final long twoToThePowerOfFourtyeight = 65536L * 65536L * 65536L; @@ -62,7 +62,7 @@ public class CounterTest { } @Test - final void testAddPoint() throws InterruptedException { + final void testAddPoint() { final String metricName = "unitTestCounter"; Point p = receiver.pointBuilder().set("x", 2L).set("y", 3.0d).set("z", "5").build(); Counter c = receiver.declareCounter(metricName, p); @@ -77,7 +77,7 @@ public class CounterTest { } @Test - final void testAddLongPoint() throws InterruptedException { + final void testAddLongPoint() { final String metricName = "unitTestCounter"; Point p = receiver.pointBuilder().set("x", 2L).set("y", 3.0d).set("z", "5").build(); Counter c = receiver.declareCounter(metricName, p); diff --git a/container-core/src/test/java/com/yahoo/metrics/simple/MetricsTest.java b/container-core/src/test/java/com/yahoo/metrics/simple/MetricsTest.java index f64998f0be4..dd949627f30 100644 --- a/container-core/src/test/java/com/yahoo/metrics/simple/MetricsTest.java +++ b/container-core/src/test/java/com/yahoo/metrics/simple/MetricsTest.java @@ -17,9 +17,10 @@ import com.yahoo.metrics.simple.jdisc.SimpleMetricConsumer; /** * Functional test for simple metric implementation. * - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + * @author Steinar Knutsen */ public class MetricsTest extends UnitTestSetup { + SimpleMetricConsumer metricApi; @BeforeEach @@ -36,7 +37,7 @@ public class MetricsTest extends UnitTestSetup { @Test final void smokeTest() throws InterruptedException { final String metricName = "testMetric"; - metricApi.set(metricName, Double.valueOf(1.0d), null); + metricApi.set(metricName, 1.0d, null); updater.gotData.await(10, TimeUnit.SECONDS); Bucket s = getUpdatedSnapshot(); Collection<Entry<Point, UntypedMetric>> values = s.getValuesForMetric(metricName); diff --git a/container-core/src/test/java/com/yahoo/metrics/simple/jdisc/SnapshotConverterTest.java b/container-core/src/test/java/com/yahoo/metrics/simple/jdisc/SnapshotConverterTest.java index 7981e5904f3..1d5cf264964 100644 --- a/container-core/src/test/java/com/yahoo/metrics/simple/jdisc/SnapshotConverterTest.java +++ b/container-core/src/test/java/com/yahoo/metrics/simple/jdisc/SnapshotConverterTest.java @@ -20,6 +20,7 @@ import java.util.Map; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; /** * @author bratseth @@ -50,7 +51,7 @@ public class SnapshotConverterTest { for (Map.Entry<MetricDimensions, MetricSet> entry : snapshot) { for (Map.Entry<String, String> dv : entry.getKey()) { - assertTrue(false); + fail(); } int cnt = 0; @@ -67,7 +68,7 @@ public class SnapshotConverterTest { assertEquals(42.25, ((GaugeMetric) mv.getValue()).getLast(), 0.001); assertEquals(1, ((GaugeMetric) mv.getValue()).getCount()); } else { - assertTrue(false); + fail(); } } assertEquals(3, cnt); diff --git a/docproc/src/main/java/com/yahoo/docproc/DocumentProcessor.java b/docproc/src/main/java/com/yahoo/docproc/DocumentProcessor.java index 7b8f07373a5..1f4ab3c8f00 100644 --- a/docproc/src/main/java/com/yahoo/docproc/DocumentProcessor.java +++ b/docproc/src/main/java/com/yahoo/docproc/DocumentProcessor.java @@ -90,7 +90,7 @@ public abstract class DocumentProcessor extends ChainedComponent { Map<String, String> ret = new HashMap<>(); for (Entry<Pair<String, String>, String> e : fieldMap.entrySet()) { // Remember to include tuple if doctype is unset in mapping - if (docType.equals(e.getKey().getFirst()) || e.getKey().getFirst()==null || "".equals(e.getKey().getFirst())) { + if (docType.equals(e.getKey().getFirst()) || e.getKey().getFirst() == null || "".equals(e.getKey().getFirst())) { ret.put(e.getKey().getSecond(), e.getValue()); } } diff --git a/document/src/main/java/com/yahoo/document/Document.java b/document/src/main/java/com/yahoo/document/Document.java index 5937ba00292..760b9de0199 100644 --- a/document/src/main/java/com/yahoo/document/Document.java +++ b/document/src/main/java/com/yahoo/document/Document.java @@ -242,8 +242,7 @@ public class Document extends StructuredFieldValue { @Override public boolean equals(Object o) { if (o == this) return true; - if (!(o instanceof Document)) return false; - Document other = (Document) o; + if (!(o instanceof Document other)) return false; return (super.equals(o) && docId.equals(other.docId) && header.equals(other.header)); } diff --git a/document/src/main/java/com/yahoo/document/DocumentUpdate.java b/document/src/main/java/com/yahoo/document/DocumentUpdate.java index befabfb6c07..0731344cea9 100644 --- a/document/src/main/java/com/yahoo/document/DocumentUpdate.java +++ b/document/src/main/java/com/yahoo/document/DocumentUpdate.java @@ -443,4 +443,5 @@ public class DocumentUpdate extends DocumentOperation implements Iterable<FieldP public Optional<Boolean> getOptionalCreateIfNonExistent() { return Optional.ofNullable(createIfNonExistent); } + } diff --git a/document/src/main/java/com/yahoo/document/datatypes/StructuredFieldValue.java b/document/src/main/java/com/yahoo/document/datatypes/StructuredFieldValue.java index 396a42b1237..6adbcda4772 100644 --- a/document/src/main/java/com/yahoo/document/datatypes/StructuredFieldValue.java +++ b/document/src/main/java/com/yahoo/document/datatypes/StructuredFieldValue.java @@ -46,14 +46,12 @@ public abstract class StructuredFieldValue extends CompositeFieldValue { * and using the returned value to call {@link #getFieldValue(Field)}. If the named field does not exist, this * method returns null. * - * @param fieldName The name of the field whose value to return. - * @return The value of the field, or null. + * @param fieldName the name of the field whose value to return. + * @return the value of the field, or null if it is not declared in this, or has no value set */ public FieldValue getFieldValue(String fieldName) { Field field = getField(fieldName); - if (field == null) { - return null; - } + if (field == null) return null; return getFieldValue(field); } @@ -61,10 +59,10 @@ public abstract class StructuredFieldValue extends CompositeFieldValue { * Sets the value of the given field. The type of the value must match the type of this field, i.e. * <pre>field.getDataType().getValueClass().isAssignableFrom(value.getClass())</pre> must be true. * - * @param field The field whose value to set. - * @param value The value to set. - * @return The previous value of the field, or null. - * @throws IllegalArgumentException If the value is not compatible with the field. + * @param field the field whose value to set + * @param value the value to set + * @return the previous value of the field, or null + * @throws IllegalArgumentException if the value is not compatible with the field */ public FieldValue setFieldValue(Field field, FieldValue value) { if (value == null) { diff --git a/document/src/main/java/com/yahoo/document/json/JsonFeedReader.java b/document/src/main/java/com/yahoo/document/json/JsonFeedReader.java index d0bd41c692c..4618a516f66 100644 --- a/document/src/main/java/com/yahoo/document/json/JsonFeedReader.java +++ b/document/src/main/java/com/yahoo/document/json/JsonFeedReader.java @@ -16,11 +16,9 @@ import com.yahoo.vespaxmlparser.RemoveFeedOperation; import java.io.InputStream; - /** * Facade between JsonReader and the FeedReader API. * - * <p> * The feed reader will take ownership of the input stream and close it when the * last parseable document has been read. * @@ -29,7 +27,7 @@ import java.io.InputStream; public class JsonFeedReader implements FeedReader { private final JsonReader reader; - private InputStream stream; + private final InputStream stream; private static final JsonFactory jsonFactory = new JsonFactoryBuilder().disable(JsonFactory.Feature.CANONICALIZE_FIELD_NAMES).build(); public JsonFeedReader(InputStream stream, DocumentTypeManager docMan) { diff --git a/document/src/main/java/com/yahoo/document/json/JsonReader.java b/document/src/main/java/com/yahoo/document/json/JsonReader.java index 94ce986fc81..f8de0fb959e 100644 --- a/document/src/main/java/com/yahoo/document/json/JsonReader.java +++ b/document/src/main/java/com/yahoo/document/json/JsonReader.java @@ -29,11 +29,6 @@ import static com.yahoo.document.json.readers.JsonParserHelpers.expectArrayStart */ public class JsonReader { - public Optional<DocumentParseInfo> parseDocument() throws IOException { - DocumentParser documentParser = new DocumentParser(parser); - return documentParser.parse(Optional.empty()); - } - private final JsonParser parser; private final DocumentTypeManager typeManager; private ReaderState state = ReaderState.AT_START; @@ -53,14 +48,19 @@ public class JsonReader { } } + public Optional<DocumentParseInfo> parseDocument() throws IOException { + DocumentParser documentParser = new DocumentParser(parser); + return documentParser.parse(Optional.empty()); + } + /** * Reads a single operation. The operation is not expected to be part of an array. * * @param operationType the type of operation (update or put) - * @param docIdString document ID. - * @return the document + * @param docIdString document ID + * @return the parsed document operation */ - public DocumentOperation readSingleDocument(DocumentOperationType operationType, String docIdString) { + public ParsedDocumentOperation readSingleDocument(DocumentOperationType operationType, String docIdString) { DocumentId docId = new DocumentId(docIdString); DocumentParseInfo documentParseInfo; try { @@ -72,9 +72,9 @@ public class JsonReader { } documentParseInfo.operationType = operationType; VespaJsonDocumentReader vespaJsonDocumentReader = new VespaJsonDocumentReader(typeManager.getIgnoreUndefinedFields()); - DocumentOperation operation = vespaJsonDocumentReader.createDocumentOperation( + ParsedDocumentOperation operation = vespaJsonDocumentReader.createDocumentOperation( getDocumentTypeFromString(documentParseInfo.documentId.getDocType(), typeManager), documentParseInfo); - operation.setCondition(TestAndSetCondition.fromConditionString(documentParseInfo.condition)); + operation.operation().setCondition(TestAndSetCondition.fromConditionString(documentParseInfo.condition)); return operation; } @@ -106,7 +106,7 @@ public class JsonReader { VespaJsonDocumentReader vespaJsonDocumentReader = new VespaJsonDocumentReader(typeManager.getIgnoreUndefinedFields()); DocumentOperation operation = vespaJsonDocumentReader.createDocumentOperation( getDocumentTypeFromString(documentParseInfo.get().documentId.getDocType(), typeManager), - documentParseInfo.get()); + documentParseInfo.get()).operation(); operation.setCondition(TestAndSetCondition.fromConditionString(documentParseInfo.get().condition)); return operation; } @@ -132,4 +132,5 @@ public class JsonReader { throw new IllegalArgumentException(e); } } + } diff --git a/document/src/main/java/com/yahoo/document/json/ParsedDocumentOperation.java b/document/src/main/java/com/yahoo/document/json/ParsedDocumentOperation.java new file mode 100644 index 00000000000..eb171983cf1 --- /dev/null +++ b/document/src/main/java/com/yahoo/document/json/ParsedDocumentOperation.java @@ -0,0 +1,16 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.json; + +import com.yahoo.document.DocumentOperation; + +/** + * The result of JSON parsing a single document operation + * + * @param operation + * the parsed operation + * @param fullyApplied + * true if all the JSON content could be applied, + * false if some (or all) of the fields were not poresent in this document and was ignored + */ +public record ParsedDocumentOperation(DocumentOperation operation, boolean fullyApplied) { +} diff --git a/document/src/main/java/com/yahoo/document/json/TokenBuffer.java b/document/src/main/java/com/yahoo/document/json/TokenBuffer.java index 4fff9c45ea5..e6d2021f90b 100644 --- a/document/src/main/java/com/yahoo/document/json/TokenBuffer.java +++ b/document/src/main/java/com/yahoo/document/json/TokenBuffer.java @@ -213,4 +213,12 @@ public class TokenBuffer { } return toReturn; } + + public void skipToRelativeNesting(int relativeNesting) { + int initialNesting = nesting(); + do { + next(); + } while ( nesting() > initialNesting + relativeNesting); + } + } diff --git a/document/src/main/java/com/yahoo/document/json/readers/CompositeReader.java b/document/src/main/java/com/yahoo/document/json/readers/CompositeReader.java index a2dd91b90a0..6cbc1c1e0b1 100644 --- a/document/src/main/java/com/yahoo/document/json/readers/CompositeReader.java +++ b/document/src/main/java/com/yahoo/document/json/readers/CompositeReader.java @@ -19,14 +19,22 @@ import static com.yahoo.document.json.readers.WeightedSetReader.fillWeightedSet; public class CompositeReader { - // TODO: reateComposite is extremely similar to add/remove, refactor + public static boolean populateComposite(TokenBuffer buffer, FieldValue fieldValue, boolean ignoreUndefinedFields) { + boolean fullyApplied = populateComposite(buffer.currentToken(), buffer, fieldValue, ignoreUndefinedFields); + expectCompositeEnd(buffer.currentToken()); + return fullyApplied; + } + + // TODO: createComposite is extremely similar to add/remove, refactor // yes, this suppresswarnings ugliness is by intention, the code relies on the contracts in the builders @SuppressWarnings({ "cast", "rawtypes" }) - public static void populateComposite(TokenBuffer buffer, FieldValue fieldValue, boolean ignoreUndefinedFields) { - JsonToken token = buffer.currentToken(); + private static boolean populateComposite(JsonToken token, TokenBuffer buffer, FieldValue fieldValue, + boolean ignoreUndefinedFields) { if ((token != JsonToken.START_OBJECT) && (token != JsonToken.START_ARRAY)) { throw new IllegalArgumentException("Expected '[' or '{'. Got '" + token + "'."); } + + boolean fullyApplied = true; if (fieldValue instanceof CollectionFieldValue) { DataType valueType = ((CollectionFieldValue) fieldValue).getDataType().getNestedType(); if (fieldValue instanceof WeightedSet) { @@ -39,13 +47,14 @@ public class CompositeReader { } else if (PositionDataType.INSTANCE.equals(fieldValue.getDataType())) { GeoPositionReader.fillGeoPosition(buffer, fieldValue); } else if (fieldValue instanceof StructuredFieldValue) { - StructReader.fillStruct(buffer, (StructuredFieldValue) fieldValue, ignoreUndefinedFields); + fullyApplied = StructReader.fillStruct(buffer, (StructuredFieldValue) fieldValue, ignoreUndefinedFields); } else if (fieldValue instanceof TensorFieldValue) { TensorReader.fillTensor(buffer, (TensorFieldValue) fieldValue); } else { throw new IllegalArgumentException("Expected a " + fieldValue.getClass().getName() + " but got an " + (token == JsonToken.START_OBJECT ? "object" : "array" )); } - expectCompositeEnd(buffer.currentToken()); + return fullyApplied; } + } diff --git a/document/src/main/java/com/yahoo/document/json/readers/SingleValueReader.java b/document/src/main/java/com/yahoo/document/json/readers/SingleValueReader.java index 1747e739bbf..3ae82676fa8 100644 --- a/document/src/main/java/com/yahoo/document/json/readers/SingleValueReader.java +++ b/document/src/main/java/com/yahoo/document/json/readers/SingleValueReader.java @@ -53,31 +53,17 @@ public class SingleValueReader { @SuppressWarnings("rawtypes") public static ValueUpdate readSingleUpdate(TokenBuffer buffer, DataType expectedType, String action, boolean ignoreUndefinedFields) { - ValueUpdate update; - - switch (action) { - case UPDATE_ASSIGN: - update = (buffer.currentToken() == JsonToken.VALUE_NULL) - ? ValueUpdate.createClear() - : ValueUpdate.createAssign(readSingleValue(buffer, expectedType, ignoreUndefinedFields)); - break; + return switch (action) { + case UPDATE_ASSIGN -> (buffer.currentToken() == JsonToken.VALUE_NULL) + ? ValueUpdate.createClear() + : ValueUpdate.createAssign(readSingleValue(buffer, expectedType, ignoreUndefinedFields)); // double is silly, but it's what is used internally anyway - case UPDATE_INCREMENT: - update = ValueUpdate.createIncrement(Double.valueOf(buffer.currentText())); - break; - case UPDATE_DECREMENT: - update = ValueUpdate.createDecrement(Double.valueOf(buffer.currentText())); - break; - case UPDATE_MULTIPLY: - update = ValueUpdate.createMultiply(Double.valueOf(buffer.currentText())); - break; - case UPDATE_DIVIDE: - update = ValueUpdate.createDivide(Double.valueOf(buffer.currentText())); - break; - default: - throw new IllegalArgumentException("Operation '" + buffer.currentName() + "' not implemented."); - } - return update; + case UPDATE_INCREMENT -> ValueUpdate.createIncrement(Double.valueOf(buffer.currentText())); + case UPDATE_DECREMENT -> ValueUpdate.createDecrement(Double.valueOf(buffer.currentText())); + case UPDATE_MULTIPLY -> ValueUpdate.createMultiply(Double.valueOf(buffer.currentText())); + case UPDATE_DIVIDE -> ValueUpdate.createDivide(Double.valueOf(buffer.currentText())); + default -> throw new IllegalArgumentException("Operation '" + buffer.currentName() + "' not implemented."); + }; } public static Matcher matchArithmeticOperation(String expression) { @@ -94,7 +80,7 @@ public class SingleValueReader { } } - private static FieldValue readReferenceFieldValue(final String refText, DataType expectedType) { + private static FieldValue readReferenceFieldValue(String refText, DataType expectedType) { final FieldValue value = expectedType.createFieldValue(); if (!refText.isEmpty()) { value.assign(new DocumentId(refText)); diff --git a/document/src/main/java/com/yahoo/document/json/readers/StructReader.java b/document/src/main/java/com/yahoo/document/json/readers/StructReader.java index b9eaf0d8ec6..b944d273a72 100644 --- a/document/src/main/java/com/yahoo/document/json/readers/StructReader.java +++ b/document/src/main/java/com/yahoo/document/json/readers/StructReader.java @@ -12,15 +12,32 @@ import static com.yahoo.document.json.readers.SingleValueReader.readSingleValue; public class StructReader { - public static void fillStruct(TokenBuffer buffer, StructuredFieldValue parent, boolean ignoreUndefinedFields) { + /** + * Fills this struct. + * + * @return true if all this was applied and false if it was ignored because the field does not exist + */ + public static boolean fillStruct(TokenBuffer buffer, StructuredFieldValue parent, boolean ignoreUndefinedFields) { // do note the order of initializing initNesting and token is relevant for empty docs - int initNesting = buffer.nesting(); + int initialNesting = buffer.nesting(); buffer.next(); - while (buffer.nesting() >= initNesting) { - Field field = getField(buffer, parent, ignoreUndefinedFields); + boolean fullyApplied = true; + while (buffer.nesting() >= initialNesting) { + Field field = parent.getField(buffer.currentName()); + if (field == null) { + if (! ignoreUndefinedFields) + throw new IllegalArgumentException("No field '" + buffer.currentName() + "' in the structure of type '" + + parent.getDataType().getDataTypeName() + + "', which has the fields: " + parent.getDataType().getFields()); + + buffer.skipToRelativeNesting(0); + fullyApplied = false; + continue; + } + try { - if (field != null && buffer.currentToken() != JsonToken.VALUE_NULL) { + if (buffer.currentToken() != JsonToken.VALUE_NULL) { FieldValue v = readSingleValue(buffer, field.getDataType(), ignoreUndefinedFields); parent.setFieldValue(field, v); } @@ -29,16 +46,7 @@ public class StructReader { throw new JsonReaderException(field, e); } } - } - - private static Field getField(TokenBuffer buffer, StructuredFieldValue parent, boolean ignoreUndefinedFields) { - Field field = parent.getField(buffer.currentName()); - if (field == null && ! ignoreUndefinedFields) { - throw new IllegalArgumentException("No field '" + buffer.currentName() + "' in the structure of type '" + - parent.getDataType().getDataTypeName() + - "', which has the fields: " + parent.getDataType().getFields()); - } - return field; + return fullyApplied; } } diff --git a/document/src/main/java/com/yahoo/document/json/readers/VespaJsonDocumentReader.java b/document/src/main/java/com/yahoo/document/json/readers/VespaJsonDocumentReader.java index 7bc462ec73a..22a9e7a1119 100644 --- a/document/src/main/java/com/yahoo/document/json/readers/VespaJsonDocumentReader.java +++ b/document/src/main/java/com/yahoo/document/json/readers/VespaJsonDocumentReader.java @@ -17,6 +17,7 @@ import com.yahoo.document.fieldpathupdate.AssignFieldPathUpdate; import com.yahoo.document.fieldpathupdate.FieldPathUpdate; import com.yahoo.document.fieldpathupdate.RemoveFieldPathUpdate; import com.yahoo.document.json.JsonReaderException; +import com.yahoo.document.json.ParsedDocumentOperation; import com.yahoo.document.json.TokenBuffer; import com.yahoo.document.update.FieldUpdate; @@ -50,67 +51,65 @@ public class VespaJsonDocumentReader { this.ignoreUndefinedFields = ignoreUndefinedFields; } - public DocumentOperation createDocumentOperation(DocumentType documentType, DocumentParseInfo documentParseInfo) { + public ParsedDocumentOperation createDocumentOperation(DocumentType documentType, DocumentParseInfo documentParseInfo) { final DocumentOperation documentOperation; + boolean fullyApplied = true; try { switch (documentParseInfo.operationType) { - case PUT: + case PUT -> { documentOperation = new DocumentPut(new Document(documentType, documentParseInfo.documentId)); - readPut(documentParseInfo.fieldsBuffer, (DocumentPut) documentOperation); + fullyApplied = readPut(documentParseInfo.fieldsBuffer, (DocumentPut) documentOperation); verifyEndState(documentParseInfo.fieldsBuffer, JsonToken.END_OBJECT); - break; - case REMOVE: - documentOperation = new DocumentRemove(documentParseInfo.documentId); - break; - case UPDATE: + } + case REMOVE -> documentOperation = new DocumentRemove(documentParseInfo.documentId); + case UPDATE -> { documentOperation = new DocumentUpdate(documentType, documentParseInfo.documentId); - readUpdate(documentParseInfo.fieldsBuffer, (DocumentUpdate) documentOperation); + fullyApplied = readUpdate(documentParseInfo.fieldsBuffer, (DocumentUpdate) documentOperation); verifyEndState(documentParseInfo.fieldsBuffer, JsonToken.END_OBJECT); - break; - default: - throw new IllegalStateException("Implementation out of sync with itself. This is a bug."); + } + default -> throw new IllegalStateException("Implementation out of sync with itself. This is a bug."); } } catch (JsonReaderException e) { throw JsonReaderException.addDocId(e, documentParseInfo.documentId); } if (documentParseInfo.create.isPresent()) { - if (! ( documentOperation instanceof DocumentUpdate)) { + if (! (documentOperation instanceof DocumentUpdate update)) { throw new IllegalArgumentException("Could not set create flag on non update operation."); } - DocumentUpdate update = (DocumentUpdate) documentOperation; update.setCreateIfNonExistent(documentParseInfo.create.get()); } - return documentOperation; + return new ParsedDocumentOperation(documentOperation, fullyApplied); } // Exposed for unit testing... - public void readPut(TokenBuffer buffer, DocumentPut put) { + public boolean readPut(TokenBuffer buffer, DocumentPut put) { try { if (buffer.isEmpty()) // no "fields" map throw new IllegalArgumentException(put + " is missing a 'fields' map"); - populateComposite(buffer, put.getDocument(), ignoreUndefinedFields); + return populateComposite(buffer, put.getDocument(), ignoreUndefinedFields); } catch (JsonReaderException e) { throw JsonReaderException.addDocId(e, put.getId()); } } // Exposed for unit testing... - public void readUpdate(TokenBuffer buffer, DocumentUpdate update) { + public boolean readUpdate(TokenBuffer buffer, DocumentUpdate update) { if (buffer.isEmpty()) - throw new IllegalArgumentException("update of document " + update.getId() + " is missing a 'fields' map"); + throw new IllegalArgumentException("Update of document " + update.getId() + " is missing a 'fields' map"); expectObjectStart(buffer.currentToken()); int localNesting = buffer.nesting(); buffer.next(); + boolean fullyApplied = true; while (localNesting <= buffer.nesting()) { expectObjectStart(buffer.currentToken()); String fieldName = buffer.currentName(); try { if (isFieldPath(fieldName)) { - addFieldPathUpdates(update, buffer, fieldName); + fullyApplied &= addFieldPathUpdates(update, buffer, fieldName); } else { - addFieldUpdates(update, buffer, fieldName); + fullyApplied &= addFieldUpdates(update, buffer, fieldName); } expectObjectEnd(buffer.currentToken()); } @@ -119,12 +118,18 @@ public class VespaJsonDocumentReader { } buffer.next(); } + return fullyApplied; } - private void addFieldUpdates(DocumentUpdate update, TokenBuffer buffer, String fieldName) { + private boolean addFieldUpdates(DocumentUpdate update, TokenBuffer buffer, String fieldName) { Field field = update.getType().getField(fieldName); - if (field == null) - throw new IllegalArgumentException("No field named '" + fieldName + "' in " + update.getType()); + if (field == null) { + if (! ignoreUndefinedFields) + throw new IllegalArgumentException("No field named '" + fieldName + "' in " + update.getType()); + buffer.skipToRelativeNesting(-1); + return false; + } + int localNesting = buffer.nesting(); FieldUpdate fieldUpdate = FieldUpdate.create(field); @@ -158,9 +163,10 @@ public class VespaJsonDocumentReader { buffer.next(); } update.addFieldUpdate(fieldUpdate); + return true; } - private void addFieldPathUpdates(DocumentUpdate update, TokenBuffer buffer, String fieldPath) { + private boolean addFieldPathUpdates(DocumentUpdate update, TokenBuffer buffer, String fieldPath) { int localNesting = buffer.nesting(); buffer.next(); @@ -185,6 +191,7 @@ public class VespaJsonDocumentReader { update.addFieldPathUpdate(fieldPathUpdate); buffer.next(); } + return true; // TODO: Track fullyApplied for fieldPath updates } private AssignFieldPathUpdate readAssignFieldPathUpdate(DocumentType documentType, String fieldPath, TokenBuffer buffer) { @@ -230,4 +237,5 @@ public class VespaJsonDocumentReader { Preconditions.checkState(buffer.next() == null, "Dangling data at end of operation"); Preconditions.checkState(buffer.size() == 0, "Dangling data at end of operation"); } + } diff --git a/document/src/main/java/com/yahoo/document/json/readers/WeightedSetReader.java b/document/src/main/java/com/yahoo/document/json/readers/WeightedSetReader.java index c9af929735e..7a9921498ef 100644 --- a/document/src/main/java/com/yahoo/document/json/readers/WeightedSetReader.java +++ b/document/src/main/java/com/yahoo/document/json/readers/WeightedSetReader.java @@ -10,12 +10,14 @@ import static com.yahoo.document.json.readers.JsonParserHelpers.expectObjectStar public class WeightedSetReader { + public static void fillWeightedSet(TokenBuffer buffer, DataType valueType, @SuppressWarnings("rawtypes") WeightedSet weightedSet) { int initNesting = buffer.nesting(); expectObjectStart(buffer.currentToken()); buffer.next(); iterateThroughWeightedSet(buffer, initNesting, valueType, weightedSet); } + public static void fillWeightedSetUpdate(TokenBuffer buffer, int initNesting, DataType valueType, @SuppressWarnings("rawtypes") WeightedSet weightedSet) { iterateThroughWeightedSet(buffer, initNesting, valueType, weightedSet); } @@ -29,4 +31,5 @@ public class WeightedSetReader { buffer.next(); } } + } diff --git a/document/src/main/java/com/yahoo/document/update/FieldUpdate.java b/document/src/main/java/com/yahoo/document/update/FieldUpdate.java index b2992bf4988..fc4a293f0fb 100644 --- a/document/src/main/java/com/yahoo/document/update/FieldUpdate.java +++ b/document/src/main/java/com/yahoo/document/update/FieldUpdate.java @@ -46,10 +46,10 @@ import java.util.List; * type - any name/value pair which existing in an updatable structure can be addressed by creating the Fields as * needed. For example: * <pre> - * FieldUpdate field=FieldUpdate.createIncrement(new Field("myattribute",DataType.INT),130); + * FieldUpdate field = FieldUpdate.createIncrement(new Field("myattribute",DataType.INT),130); * </pre> * - * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + * @author Einar M R Rosenvinge * @see com.yahoo.document.update.ValueUpdate * @see com.yahoo.document.DocumentUpdate */ @@ -135,7 +135,7 @@ public class FieldUpdate { * @throws IllegalArgumentException if the data type of the value update is not equal to the data type of this field */ public FieldUpdate addValueUpdate(ValueUpdate valueUpdate) { - valueUpdate.checkCompatibility(field.getDataType()); //will throw exception + valueUpdate.checkCompatibility(field.getDataType()); // will throw exception valueUpdates.add(valueUpdate); return this; } @@ -149,7 +149,7 @@ public class FieldUpdate { * @throws IllegalArgumentException if the data type of the value update is not equal to the data type of this field */ public FieldUpdate addValueUpdate(int index, ValueUpdate valueUpdate) { - valueUpdate.checkCompatibility(field.getDataType()); //will throw exception + valueUpdate.checkCompatibility(field.getDataType()); // will throw exception valueUpdates.add(index, valueUpdate); return this; } diff --git a/document/src/main/java/com/yahoo/document/update/ValueUpdate.java b/document/src/main/java/com/yahoo/document/update/ValueUpdate.java index 3aa728ca5b2..74f5ba9d30a 100644 --- a/document/src/main/java/com/yahoo/document/update/ValueUpdate.java +++ b/document/src/main/java/com/yahoo/document/update/ValueUpdate.java @@ -13,7 +13,7 @@ import java.util.List; /** * A value update represents some action to perform to a value. * - * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + * @author Einar M R Rosenvinge * @see com.yahoo.document.update.FieldUpdate * @see com.yahoo.document.DocumentUpdate * @see AddValueUpdate @@ -31,11 +31,7 @@ public abstract class ValueUpdate<T extends FieldValue> { this.valueUpdateClassID = valueUpdateClassID; } - /** - * Returns the valueUpdateClassID of this value update. - * - * @return the valueUpdateClassID of this ValueUpdate - */ + /** Returns the valueUpdateClassID of this value update. */ public ValueUpdateClassID getValueUpdateClassID() { return valueUpdateClassID; } @@ -46,7 +42,7 @@ public abstract class ValueUpdate<T extends FieldValue> { @Override public boolean equals(Object o) { - return o instanceof ValueUpdate && valueUpdateClassID == ((ValueUpdate) o).valueUpdateClassID; + return o instanceof ValueUpdate && valueUpdateClassID == ((ValueUpdate<?>) o).valueUpdateClassID; } @Override diff --git a/document/src/main/java/com/yahoo/vespaxmlparser/FeedOperation.java b/document/src/main/java/com/yahoo/vespaxmlparser/FeedOperation.java index 6b084a55309..69d851b09bc 100644 --- a/document/src/main/java/com/yahoo/vespaxmlparser/FeedOperation.java +++ b/document/src/main/java/com/yahoo/vespaxmlparser/FeedOperation.java @@ -7,7 +7,9 @@ import com.yahoo.document.DocumentUpdate; import com.yahoo.document.TestAndSetCondition; public class FeedOperation { + public enum Type {DOCUMENT, REMOVE, UPDATE, INVALID} + public static final FeedOperation INVALID = new FeedOperation(Type.INVALID); private Type type; @@ -36,4 +38,5 @@ public class FeedOperation { " testandset=" + getCondition() + '}'; } + }
\ No newline at end of file diff --git a/document/src/test/java/com/yahoo/document/json/DocumentUpdateJsonSerializerTest.java b/document/src/test/java/com/yahoo/document/json/DocumentUpdateJsonSerializerTest.java index e396fe8912b..7bdb526bb1c 100644 --- a/document/src/test/java/com/yahoo/document/json/DocumentUpdateJsonSerializerTest.java +++ b/document/src/test/java/com/yahoo/document/json/DocumentUpdateJsonSerializerTest.java @@ -96,7 +96,7 @@ public class DocumentUpdateJsonSerializerTest { private static DocumentUpdate jsonToDocumentUpdate(String jsonDoc, String docId) { final InputStream rawDoc = new ByteArrayInputStream(Utf8.toBytes(jsonDoc)); JsonReader reader = new JsonReader(types, rawDoc, parserFactory); - return (DocumentUpdate) reader.readSingleDocument(DocumentOperationType.UPDATE, docId); + return (DocumentUpdate) reader.readSingleDocument(DocumentOperationType.UPDATE, docId).operation(); } private static String documentUpdateToJson(DocumentUpdate update) { diff --git a/document/src/test/java/com/yahoo/document/json/JsonReaderTestCase.java b/document/src/test/java/com/yahoo/document/json/JsonReaderTestCase.java index 441f1fd28ea..b86d05f2d3d 100644 --- a/document/src/test/java/com/yahoo/document/json/JsonReaderTestCase.java +++ b/document/src/test/java/com/yahoo/document/json/JsonReaderTestCase.java @@ -78,6 +78,7 @@ import static com.yahoo.document.json.readers.SingleValueReader.UPDATE_INCREMENT import static com.yahoo.document.json.readers.SingleValueReader.UPDATE_MULTIPLY; import static com.yahoo.test.json.JsonTestHelper.inputJson; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertSame; @@ -103,6 +104,8 @@ public class JsonReaderTestCase { DocumentType x = new DocumentType("smoke"); x.addField(new Field("something", DataType.STRING)); x.addField(new Field("nalle", DataType.STRING)); + x.addField(new Field("field1", DataType.STRING)); + x.addField(new Field("field2", DataType.STRING)); x.addField(new Field("int1", DataType.INT)); x.addField(new Field("flag", DataType.BOOL)); types.registerDocumentType(x); @@ -216,7 +219,7 @@ public class JsonReaderTestCase { " }", "}")); DocumentPut put = (DocumentPut) r.readSingleDocument(DocumentOperationType.PUT, - "id:unittest:smoke::doc1"); + "id:unittest:smoke::doc1").operation(); smokeTestDoc(put.getDocument()); } @@ -226,7 +229,7 @@ public class JsonReaderTestCase { " 'fields': {", " 'something': {", " 'assign': 'orOther' }}}")); - DocumentUpdate doc = (DocumentUpdate) r.readSingleDocument(DocumentOperationType.UPDATE, "id:unittest:smoke::whee"); + DocumentUpdate doc = (DocumentUpdate) r.readSingleDocument(DocumentOperationType.UPDATE, "id:unittest:smoke::whee").operation(); FieldUpdate f = doc.getFieldUpdate("something"); assertEquals(1, f.size()); assertTrue(f.getValueUpdate(0) instanceof AssignValueUpdate); @@ -238,7 +241,7 @@ public class JsonReaderTestCase { " 'fields': {", " 'int1': {", " 'assign': null }}}")); - DocumentUpdate doc = (DocumentUpdate) r.readSingleDocument(DocumentOperationType.UPDATE, "id:unittest:smoke::whee"); + DocumentUpdate doc = (DocumentUpdate) r.readSingleDocument(DocumentOperationType.UPDATE, "id:unittest:smoke::whee").operation(); FieldUpdate f = doc.getFieldUpdate("int1"); assertEquals(1, f.size()); assertTrue(f.getValueUpdate(0) instanceof ClearValueUpdate); @@ -1006,17 +1009,65 @@ public class JsonReaderTestCase { } @Test - public void nonExistingFieldCanBeIgnored() throws IOException{ + public void nonExistingFieldsCanBeIgnoredInPut() throws IOException{ JsonReader r = createReader(inputJson( - "{ 'put': 'id:unittest:smoke::whee',", + "{ ", + " 'put': 'id:unittest:smoke::doc1',", " 'fields': {", - " 'smething': 'smoketest',", - " 'nalle': 'bamse' }}")); + " 'nonexisting1': 'ignored value',", + " 'field1': 'value1',", + " 'nonexisting2': {", + " 'blocks':{", + " 'a':[2.0,3.0],", + " 'b':[4.0,5.0]", + " }", + " },", + " 'field2': 'value2',", + " 'nonexisting3': 'ignored value'", + " }", + "}")); DocumentParseInfo parseInfo = r.parseDocument().get(); DocumentType docType = r.readDocumentType(parseInfo.documentId); DocumentPut put = new DocumentPut(new Document(docType, parseInfo.documentId)); + boolean fullyApplied = new VespaJsonDocumentReader(true).readPut(parseInfo.fieldsBuffer, put); + assertFalse(fullyApplied); + assertNull(put.getDocument().getField("nonexisting1")); + assertEquals("value1", put.getDocument().getFieldValue("field1").toString()); + assertNull(put.getDocument().getField("nonexisting2")); + assertEquals("value2", put.getDocument().getFieldValue("field2").toString()); + assertNull(put.getDocument().getField("nonexisting3")); + } - new VespaJsonDocumentReader(true).readPut(parseInfo.fieldsBuffer, put); + @Test + public void nonExistingFieldsCanBeIgnoredInUpdate() throws IOException{ + JsonReader r = createReader(inputJson( + "{ ", + " 'update': 'id:unittest:smoke::doc1',", + " 'fields': {", + " 'nonexisting1': { 'assign': 'ignored value' },", + " 'field1': { 'assign': 'value1' },", +// " 'nonexisting2': { " + +// " 'assign': {", +// " 'blocks': {", +// " 'a':[2.0,3.0],", +// " 'b':[4.0,5.0]", +// " }", +// " }", +// " },", + " 'field2': { 'assign': 'value2' }", +// " 'nonexisting3': { 'assign': 'ignored value' }", + " }", + "}")); + DocumentParseInfo parseInfo = r.parseDocument().get(); + DocumentType docType = r.readDocumentType(parseInfo.documentId); + DocumentUpdate update = new DocumentUpdate(docType, parseInfo.documentId); + boolean fullyApplied = new VespaJsonDocumentReader(true).readUpdate(parseInfo.fieldsBuffer, update); + assertFalse(fullyApplied); + assertNull(update.getFieldUpdate("nonexisting1")); + assertEquals("value1", update.getFieldUpdate("field1").getValueUpdates().get(0).getValue().getWrappedValue().toString()); + assertNull(update.getFieldUpdate("nonexisting2")); + assertEquals("value2", update.getFieldUpdate("field2").getValueUpdates().get(0).getValue().getWrappedValue().toString()); + assertNull(update.getFieldUpdate("nonexisting3")); } @Test @@ -1044,7 +1095,8 @@ public class JsonReaderTestCase { DocumentParseInfo parseInfo = r.parseDocument().get(); DocumentType docType = r.readDocumentType(parseInfo.documentId); DocumentPut put = new DocumentPut(new Document(docType, parseInfo.documentId)); - new VespaJsonDocumentReader(false).readPut(parseInfo.fieldsBuffer, put); + boolean fullyApplied = new VespaJsonDocumentReader(false).readPut(parseInfo.fieldsBuffer, put); + assertTrue(fullyApplied); smokeTestDoc(put.getDocument()); } @@ -1271,7 +1323,7 @@ public class JsonReaderTestCase { fail("Expected exception"); } catch (IllegalArgumentException e) { - assertEquals("update of document id:unittest:smoke::whee is missing a 'fields' map", e.getMessage()); + assertEquals("Update of document id:unittest:smoke::whee is missing a 'fields' map", e.getMessage()); } } @@ -1288,7 +1340,7 @@ public class JsonReaderTestCase { " }", "}")); DocumentPut put = (DocumentPut) r.readSingleDocument(DocumentOperationType.PUT, - "id:unittest:testnull::doc1"); + "id:unittest:testnull::doc1").operation(); Document doc = put.getDocument(); assertFieldValueNull(doc, "intfield"); assertFieldValueNull(doc, "stringfield"); diff --git a/documentapi/src/main/java/com/yahoo/documentapi/ProgressToken.java b/documentapi/src/main/java/com/yahoo/documentapi/ProgressToken.java index 4a77d30ec92..682bcc54a56 100644 --- a/documentapi/src/main/java/com/yahoo/documentapi/ProgressToken.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/ProgressToken.java @@ -22,6 +22,7 @@ import com.yahoo.vespa.objects.BufferSerializer; public class ProgressToken { private static final Logger log = Logger.getLogger(ProgressToken.class.getName()); + /** * Any bucket kept track of by a <code>ProgressToken</code> instance may * be in one of two states: pending or active. <em>Pending</em> means that @@ -848,4 +849,5 @@ public class ProgressToken { pendingBucketCount = 0; activeBucketCount = 0; } + } diff --git a/documentapi/src/main/java/com/yahoo/documentapi/Response.java b/documentapi/src/main/java/com/yahoo/documentapi/Response.java index 3778189e272..133c3586276 100644 --- a/documentapi/src/main/java/com/yahoo/documentapi/Response.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/Response.java @@ -9,10 +9,8 @@ import static com.yahoo.documentapi.Response.Outcome.ERROR; import static com.yahoo.documentapi.Response.Outcome.SUCCESS; /** - * <p>An asynchronous response from the document api. - * Subclasses of this provide additional response information for particular operations.</p> - * - * <p>This is a <i>value object</i>.</p> + * An asynchronous response from the document api. + * Subclasses of this provide additional response information for particular operations. * * @author bratseth */ diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/MapVisitorMessage.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/MapVisitorMessage.java index 15e971329e6..eb9640a38b8 100644 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/MapVisitorMessage.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/MapVisitorMessage.java @@ -44,4 +44,5 @@ public class MapVisitorMessage extends VisitorMessage { public String toString() { return "MapVisitorMessage(" + data.toString() + ")"; } + } diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutableFactories60.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutableFactories60.java index c4d4fb216be..099839672a2 100644 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutableFactories60.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutableFactories60.java @@ -202,10 +202,10 @@ public abstract class RoutableFactories60 { buf.putInt(null, msg.getBuckets().size()); for (BucketId id : msg.getBuckets()) { long rawid = id.getRawId(); - long reversed = ((rawid >>> 56) & 0x00000000000000FFl) | ((rawid >>> 40) & 0x000000000000FF00l) | - ((rawid >>> 24) & 0x0000000000FF0000l) | ((rawid >>> 8) & 0x00000000FF000000l) | - ((rawid << 8) & 0x000000FF00000000l) | ((rawid << 24) & 0x0000FF0000000000l) | - ((rawid << 40) & 0x00FF000000000000l) | ((rawid << 56) & 0xFF00000000000000l); + long reversed = ((rawid >>> 56) & 0x00000000000000FFL) | ((rawid >>> 40) & 0x000000000000FF00L) | + ((rawid >>> 24) & 0x0000000000FF0000L) | ((rawid >>> 8) & 0x00000000FF000000L) | + ((rawid << 8) & 0x000000FF00000000L) | ((rawid << 24) & 0x0000FF0000000000L) | + ((rawid << 40) & 0x00FF000000000000L) | ((rawid << 56) & 0xFF00000000000000L); buf.putLong(null, reversed); } diff --git a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/MessageBusVisitorSessionTestCase.java b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/MessageBusVisitorSessionTestCase.java index 6b191b50a57..04f4800231f 100755 --- a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/MessageBusVisitorSessionTestCase.java +++ b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/MessageBusVisitorSessionTestCase.java @@ -56,6 +56,7 @@ import static org.junit.Assert.fail; // TODO replace explicit pre-mockito mock classes with proper mockito mocks wherever possible public class MessageBusVisitorSessionTestCase { + private class MockSender implements MessageBusVisitorSession.Sender { private int maxPending = 1000; private int pendingCount = 0; @@ -2520,4 +2521,5 @@ public class MessageBusVisitorSessionTestCase { // TODO: consider refactoring locking granularity // TODO: figure out if we risk a re-run of the "too many tasks" issue + } diff --git a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/ErrorCodesTest.java b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/ErrorCodesTest.java index 3b4fac60fcd..ba4ae381057 100644 --- a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/ErrorCodesTest.java +++ b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/ErrorCodesTest.java @@ -18,6 +18,7 @@ import static org.junit.Assert.assertEquals; * @author vekterli */ public class ErrorCodesTest { + private class NamedErrorCodes { private final TreeMap<String, Integer> nameAndCode = new TreeMap<>(); diff --git a/flags/src/main/java/com/yahoo/vespa/flags/json/DimensionHelper.java b/flags/src/main/java/com/yahoo/vespa/flags/json/DimensionHelper.java index c2a7ce56054..ad1242aa7e9 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/json/DimensionHelper.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/json/DimensionHelper.java @@ -11,7 +11,9 @@ import java.util.stream.Collectors; * @author hakonhall */ public class DimensionHelper { - private static Map<FetchVector.Dimension, String> serializedDimensions = new HashMap<>(); + + private static final Map<FetchVector.Dimension, String> serializedDimensions = new HashMap<>(); + static { serializedDimensions.put(FetchVector.Dimension.ZONE_ID, "zone"); serializedDimensions.put(FetchVector.Dimension.HOSTNAME, "hostname"); @@ -29,7 +31,7 @@ public class DimensionHelper { } } - private static Map<String, FetchVector.Dimension> deserializedDimensions = serializedDimensions. + private static final Map<String, FetchVector.Dimension> deserializedDimensions = serializedDimensions. entrySet().stream().collect(Collectors.toMap(Map.Entry::getValue, Map.Entry::getKey)); public static String toWire(FetchVector.Dimension dimension) { @@ -51,4 +53,5 @@ public class DimensionHelper { } private DimensionHelper() { } + } diff --git a/flags/src/main/java/com/yahoo/vespa/flags/json/FetchVectorHelper.java b/flags/src/main/java/com/yahoo/vespa/flags/json/FetchVectorHelper.java index 0b19be21b76..c0bb3128924 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/json/FetchVectorHelper.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/json/FetchVectorHelper.java @@ -10,6 +10,7 @@ import java.util.stream.Collectors; * @author hakonhall */ public class FetchVectorHelper { + public static Map<String, String> toWire(FetchVector vector) { Map<FetchVector.Dimension, String> map = vector.toMap(); if (map.isEmpty()) return null; @@ -24,4 +25,5 @@ public class FetchVectorHelper { entry -> DimensionHelper.fromWire(entry.getKey()), Map.Entry::getValue))); } + } diff --git a/integration/intellij/src/main/java/ai/vespa/intellij/schema/findUsages/FunctionDefinitionFinder.java b/integration/intellij/src/main/java/ai/vespa/intellij/schema/findUsages/FunctionDefinitionFinder.java index 7d8aafc8f8b..89e0165d55e 100644 --- a/integration/intellij/src/main/java/ai/vespa/intellij/schema/findUsages/FunctionDefinitionFinder.java +++ b/integration/intellij/src/main/java/ai/vespa/intellij/schema/findUsages/FunctionDefinitionFinder.java @@ -51,7 +51,7 @@ public class FunctionDefinitionFinder extends UsageFinder { private boolean findDefinitionIn(RankProfile profile) { for (var entry : profile.definedFunctions().entrySet()) { // TODO: Resolve the right function in the list by parameter count - if (entry.getKey().equals(functionNameToFind) && entry.getValue().size() > 0) + if (entry.key().equals(functionNameToFind) && entry.getValue().size() > 0) processor().process(new UsageInfo(entry.getValue().get(0).definition())); } return true; diff --git a/integration/intellij/src/main/java/ai/vespa/intellij/schema/model/Schema.java b/integration/intellij/src/main/java/ai/vespa/intellij/schema/model/Schema.java index 88cbdd3e07a..5479949d25a 100644 --- a/integration/intellij/src/main/java/ai/vespa/intellij/schema/model/Schema.java +++ b/integration/intellij/src/main/java/ai/vespa/intellij/schema/model/Schema.java @@ -92,7 +92,7 @@ public class Schema { Map<String, List<Function>> functions = new HashMap<>(); for (var profile : rankProfiles().values()) { for (var entry : profile.definedFunctions().entrySet()) - functions.computeIfAbsent(entry.getKey(), k -> new ArrayList<>()).addAll(entry.getValue()); + functions.computeIfAbsent(entry.key(), k -> new ArrayList<>()).addAll(entry.getValue()); } return functions; } diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/HeaderFields.java b/jdisc_core/src/main/java/com/yahoo/jdisc/HeaderFields.java index af57cf39e73..fc8699c907f 100644 --- a/jdisc_core/src/main/java/com/yahoo/jdisc/HeaderFields.java +++ b/jdisc_core/src/main/java/com/yahoo/jdisc/HeaderFields.java @@ -86,8 +86,8 @@ public class HeaderFields implements Map<String, List<String>> { } /** - * <p>Adds the given value to the entry of the specified key. If no entry exists for the given key, a new one is - * created containing only the given value.</p> + * Adds the given value to the entry of the specified key. If no entry exists for the given key, a new one is + * created containing only the given value. * * @param key The key with which the specified value is to be associated. * @param value The value to be added to the list associated with the specified key. @@ -102,11 +102,11 @@ public class HeaderFields implements Map<String, List<String>> { } /** - * <p>Adds the given values to the entry of the specified key. If no entry exists for the given key, a new one is - * created containing only the given values.</p> + * Adds the given values to the entry of the specified key. If no entry exists for the given key, a new one is + * created containing only the given values. * - * @param key The key with which the specified value is to be associated. - * @param values The values to be added to the list associated with the specified key. + * @param key the key with which the specified value is to be associated. + * @param values the values to be added to the list associated with the specified key. */ public void add(String key, List<String> values) { List<String> lst = content.get(key); @@ -118,10 +118,10 @@ public class HeaderFields implements Map<String, List<String>> { } /** - * <p>Adds all the entries of the given map to this. This is the same as calling {@link #add(String, List)} for each - * entry in <code>values</code>.</p> + * Adds all the entries of the given map to this. This is the same as calling {@link #add(String, List)} for each + * entry in <code>values</code>. * - * @param values The values to be added to this. + * @param values the values to be added to this. */ public void addAll(Map<? extends String, ? extends List<String>> values) { for (Entry<? extends String, ? extends List<String>> entry : values.entrySet()) { @@ -193,11 +193,11 @@ public class HeaderFields implements Map<String, List<String>> { } /** - * <p>Convenience method for retrieving the first value of a named header field. If the header is not set, or if the - * value list is empty, this method returns null.</p> + * Convenience method for retrieving the first value of a named header field. If the header is not set, or if the + * value list is empty, this method returns null. * - * @param key The key whose first value to return. - * @return The first value of the named header, or null. + * @param key the key whose first value to return + * @return the first value of the named header, or null */ public String getFirst(String key) { List<String> lst = get(key); @@ -208,12 +208,12 @@ public class HeaderFields implements Map<String, List<String>> { } /** - * <p>Convenience method for checking whether or not a named header field is <em>true</em>. To satisfy this, the + * Convenience method for checking whether a named header field is <em>true</em>. To satisfy this, the * header field needs to have at least 1 entry, and Boolean.valueOf() of all its values must parse as - * <em>true</em>.</p> + * <em>true</em>. * - * @param key The key whose values to parse as a boolean. - * @return The boolean value of the named header. + * @param key the key whose values to parse as a boolean. + * @return the boolean value of the named header. */ public boolean isTrue(String key) { List<String> lst = content.get(key); @@ -249,10 +249,10 @@ public class HeaderFields implements Map<String, List<String>> { } /** - * <p>Returns an unmodifiable list of all key-value pairs of this. This provides a flattened view on the content of - * this map.</p> + * Returns an unmodifiable list of all key-value pairs of this. This provides a flattened view on the content of + * this map. * - * @return The collection of entries. + * @return the collection of entries */ public List<Entry<String, String>> entries() { List<Entry<String, String>> list = new ArrayList<>(content.size()); @@ -304,6 +304,7 @@ public class HeaderFields implements Map<String, List<String>> { public String toString() { return key + '=' + value; } + } } diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/handler/ResponseHandler.java b/jdisc_core/src/main/java/com/yahoo/jdisc/handler/ResponseHandler.java index d450d18d952..318ce5fa61b 100644 --- a/jdisc_core/src/main/java/com/yahoo/jdisc/handler/ResponseHandler.java +++ b/jdisc_core/src/main/java/com/yahoo/jdisc/handler/ResponseHandler.java @@ -11,7 +11,7 @@ import com.yahoo.jdisc.service.ClientProvider; * {@link ClientProvider#handleRequest(Request, ResponseHandler)} and {@link RequestHandler#handleRequest(Request, * ResponseHandler)}).</p> * - * <p>The jDISC API has intentionally been designed as not to provide a implicit reference from Response to + * <p>The jDISC API is designed to not provide an implicit reference from Response to * corresponding Request, but rather leave that to the implementation of context-aware ResponseHandlers. By creating * light-weight ResponseHandlers on a per-Request basis, any necessary reference can be embedded within.</p> * diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/refcount/DebugReferencesWithStack.java b/jdisc_core/src/main/java/com/yahoo/jdisc/refcount/DebugReferencesWithStack.java index db6c266534f..1910c088263 100644 --- a/jdisc_core/src/main/java/com/yahoo/jdisc/refcount/DebugReferencesWithStack.java +++ b/jdisc_core/src/main/java/com/yahoo/jdisc/refcount/DebugReferencesWithStack.java @@ -17,6 +17,7 @@ import java.util.logging.Logger; * @author baldersheim */ public class DebugReferencesWithStack implements References { + private static final Logger log = Logger.getLogger(DebugReferencesWithStack.class.getName()); private final Map<Throwable, Object> activeReferences = new HashMap<>(); private final DestructableResource resource; diff --git a/logserver/src/test/java/com/yahoo/logserver/handlers/logmetrics/test/LogMetricsTestCase.java b/logserver/src/test/java/com/yahoo/logserver/handlers/logmetrics/test/LogMetricsTestCase.java index 5602d5be0d6..ffd9ce15778 100644 --- a/logserver/src/test/java/com/yahoo/logserver/handlers/logmetrics/test/LogMetricsTestCase.java +++ b/logserver/src/test/java/com/yahoo/logserver/handlers/logmetrics/test/LogMetricsTestCase.java @@ -17,6 +17,7 @@ import static org.junit.Assert.*; * @author hmusum */ public class LogMetricsTestCase { + // Some of the tests depend upon the number of messages for a // host, log level etc. to succeed, so you may have update the // tests if you change something in mStrings. config, debug and @@ -72,7 +73,7 @@ public class LogMetricsTestCase { * each level). */ @Test - public void testLevelCountAggregated() throws java.io.IOException, InvalidLogFormatException { + public void testLevelCountAggregated() { LogMetricsHandler a = new LogMetricsHandler(); for (LogMessage aMsg : msg) { @@ -84,28 +85,13 @@ public class LogMetricsTestCase { for (Map.Entry<String, Long> entry : levelCount.entrySet()) { String key = entry.getKey(); switch (key) { - case "config": - assertEquals(entry.getValue(), Long.valueOf(1)); - break; - case "info": - assertEquals(entry.getValue(), Long.valueOf(4)); - break; - case "warning": - assertEquals(entry.getValue(), Long.valueOf(1)); - break; - case "severe": - assertEquals(entry.getValue(), Long.valueOf(0)); - break; - case "error": - assertEquals(entry.getValue(), Long.valueOf(1)); - break; - case "fatal": - assertEquals(entry.getValue(), Long.valueOf(1)); - break; - case "debug": - assertEquals(entry.getValue(), Long.valueOf(0)); // always 0 - - break; + case "config" -> assertEquals(entry.getValue(), Long.valueOf(1)); + case "info" -> assertEquals(entry.getValue(), Long.valueOf(4)); + case "warning" -> assertEquals(entry.getValue(), Long.valueOf(1)); + case "severe" -> assertEquals(entry.getValue(), Long.valueOf(0)); + case "error" -> assertEquals(entry.getValue(), Long.valueOf(1)); + case "fatal" -> assertEquals(entry.getValue(), Long.valueOf(1)); + case "debug" -> assertEquals(entry.getValue(), Long.valueOf(0)); // always 0 } } a.close(); diff --git a/messagebus/src/main/java/com/yahoo/messagebus/routing/RoutingTable.java b/messagebus/src/main/java/com/yahoo/messagebus/routing/RoutingTable.java index 30c1a2fd5b3..74ae46353c5 100644 --- a/messagebus/src/main/java/com/yahoo/messagebus/routing/RoutingTable.java +++ b/messagebus/src/main/java/com/yahoo/messagebus/routing/RoutingTable.java @@ -165,43 +165,29 @@ public class RoutingTable { * Constructs a new iterator based on a given map. This is private so that only a {@link RoutingTable} can * create one. * - * @param hops The map to iterate through. + * @param hops the map to iterate through */ private HopIterator(Map<String, HopBlueprint> hops) { it = hops.entrySet().iterator(); next(); } - /** - * Steps to the next hop in the map. - */ + /** Steps to the next hop in the map. */ public void next() { entry = it.hasNext() ? it.next() : null; } - /** - * Returns whether or not this iterator is valid. - * - * @return True if valid. - */ + /** Returns whether this iterator is valid. */ public boolean isValid() { return entry != null; } - /** - * Returns the name of the current hop. - * - * @return The name. - */ + /** Returns the name of the current hop. */ public String getName() { return entry.getKey(); } - /** - * Returns the current hop. - * - * @return The hop. - */ + /** Returns the current hop. */ public HopBlueprint getHop() { return entry.getValue(); } @@ -220,43 +206,29 @@ public class RoutingTable { * Constructs a new iterator based on a given map. This is private so that only a {@link RoutingTable} can * create one. * - * @param routes The map to iterate through. + * @param routes the map to iterate through */ private RouteIterator(Map<String, Route> routes) { it = routes.entrySet().iterator(); next(); } - /** - * Steps to the next route in the map. - */ + /** Steps to the next route in the map. */ public void next() { entry = it.hasNext() ? it.next() : null; } - /** - * Returns whether or not this iterator is valid. - * - * @return True if valid. - */ + /** Returns whether this iterator is valid. */ public boolean isValid() { return entry != null; } - /** - * Returns the name of the current route. - * - * @return The name. - */ + /** Returns the name of the current route. */ public String getName() { return entry.getKey(); } - /** - * Returns the current route. - * - * @return The route. - */ + /** Returns the current route. */ public Route getRoute() { return entry.getValue(); } diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/ApplicationMetricsRetriever.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/ApplicationMetricsRetriever.java index 1cb6e1d665b..900f7df6ea2 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/ApplicationMetricsRetriever.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/ApplicationMetricsRetriever.java @@ -167,7 +167,7 @@ public class ApplicationMetricsRetriever extends AbstractComponent implements Ru } catch (InterruptedException | ExecutionException | TimeoutException e) { Throwable cause = e.getCause(); if ( e instanceof ExecutionException && ((cause instanceof SocketException) || cause instanceof ConnectTimeoutException)) { - log.log(Level.FINE, "Failed retrieving metrics for '" + entry.getKey() + "' : " + cause.getMessage()); + log.log(Level.FINE, "Failed retrieving metrics for '" + entry.getKey() + "' : " + cause.getMessage()); } else { log.log(Level.WARNING, "Failed retrieving metrics for '" + entry.getKey() + "' : ", e); } diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/json/YamasJsonUtil.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/json/YamasJsonUtil.java index b5b8e30a218..13b2a8d859c 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/json/YamasJsonUtil.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/json/YamasJsonUtil.java @@ -27,6 +27,7 @@ import static java.util.logging.Level.WARNING; * @author gjoranv */ public class YamasJsonUtil { + private static final Logger log = Logger.getLogger(YamasJsonUtil.class.getName()); private static final JsonFactory factory = JsonFactory.builder() .enable(StreamWriteFeature.WRITE_BIGDECIMAL_AS_PLAIN) diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/health/ApplicationHealthMonitor.java b/service-monitor/src/main/java/com/yahoo/vespa/service/health/ApplicationHealthMonitor.java index a209b4111ed..b7bf023eebd 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/health/ApplicationHealthMonitor.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/health/ApplicationHealthMonitor.java @@ -23,6 +23,7 @@ import java.util.stream.Collectors; * @author hakon */ class ApplicationHealthMonitor implements ServiceStatusProvider, AutoCloseable { + private final ApplicationId applicationId; private final StateV1HealthModel healthModel; @@ -86,4 +87,5 @@ class ApplicationHealthMonitor implements ServiceStatusProvider, AutoCloseable { monitors.clear(); } } + } diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/DocumentV1ApiHandler.java b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/DocumentV1ApiHandler.java index c72bc1ef4c5..2d97c33741e 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/DocumentV1ApiHandler.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/DocumentV1ApiHandler.java @@ -12,7 +12,6 @@ import com.yahoo.container.core.documentapi.VespaDocumentAccess; import com.yahoo.container.jdisc.ContentChannelOutputStream; import com.yahoo.document.Document; import com.yahoo.document.DocumentId; -import com.yahoo.document.DocumentOperation; import com.yahoo.document.DocumentPut; import com.yahoo.document.DocumentRemove; import com.yahoo.document.DocumentTypeManager; @@ -26,6 +25,7 @@ import com.yahoo.document.idstring.IdIdString; import com.yahoo.document.json.DocumentOperationType; import com.yahoo.document.json.JsonReader; import com.yahoo.document.json.JsonWriter; +import com.yahoo.document.json.ParsedDocumentOperation; import com.yahoo.document.restapi.DocumentOperationExecutorConfig; import com.yahoo.document.select.parser.ParseException; import com.yahoo.documentapi.AckToken; @@ -67,6 +67,7 @@ import com.yahoo.restapi.Path; import com.yahoo.search.query.ParameterParser; import com.yahoo.text.Text; import com.yahoo.vespa.config.content.AllClustersBucketSpacesConfig; +import com.yahoo.vespa.http.server.Headers; import com.yahoo.vespa.http.server.MetricNames; import com.yahoo.yolean.Exceptions; import com.yahoo.yolean.Exceptions.RunnableThrowingIOException; @@ -215,7 +216,7 @@ public class DocumentV1ApiHandler extends AbstractRequestHandler { this.operations = new ConcurrentLinkedDeque<>(); long resendDelayMS = SystemTimer.adjustTimeoutByDetectedHz(Duration.ofMillis(executorConfig.resendDelayMillis())).toMillis(); - // TODO: Here it would be better do have dedicated threads with different wait depending on blocked or empty. + // TODO: Here it would be better to have dedicated threads with different wait depending on blocked or empty. this.dispatcher.scheduleWithFixedDelay(this::dispatchEnqueued, resendDelayMS, resendDelayMS, MILLISECONDS); this.visitDispatcher.scheduleWithFixedDelay(this::dispatchVisitEnqueued, resendDelayMS, resendDelayMS, MILLISECONDS); } @@ -238,7 +239,7 @@ public class DocumentV1ApiHandler extends AbstractRequestHandler { MILLISECONDS); Path requestPath = Path.withoutValidation(request.getUri()); // No segment validation here, as document IDs can be anything. - for (String path : handlers.keySet()) + for (String path : handlers.keySet()) { if (requestPath.matches(path)) { Map<Method, Handler> methods = handlers.get(path); if (methods.containsKey(request.getMethod())) @@ -249,6 +250,7 @@ public class DocumentV1ApiHandler extends AbstractRequestHandler { methodNotAllowed(request, methods.keySet(), responseHandler); } + } notFound(request, handlers.keySet(), responseHandler); } catch (IllegalArgumentException e) { @@ -398,10 +400,10 @@ public class DocumentV1ApiHandler extends AbstractRequestHandler { parameters.setFieldSet(DocIdOnly.NAME); String type = path.documentType().orElseThrow(() -> new IllegalStateException("Document type must be specified for mass updates")); IdIdString dummyId = new IdIdString("dummy", type, "", ""); - DocumentUpdate update = parser.parseUpdate(in, dummyId.toString()); - update.setCondition(new TestAndSetCondition(requireProperty(request, SELECTION))); + ParsedDocumentOperation update = parser.parseUpdate(in, dummyId.toString()); + update.operation().setCondition(new TestAndSetCondition(requireProperty(request, SELECTION))); return () -> { - visitAndUpdate(request, parameters, handler, update, cluster.name()); + visitAndUpdate(request, parameters, update.fullyApplied(), handler, (DocumentUpdate)update.operation(), cluster.name()); return true; // VisitorSession has its own throttle handling. }; }); @@ -448,21 +450,21 @@ public class DocumentV1ApiHandler extends AbstractRequestHandler { private ContentChannel postDocument(HttpRequest request, DocumentPath path, ResponseHandler rawHandler) { ResponseHandler handler = new MeasuringResponseHandler(rawHandler, com.yahoo.documentapi.metrics.DocumentOperationType.PUT, clock.instant()); if (getProperty(request, DRY_RUN, booleanParser).orElse(false)) { - handleFeedOperation(path, handler, new com.yahoo.documentapi.Response(-1)); + handleFeedOperation(path, true, handler, new com.yahoo.documentapi.Response(-1)); return ignoredContent; } return new ForwardingContentChannel(in -> { enqueueAndDispatch(request, handler, () -> { - DocumentPut put = parser.parsePut(in, path.id().toString()); - getProperty(request, CONDITION).map(TestAndSetCondition::new).ifPresent(put::setCondition); + ParsedDocumentOperation put = parser.parsePut(in, path.id().toString()); + getProperty(request, CONDITION).map(TestAndSetCondition::new).ifPresent(c -> put.operation().setCondition(c)); DocumentOperationParameters parameters = parametersFromRequest(request, ROUTE) .withResponseHandler(response -> { outstanding.decrementAndGet(); updatePutMetrics(response.outcome()); - handleFeedOperation(path, handler, response); + handleFeedOperation(path, put.fullyApplied(), handler, response); }); - return () -> dispatchOperation(() -> asyncSession.put(put, parameters)); + return () -> dispatchOperation(() -> asyncSession.put((DocumentPut)put.operation(), parameters)); }); }); } @@ -470,20 +472,21 @@ public class DocumentV1ApiHandler extends AbstractRequestHandler { private ContentChannel putDocument(HttpRequest request, DocumentPath path, ResponseHandler rawHandler) { ResponseHandler handler = new MeasuringResponseHandler(rawHandler, com.yahoo.documentapi.metrics.DocumentOperationType.UPDATE, clock.instant()); if (getProperty(request, DRY_RUN, booleanParser).orElse(false)) { - handleFeedOperation(path, handler, new com.yahoo.documentapi.Response(-1)); + handleFeedOperation(path, true, handler, new com.yahoo.documentapi.Response(-1)); return ignoredContent; } return new ForwardingContentChannel(in -> { enqueueAndDispatch(request, handler, () -> { - DocumentUpdate update = parser.parseUpdate(in, path.id().toString()); + ParsedDocumentOperation parsed = parser.parseUpdate(in, path.id().toString()); + DocumentUpdate update = (DocumentUpdate)parsed.operation(); getProperty(request, CONDITION).map(TestAndSetCondition::new).ifPresent(update::setCondition); getProperty(request, CREATE, booleanParser).ifPresent(update::setCreateIfNonExistent); DocumentOperationParameters parameters = parametersFromRequest(request, ROUTE) .withResponseHandler(response -> { outstanding.decrementAndGet(); updateUpdateMetrics(response.outcome(), update.getCreateIfNonExistent()); - handleFeedOperation(path, handler, response); + handleFeedOperation(path, parsed.fullyApplied(), handler, response); }); return () -> dispatchOperation(() -> asyncSession.update(update, parameters)); }); @@ -493,7 +496,7 @@ public class DocumentV1ApiHandler extends AbstractRequestHandler { private ContentChannel deleteDocument(HttpRequest request, DocumentPath path, ResponseHandler rawHandler) { ResponseHandler handler = new MeasuringResponseHandler(rawHandler, com.yahoo.documentapi.metrics.DocumentOperationType.REMOVE, clock.instant()); if (getProperty(request, DRY_RUN, booleanParser).orElse(false)) { - handleFeedOperation(path, handler, new com.yahoo.documentapi.Response(-1)); + handleFeedOperation(path, true, handler, new com.yahoo.documentapi.Response(-1)); return ignoredContent; } @@ -504,7 +507,7 @@ public class DocumentV1ApiHandler extends AbstractRequestHandler { .withResponseHandler(response -> { outstanding.decrementAndGet(); updateRemoveMetrics(response.outcome()); - handleFeedOperation(path, handler, response); + handleFeedOperation(path, true, handler, response); }); return () -> dispatchOperation(() -> asyncSession.remove(remove, parameters)); }); @@ -659,10 +662,16 @@ public class DocumentV1ApiHandler extends AbstractRequestHandler { return response; } - /** Commits a response with the given status code and some default headers, and writes whatever content is buffered. */ synchronized void commit(int status) throws IOException { + commit(status, true); + } + + /** Commits a response with the given status code and some default headers, and writes whatever content is buffered. */ + synchronized void commit(int status, boolean fullyApplied) throws IOException { Response response = new Response(status); - response.headers().addAll(Map.of("Content-Type", List.of("application/json; charset=UTF-8"))); + response.headers().add("Content-Type", List.of("application/json; charset=UTF-8")); + if (! fullyApplied) + response.headers().add(Headers.IGNORED_FIELDS, "true"); try { channel = handler.handleResponse(response); buffer.connectTo(channel); @@ -1023,15 +1032,15 @@ public class DocumentV1ApiHandler extends AbstractRequestHandler { this.manager = new DocumentTypeManager(config); } - DocumentPut parsePut(InputStream inputStream, String docId) { - return (DocumentPut) parse(inputStream, docId, DocumentOperationType.PUT); + ParsedDocumentOperation parsePut(InputStream inputStream, String docId) { + return parse(inputStream, docId, DocumentOperationType.PUT); } - DocumentUpdate parseUpdate(InputStream inputStream, String docId) { - return (DocumentUpdate) parse(inputStream, docId, DocumentOperationType.UPDATE); + ParsedDocumentOperation parseUpdate(InputStream inputStream, String docId) { + return parse(inputStream, docId, DocumentOperationType.UPDATE); } - private DocumentOperation parse(InputStream inputStream, String docId, DocumentOperationType operation) { + private ParsedDocumentOperation parse(InputStream inputStream, String docId, DocumentOperationType operation) { return new JsonReader(manager, inputStream, jsonFactory).readSingleDocument(operation, docId); } @@ -1041,7 +1050,11 @@ public class DocumentV1ApiHandler extends AbstractRequestHandler { void onSuccess(Document document, JsonResponse response) throws IOException; } - private static void handle(DocumentPath path, HttpRequest request, ResponseHandler handler, com.yahoo.documentapi.Response response, SuccessCallback callback) { + private static void handle(DocumentPath path, + HttpRequest request, + ResponseHandler handler, + com.yahoo.documentapi.Response response, + SuccessCallback callback) { try (JsonResponse jsonResponse = JsonResponse.create(path, handler, request)) { jsonResponse.writeTrace(response.getTrace()); if (response.isSuccess()) @@ -1049,25 +1062,18 @@ public class DocumentV1ApiHandler extends AbstractRequestHandler { else { jsonResponse.writeMessage(response.getTextMessage()); switch (response.outcome()) { - case NOT_FOUND: - jsonResponse.commit(Response.Status.NOT_FOUND); - break; - case CONDITION_FAILED: - jsonResponse.commit(Response.Status.PRECONDITION_FAILED); - break; - case INSUFFICIENT_STORAGE: - jsonResponse.commit(Response.Status.INSUFFICIENT_STORAGE); - break; - case TIMEOUT: - jsonResponse.commit(Response.Status.GATEWAY_TIMEOUT); - break; - case ERROR: + case NOT_FOUND -> jsonResponse.commit(Response.Status.NOT_FOUND); + case CONDITION_FAILED -> jsonResponse.commit(Response.Status.PRECONDITION_FAILED); + case INSUFFICIENT_STORAGE -> jsonResponse.commit(Response.Status.INSUFFICIENT_STORAGE); + case TIMEOUT -> jsonResponse.commit(Response.Status.GATEWAY_TIMEOUT); + case ERROR -> { log.log(FINE, () -> "Exception performing document operation: " + response.getTextMessage()); jsonResponse.commit(Response.Status.BAD_GATEWAY); - break; - default: + } + default -> { log.log(WARNING, "Unexpected document API operation outcome '" + response.outcome() + "' " + response.getTextMessage()); jsonResponse.commit(Response.Status.BAD_GATEWAY); + } } } } @@ -1076,8 +1082,11 @@ public class DocumentV1ApiHandler extends AbstractRequestHandler { } } - private static void handleFeedOperation(DocumentPath path, ResponseHandler handler, com.yahoo.documentapi.Response response) { - handle(path, null, handler, response, (document, jsonResponse) -> jsonResponse.commit(Response.Status.OK)); + private static void handleFeedOperation(DocumentPath path, + boolean fullyApplied, + ResponseHandler handler, + com.yahoo.documentapi.Response response) { + handle(path, null, handler, response, (document, jsonResponse) -> jsonResponse.commit(Response.Status.OK, fullyApplied)); } private void updatePutMetrics(Outcome outcome) { @@ -1188,7 +1197,7 @@ public class DocumentV1ApiHandler extends AbstractRequestHandler { private interface VisitCallback { /** Called at the start of response rendering. */ - default void onStart(JsonResponse response) throws IOException { } + default void onStart(JsonResponse response, boolean fullyApplied) throws IOException { } /** Called for every document received from backend visitors—must call the ack for these to proceed. */ default void onDocument(JsonResponse response, Document document, Runnable ack, Consumer<String> onError) { } @@ -1199,25 +1208,26 @@ public class DocumentV1ApiHandler extends AbstractRequestHandler { private void visitAndDelete(HttpRequest request, VisitorParameters parameters, ResponseHandler handler, TestAndSetCondition condition, String route) { - visitAndProcess(request, parameters, handler, route, (id, operationParameters) -> { + visitAndProcess(request, parameters, true, handler, route, (id, operationParameters) -> { DocumentRemove remove = new DocumentRemove(id); remove.setCondition(condition); return asyncSession.remove(remove, operationParameters); }); } - private void visitAndUpdate(HttpRequest request, VisitorParameters parameters, ResponseHandler handler, - DocumentUpdate protoUpdate, String route) { - visitAndProcess(request, parameters, handler, route, (id, operationParameters) -> { + private void visitAndUpdate(HttpRequest request, VisitorParameters parameters, boolean fullyApplied, + ResponseHandler handler, DocumentUpdate protoUpdate, String route) { + visitAndProcess(request, parameters, fullyApplied, handler, route, (id, operationParameters) -> { DocumentUpdate update = new DocumentUpdate(protoUpdate); update.setId(id); return asyncSession.update(update, operationParameters); }); } - private void visitAndProcess(HttpRequest request, VisitorParameters parameters, ResponseHandler handler, + private void visitAndProcess(HttpRequest request, VisitorParameters parameters, boolean fullyApplied, + ResponseHandler handler, String route, BiFunction<DocumentId, DocumentOperationParameters, Result> operation) { - visit(request, parameters, false, handler, new VisitCallback() { + visit(request, parameters, false, fullyApplied, handler, new VisitCallback() { @Override public void onDocument(JsonResponse response, Document document, Runnable ack, Consumer<String> onError) { DocumentOperationParameters operationParameters = parameters().withRoute(route) .withResponseHandler(operationResponse -> { @@ -1255,10 +1265,10 @@ public class DocumentV1ApiHandler extends AbstractRequestHandler { } private void visitAndWrite(HttpRequest request, VisitorParameters parameters, ResponseHandler handler, boolean streamed) { - visit(request, parameters, streamed, handler, new VisitCallback() { - @Override public void onStart(JsonResponse response) throws IOException { + visit(request, parameters, streamed, true, handler, new VisitCallback() { + @Override public void onStart(JsonResponse response, boolean fullyApplied) throws IOException { if (streamed) - response.commit(Response.Status.OK); + response.commit(Response.Status.OK, fullyApplied); response.writeDocumentsArrayStart(); } @@ -1288,16 +1298,16 @@ public class DocumentV1ApiHandler extends AbstractRequestHandler { } private void visitWithRemote(HttpRequest request, VisitorParameters parameters, ResponseHandler handler) { - visit(request, parameters, false, handler, new VisitCallback() { }); + visit(request, parameters, false, true, handler, new VisitCallback() { }); } @SuppressWarnings("fallthrough") - private void visit(HttpRequest request, VisitorParameters parameters, boolean streaming, ResponseHandler handler, VisitCallback callback) { + private void visit(HttpRequest request, VisitorParameters parameters, boolean streaming, boolean fullyApplied, ResponseHandler handler, VisitCallback callback) { try { JsonResponse response = JsonResponse.create(request, handler); Phaser phaser = new Phaser(2); // Synchronize this thread (dispatch) with the visitor callback thread. AtomicReference<String> error = new AtomicReference<>(); // Set if error occurs during processing of visited documents. - callback.onStart(response); + callback.onStart(response, fullyApplied); VisitorControlHandler controller = new VisitorControlHandler() { final ScheduledFuture<?> abort = streaming ? visitDispatcher.schedule(this::abort, request.getTimeout(MILLISECONDS), MILLISECONDS) : null; @Override public void onDone(CompletionCode code, String message) { @@ -1332,7 +1342,7 @@ public class DocumentV1ApiHandler extends AbstractRequestHandler { response.writeMessage(error.get() != null ? error.get() : message != null ? message : "Visiting failed"); } if ( ! streaming) - response.commit(status); + response.commit(status, fullyApplied); } }); if (abort != null) abort.cancel(false); // Avoid keeping scheduled future alive if this completes in any other fashion. diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/ClientFeederV3.java b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/ClientFeederV3.java index 8ea9234009d..438248f31a7 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/ClientFeederV3.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/ClientFeederV3.java @@ -4,7 +4,6 @@ package com.yahoo.vespa.http.server; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.document.DocumentTypeManager; -import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol; import com.yahoo.jdisc.Metric; import com.yahoo.jdisc.ReferencedResource; import com.yahoo.jdisc.ResourceReference; @@ -53,13 +52,12 @@ class ClientFeederV3 { private final AtomicInteger ongoingRequests = new AtomicInteger(0); private final String hostName; - ClientFeederV3( - ReferencedResource<SharedSourceSession> sourceSession, - FeedReaderFactory feedReaderFactory, - DocumentTypeManager docTypeManager, - String clientId, - Metric metric, - ReplyHandler feedReplyHandler) { + ClientFeederV3(ReferencedResource<SharedSourceSession> sourceSession, + FeedReaderFactory feedReaderFactory, + DocumentTypeManager docTypeManager, + String clientId, + Metric metric, + ReplyHandler feedReplyHandler) { this.sourceSession = sourceSession; this.clientId = clientId; this.feedReplyHandler = feedReplyHandler; @@ -220,10 +218,7 @@ class ClientFeederV3 { // This is a bit hard to set up while testing, so we accept that things are not perfect. if (sourceSession.getResource().session() != null) { - metric.set( - MetricNames.PENDING, - Double.valueOf(sourceSession.getResource().session().getPendingCount()), - null); + metric.set(MetricNames.PENDING, (double) sourceSession.getResource().session().getPendingCount(), null); } DocumentOperationMessageV3 message = DocumentOperationMessageV3.create(operation, operationId, metric); @@ -231,7 +226,7 @@ class ClientFeederV3 { // typical end of feed return null; } - metric.add(MetricNames.NUM_OPERATIONS, 1, null /*metricContext*/); + metric.add(MetricNames.NUM_OPERATIONS, 1, null); log(Level.FINE, "Successfully deserialized document id: ", message.getOperationId()); return message; } @@ -241,14 +236,6 @@ class ClientFeederV3 { if (settings.traceLevel != null) { msg.getMessage().getTrace().setLevel(settings.traceLevel); } - if (settings.priority != null) { - try { - DocumentProtocol.Priority priority = DocumentProtocol.Priority.valueOf(settings.priority); - } - catch (IllegalArgumentException i) { - log.severe(i.getMessage()); - } - } } private void setRoute(DocumentOperationMessageV3 msg, FeederSettings settings) { @@ -272,7 +259,7 @@ class ClientFeederV3 { if (now.plusSeconds(1).isAfter(prevOpsPerSecTime)) { Duration duration = Duration.between(now, prevOpsPerSecTime); double opsPerSec = operationsForOpsPerSec / (duration.toMillis() / 1000.); - metric.set(MetricNames.OPERATIONS_PER_SEC, opsPerSec, null /*metricContext*/); + metric.set(MetricNames.OPERATIONS_PER_SEC, opsPerSec, null); operationsForOpsPerSec = 1.0d; prevOpsPerSecTime = now; } else { @@ -280,4 +267,5 @@ class ClientFeederV3 { } } } + } diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/DocumentOperationMessageV3.java b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/DocumentOperationMessageV3.java index 25bf5815907..a12fe4efa8b 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/DocumentOperationMessageV3.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/DocumentOperationMessageV3.java @@ -14,8 +14,6 @@ import com.yahoo.vespaxmlparser.FeedOperation; /** * Keeps an operation with its message. * - * This implementation is based on V2, but the code is restructured. - * * @author dybis */ class DocumentOperationMessageV3 { @@ -66,13 +64,13 @@ class DocumentOperationMessageV3 { static DocumentOperationMessageV3 create(FeedOperation operation, String operationId, Metric metric) { switch (operation.getType()) { case DOCUMENT: - metric.add(MetricNames.NUM_PUTS, 1, null /*metricContext*/); + metric.add(MetricNames.NUM_PUTS, 1, null); return newPutMessage(operation, operationId); case REMOVE: - metric.add(MetricNames.NUM_REMOVES, 1, null /*metricContext*/); + metric.add(MetricNames.NUM_REMOVES, 1, null); return newRemoveMessage(operation, operationId); case UPDATE: - metric.add(MetricNames.NUM_UPDATES, 1, null /*metricContext*/); + metric.add(MetricNames.NUM_UPDATES, 1, null); return newUpdateMessage(operation, operationId); default: // typical end of feed diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedHandler.java b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedHandler.java index 74665d60a04..3c1f376b4eb 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedHandler.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedHandler.java @@ -29,14 +29,14 @@ import java.util.regex.Pattern; import java.util.zip.GZIPInputStream; /** - * Accept feeds from outside of the Vespa cluster. + * Accept feeds from outside the Vespa cluster. * * @author Steinar Knutsen */ public class FeedHandler extends ThreadedHttpRequestHandler { protected final ReplyHandler feedReplyHandler; - private static final List<Integer> serverSupportedVersions = Collections.unmodifiableList(Arrays.asList(3)); + private static final List<Integer> serverSupportedVersions = List.of(3); private static final Pattern USER_AGENT_PATTERN = Pattern.compile("vespa-http-client \\((.+)\\)"); private final FeedHandlerV3 feedHandlerV3; private final DocumentApiMetrics metricsHelper; @@ -144,4 +144,5 @@ public class FeedHandler extends ThreadedHttpRequestHandler { } @Override protected void destroy() { feedHandlerV3.destroy(); } + } diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedHandlerV3.java b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedHandlerV3.java index f9ae04623e6..4de3eebec2d 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedHandlerV3.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedHandlerV3.java @@ -24,9 +24,8 @@ import java.util.logging.Level; import java.util.logging.Logger; /** - * This code is based on v2 code, however, in v3, one client has one ClientFeederV3 shared between all client threads. - * The new API has more logic for shutting down cleanly as the server is more likely to be upgraded. - * The code is restructured a bit. + * One client has one ClientFeederV3 shared between all client threads. + * Contains logic for shutting down cleanly as the server is upgraded. * * @author dybis */ @@ -60,7 +59,7 @@ public class FeedHandlerV3 extends ThreadedHttpRequestHandler { } // TODO: If this is set up to run without first invoking the old FeedHandler code, we should - // verify the version header first. This is done in the old code. + // verify the version header first. @Override public HttpResponse handle(HttpRequest request) { String clientId = clientId(request); @@ -70,7 +69,7 @@ public class FeedHandlerV3 extends ThreadedHttpRequestHandler { SourceSessionParams sourceSessionParams = sourceSessionParams(request); clientFeederByClientId.put(clientId, new ClientFeederV3(retainSource(sessionCache, sourceSessionParams), - new FeedReaderFactory(true), //TODO make error debugging configurable + new FeedReaderFactory(true), // TODO: Make error debugging configurable docTypeManager, clientId, metric, diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeederSettings.java b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeederSettings.java index 9bb8a58d6f6..a8175a48a39 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeederSettings.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeederSettings.java @@ -17,14 +17,12 @@ public class FeederSettings { public final boolean drain; // TODO: Implement drain=true public final Route route; public final FeedParams.DataFormat dataFormat; - public final String priority; public final Integer traceLevel; public FeederSettings(HttpRequest request) { this.drain = Optional.ofNullable(request.getHeader(Headers.DRAIN)).map(Boolean::parseBoolean).orElse(false); this.route = Optional.ofNullable(request.getHeader(Headers.ROUTE)).map(Route::parse).orElse(DEFAULT_ROUTE); this.dataFormat = Optional.ofNullable(request.getHeader(Headers.DATA_FORMAT)).map(FeedParams.DataFormat::valueOf).orElse(FeedParams.DataFormat.JSON_UTF8); - this.priority = request.getHeader(Headers.PRIORITY); this.traceLevel = Optional.ofNullable(request.getHeader(Headers.TRACE_LEVEL)).map(Integer::valueOf).orElse(null); } diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/Headers.java b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/Headers.java index 16bff38af4b..657c22ba7ee 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/Headers.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/Headers.java @@ -6,7 +6,7 @@ package com.yahoo.vespa.http.server; * * @author Steinar Knutsen */ -final class Headers { +public final class Headers { private Headers() { } @@ -23,7 +23,6 @@ final class Headers { // This value can be used to route the request to a specific server when using // several servers. It is a random value that is the same for the whole session. public static final String SHARDING_KEY = "X-Yahoo-Feed-Sharding-Key"; - public static final String PRIORITY = "X-Yahoo-Feed-Priority"; public static final String TRACE_LEVEL = "X-Yahoo-Feed-Trace-Level"; public static final int HTTP_NOT_ACCEPTABLE = 406; @@ -34,4 +33,8 @@ final class Headers { public static final String HOSTNAME = "X-Yahoo-Hostname"; public static final String SILENTUPGRADE = "X-Yahoo-Silent-Upgrade"; + // A response header present and set to "true" onlynif any fields of a document operation were ignored + // because they were not declared in the target document type. + public static final String IGNORED_FIELDS = "X-Vespa-Ignored-Fields"; + } diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/StreamReaderV3.java b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/StreamReaderV3.java index c2c6d00fa25..3d82919d503 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/StreamReaderV3.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/StreamReaderV3.java @@ -10,17 +10,15 @@ import com.yahoo.vespaxmlparser.FeedReader; import java.io.IOException; import java.io.InputStream; import java.util.Optional; -import java.util.logging.Logger; import java.util.zip.GZIPInputStream; /** * This code is based on v2 code, but restructured so stream reading code is in one dedicated class. + * * @author dybis */ public class StreamReaderV3 { - protected static final Logger log = Logger.getLogger(StreamReaderV3.class.getName()); - private final FeedReaderFactory feedReaderFactory; private final DocumentTypeManager docTypeManager; @@ -30,15 +28,11 @@ public class StreamReaderV3 { } public FeedOperation getNextOperation(InputStream requestInputStream, FeederSettings settings) throws Exception { - FeedOperation op = null; - int length = readByteLength(requestInputStream); - try (InputStream limitedInputStream = new ByteLimitedInputStream(requestInputStream, length)){ FeedReader reader = feedReaderFactory.createReader(limitedInputStream, docTypeManager, settings.dataFormat); - op = reader.read(); + return reader.read(); } - return op; } public Optional<String> getNextOperationId(InputStream requestInputStream) throws IOException { @@ -48,7 +42,7 @@ public class StreamReaderV3 { if (c == 32) { break; } - idBuf.append((char) c); //it's ASCII + idBuf.append((char) c); // it's ASCII } if (idBuf.length() == 0) { return Optional.empty(); @@ -63,7 +57,7 @@ public class StreamReaderV3 { if (c == 10) { break; } - lenBuf.append((char) c); //it's ASCII + lenBuf.append((char) c); // it's ASCII } if (lenBuf.length() == 0) { throw new IllegalStateException("Operation length missing."); @@ -71,9 +65,8 @@ public class StreamReaderV3 { return Integer.valueOf(lenBuf.toString(), 16); } - public static InputStream unzipStreamIfNeeded(final HttpRequest httpRequest) - throws IOException { - final String contentEncodingHeader = httpRequest.getHeader("content-encoding"); + public static InputStream unzipStreamIfNeeded(final HttpRequest httpRequest) throws IOException { + String contentEncodingHeader = httpRequest.getHeader("content-encoding"); if ("gzip".equals(contentEncodingHeader)) { return new GZIPInputStream(httpRequest.getData()); } else { diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/package-info.java b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/package-info.java index ea01137d9af..3678a0b9fac 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/package-info.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/package-info.java @@ -1,6 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. /** - * Server side of programmatic API for feeding into Vespa from outside of the + * Server side of programmatic API for feeding into Vespa from outside the * clusters. Not a public API, not meant for direct use. */ @com.yahoo.api.annotations.PackageMarker diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/resource/DocumentV1ApiTest.java b/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/resource/DocumentV1ApiTest.java index 7f77ce9d0d5..cd57818e74e 100644 --- a/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/resource/DocumentV1ApiTest.java +++ b/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/resource/DocumentV1ApiTest.java @@ -113,7 +113,8 @@ public class DocumentV1ApiTest { .maxThrottled(2) .resendDelayMillis(1 << 30) .build(); - final DocumentmanagerConfig docConfig = Deriver.getDocumentManagerConfig("src/test/cfg/music.sd").build(); + final DocumentmanagerConfig docConfig = Deriver.getDocumentManagerConfig("src/test/cfg/music.sd") + .ignoreundefinedfields(true).build(); final DocumentTypeManager manager = new DocumentTypeManager(docConfig); final Document doc1 = new Document(manager.getDocumentType("music"), "id:space:music::one"); final Document doc2 = new Document(manager.getDocumentType("music"), "id:space:music:n=1:two"); @@ -330,6 +331,7 @@ public class DocumentV1ApiTest { " \"message\": \"failure?\"" + "}", response.readAll()); assertEquals(200, response.getStatus()); + assertNull(response.getResponse().headers().get("X-Vespa-Ignored-Fields")); // POST with namespace and document type is a restricted visit with a required destination cluster ("destinationCluster") access.expect(parameters -> { @@ -376,13 +378,15 @@ public class DocumentV1ApiTest { response = driver.sendRequest("http://localhost/document/v1/space/music/docid?selection=true&cluster=content&timeChunk=10", PUT, "{" + " \"fields\": {" + - " \"artist\": { \"assign\": \"Lisa Ekdahl\" }" + + " \"artist\": { \"assign\": \"Lisa Ekdahl\" }, " + + " \"nonexisting\": { \"assign\": \"Ignored\" }" + " }" + "}"); assertSameJson("{" + " \"pathId\": \"/document/v1/space/music/docid\"" + "}", response.readAll()); assertEquals(200, response.getStatus()); + assertEquals("true", response.getResponse().headers().get("X-Vespa-Ignored-Fields").get(0).toString()); // PUT with namespace, document type and group is also a restricted visit which requires a cluster. access.expect(parameters -> { diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/FeedHandlerV3Test.java b/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/FeedHandlerV3Test.java index 5b8b5b1827f..dbbe664c9f8 100644 --- a/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/FeedHandlerV3Test.java +++ b/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/FeedHandlerV3Test.java @@ -33,6 +33,7 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; public class FeedHandlerV3Test { + final CollectingMetric metric = new CollectingMetric(); private final Executor simpleThreadpool = Executors.newCachedThreadPool(); @@ -101,7 +102,6 @@ public class FeedHandlerV3Test { request.getJDiscRequest().headers().add(Headers.DATA_FORMAT, FeedParams.DataFormat.JSON_UTF8.name()); request.getJDiscRequest().headers().add(Headers.TIMEOUT, "1000000000"); request.getJDiscRequest().headers().add(Headers.CLIENT_ID, "client123"); - request.getJDiscRequest().headers().add(Headers.PRIORITY, "LOWEST"); request.getJDiscRequest().headers().add(Headers.TRACE_LEVEL, "4"); request.getJDiscRequest().headers().add(Headers.DRAIN, "true"); return request; diff --git a/vespaclient-core/src/main/java/com/yahoo/feedapi/Feeder.java b/vespaclient-core/src/main/java/com/yahoo/feedapi/Feeder.java index cd888b11d64..d8b69bd4a85 100644 --- a/vespaclient-core/src/main/java/com/yahoo/feedapi/Feeder.java +++ b/vespaclient-core/src/main/java/com/yahoo/feedapi/Feeder.java @@ -52,9 +52,8 @@ public abstract class Feeder { StringWriter sw = new StringWriter(); PrintWriter pw = new PrintWriter(sw); e.printStackTrace(pw); - message = "(no message) " + sw.toString(); + message = "(no message) " + sw; } - addError("ERROR: " + message); } @@ -84,17 +83,9 @@ public abstract class Feeder { if (createIfNonExistent && op.getDocumentUpdate() != null) { op.getDocumentUpdate().setCreateIfNonExistent(true); } - - // Done feeding. - if (op.getType() == FeedOperation.Type.INVALID) { - break; - } else { - sender.sendOperation(op); - } - } catch (XMLStreamException e) { - addException(e); - break; - } catch (NullPointerException e) { + if (op.getType() == FeedOperation.Type.INVALID) break; // Done feeding + sender.sendOperation(op); + } catch (XMLStreamException | NullPointerException e) { addException(e); break; } catch (Exception e) { diff --git a/vespaclient-java/src/main/java/com/yahoo/vespavisit/VdsVisit.java b/vespaclient-java/src/main/java/com/yahoo/vespavisit/VdsVisit.java index 42c23c4a961..9a5df96b705 100644 --- a/vespaclient-java/src/main/java/com/yahoo/vespavisit/VdsVisit.java +++ b/vespaclient-java/src/main/java/com/yahoo/vespavisit/VdsVisit.java @@ -25,6 +25,7 @@ import org.apache.commons.cli.Options; import java.io.*; import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; import java.util.Map; import java.util.stream.Collectors; @@ -649,7 +650,7 @@ public class VdsVisit { out.println("Adding the following library specific parameters:"); for (Map.Entry<String, byte[]> entry : params.getLibraryParameters().entrySet()) { out.println(" " + entry.getKey() + " = " + - new String(entry.getValue(), Charset.forName("utf-8"))); + new String(entry.getValue(), StandardCharsets.UTF_8)); } } if (params.getPriority() != DocumentProtocol.Priority.NORMAL_3) { diff --git a/vespajlib/src/main/java/com/yahoo/tensor/functions/Join.java b/vespajlib/src/main/java/com/yahoo/tensor/functions/Join.java index 4ec5b196dbc..1009177761b 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/functions/Join.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/functions/Join.java @@ -304,7 +304,7 @@ public class Join<NAMETYPE extends Name> extends PrimitiveTensorFunction<NAMETYP for (Iterator<Tensor.Cell> bIterator = b.cellIterator(); bIterator.hasNext(); ) { Map.Entry<TensorAddress, Double> bCell = bIterator.next(); TensorAddress combinedAddress = joinAddresses(aCell.getKey(), aToIndexes, - bCell.getKey(), bToIndexes, joinedType); + bCell.getKey(), bToIndexes, joinedType); if (combinedAddress == null) continue; // not combinable builder.cell(combinedAddress, combinator.applyAsDouble(aCell.getValue(), bCell.getValue())); } @@ -347,7 +347,7 @@ public class Join<NAMETYPE extends Name> extends PrimitiveTensorFunction<NAMETYP TensorAddress partialCommonAddress = partialCommonAddress(bCell, bIndexesInCommon); for (Tensor.Cell aCell : aCellsByCommonAddress.getOrDefault(partialCommonAddress, Collections.emptyList())) { TensorAddress combinedAddress = joinAddresses(aCell.getKey(), aIndexesInJoined, - bCell.getKey(), bIndexesInJoined, joinedType); + bCell.getKey(), bIndexesInJoined, joinedType); if (combinedAddress == null) continue; // not combinable double combinedValue = swapTensors ? combinator.applyAsDouble(bCell.getValue(), aCell.getValue()) : |