diff options
65 files changed, 261 insertions, 141 deletions
diff --git a/config-model/src/main/java/com/yahoo/schema/processing/TextMatch.java b/config-model/src/main/java/com/yahoo/schema/processing/TextMatch.java index e29f683761f..1ff019038fc 100644 --- a/config-model/src/main/java/com/yahoo/schema/processing/TextMatch.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/TextMatch.java @@ -64,7 +64,7 @@ public class TextMatch extends Processor { if (fieldMatching != null) { var maxLength = fieldMatching.maxLength(); if (maxLength != null) { - ret.setMaxTokenLength(maxLength); + ret.setMaxTokenizeLength(maxLength); } var maxTermOccurrences = fieldMatching.maxTermOccurrences(); if (maxTermOccurrences != null) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ConstantTensorJsonValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ConstantTensorJsonValidator.java index 02a6b243054..40c9a03b126 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ConstantTensorJsonValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ConstantTensorJsonValidator.java @@ -132,7 +132,7 @@ public class ConstantTensorJsonValidator { private void consumeTopObject() throws IOException { for (var cur = parser.nextToken(); cur != JsonToken.END_OBJECT; cur = parser.nextToken()) { assertCurrentTokenIs(JsonToken.FIELD_NAME); - String fieldName = parser.currentName(); + String fieldName = parser.getCurrentName(); switch (fieldName) { case FIELD_TYPE -> consumeTypeField(); case FIELD_VALUES -> consumeValuesField(); @@ -189,7 +189,7 @@ public class ConstantTensorJsonValidator { } for (var cur = parser.nextToken(); cur != JsonToken.END_OBJECT; cur = parser.nextToken()) { assertCurrentTokenIs(JsonToken.FIELD_NAME); - validateNumeric(parser.currentName(), parser.nextToken()); + validateNumeric(parser.getCurrentName(), parser.nextToken()); } } @@ -199,7 +199,7 @@ public class ConstantTensorJsonValidator { boolean seenValue = false; for (int i = 0; i < 2; i++) { assertNextTokenIs(JsonToken.FIELD_NAME); - String fieldName = parser.currentName(); + String fieldName = parser.getCurrentName(); switch (fieldName) { case FIELD_ADDRESS -> { validateTensorAddress(new HashSet<>(tensorDimensions.keySet())); @@ -228,13 +228,13 @@ public class ConstantTensorJsonValidator { // Iterate within the address key, value pairs while ((parser.nextToken() != JsonToken.END_OBJECT)) { assertCurrentTokenIs(JsonToken.FIELD_NAME); - String dimensionName = parser.currentName(); + String dimensionName = parser.getCurrentName(); TensorType.Dimension dimension = tensorDimensions.get(dimensionName); if (dimension == null) { - throw new InvalidConstantTensorException(parser, String.format("Tensor dimension '%s' does not exist", dimensionName)); + throw new InvalidConstantTensorException(parser, String.format("Tensor dimension '%s' does not exist", parser.getCurrentName())); } if (!cellDimensions.contains(dimensionName)) { - throw new InvalidConstantTensorException(parser, String.format("Duplicate tensor dimension '%s'", dimensionName)); + throw new InvalidConstantTensorException(parser, String.format("Duplicate tensor dimension '%s'", parser.getCurrentName())); } cellDimensions.remove(dimensionName); validateLabel(dimension); @@ -300,7 +300,7 @@ public class ConstantTensorJsonValidator { } private void assertCurrentTokenIs(JsonToken wantedToken) { - assertTokenIs(parser.currentToken(), wantedToken); + assertTokenIs(parser.getCurrentToken(), wantedToken); } private void assertNextTokenIs(JsonToken wantedToken) throws IOException { @@ -316,11 +316,11 @@ public class ConstantTensorJsonValidator { static class InvalidConstantTensorException extends IllegalArgumentException { InvalidConstantTensorException(JsonParser parser, String message) { - super(message + " " + parser.currentLocation().toString()); + super(message + " " + parser.getCurrentLocation().toString()); } InvalidConstantTensorException(JsonParser parser, Exception base) { - super("Failed to parse JSON stream " + parser.currentLocation().toString(), base); + super("Failed to parse JSON stream " + parser.getCurrentLocation().toString(), base); } InvalidConstantTensorException(IOException base) { @@ -412,7 +412,7 @@ public class ConstantTensorJsonValidator { boolean seenValues = false; for (int i = 0; i < 2; i++) { assertNextTokenIs(JsonToken.FIELD_NAME); - String fieldName = parser.currentName(); + String fieldName = parser.getCurrentName(); switch (fieldName) { case FIELD_ADDRESS -> { validateTensorAddress(new HashSet<>(mappedDims)); diff --git a/config-model/src/test/derived/advanced/ilscripts.cfg b/config-model/src/test/derived/advanced/ilscripts.cfg index 51a49502b64..d633cd97f0c 100644 --- a/config-model/src/test/derived/advanced/ilscripts.cfg +++ b/config-model/src/test/derived/advanced/ilscripts.cfg @@ -1,4 +1,5 @@ maxtermoccurrences 10000 +maxtokenlength 1000 fieldmatchmaxlength 1000000 ilscript[].doctype "advanced" ilscript[].docfield[] "debug_src" diff --git a/config-model/src/test/derived/annotationsimplicitstruct/ilscripts.cfg b/config-model/src/test/derived/annotationsimplicitstruct/ilscripts.cfg index 767c3af3c19..53dc789fbb7 100644 --- a/config-model/src/test/derived/annotationsimplicitstruct/ilscripts.cfg +++ b/config-model/src/test/derived/annotationsimplicitstruct/ilscripts.cfg @@ -1,3 +1,4 @@ maxtermoccurrences 10000 +maxtokenlength 1000 fieldmatchmaxlength 1000000 ilscript[].doctype "annotationsimplicitstruct" diff --git a/config-model/src/test/derived/annotationsinheritance/ilscripts.cfg b/config-model/src/test/derived/annotationsinheritance/ilscripts.cfg index d8e6c882b80..b0a69c5408a 100644 --- a/config-model/src/test/derived/annotationsinheritance/ilscripts.cfg +++ b/config-model/src/test/derived/annotationsinheritance/ilscripts.cfg @@ -1,3 +1,4 @@ maxtermoccurrences 10000 +maxtokenlength 1000 fieldmatchmaxlength 1000000 ilscript[].doctype "annotationsinheritance" diff --git a/config-model/src/test/derived/annotationsinheritance2/ilscripts.cfg b/config-model/src/test/derived/annotationsinheritance2/ilscripts.cfg index ae4ea621583..5ec1f839429 100644 --- a/config-model/src/test/derived/annotationsinheritance2/ilscripts.cfg +++ b/config-model/src/test/derived/annotationsinheritance2/ilscripts.cfg @@ -1,3 +1,4 @@ maxtermoccurrences 10000 +maxtokenlength 1000 fieldmatchmaxlength 1000000 ilscript[].doctype "annotationsinheritance2" diff --git a/config-model/src/test/derived/annotationsreference/ilscripts.cfg b/config-model/src/test/derived/annotationsreference/ilscripts.cfg index 812f5e44545..eaa20043be8 100644 --- a/config-model/src/test/derived/annotationsreference/ilscripts.cfg +++ b/config-model/src/test/derived/annotationsreference/ilscripts.cfg @@ -1,3 +1,4 @@ maxtermoccurrences 10000 +maxtokenlength 1000 fieldmatchmaxlength 1000000 ilscript[].doctype "annotationsreference" diff --git a/config-model/src/test/derived/annotationssimple/ilscripts.cfg b/config-model/src/test/derived/annotationssimple/ilscripts.cfg index 9d0962df5be..af179221eb4 100644 --- a/config-model/src/test/derived/annotationssimple/ilscripts.cfg +++ b/config-model/src/test/derived/annotationssimple/ilscripts.cfg @@ -1,3 +1,4 @@ maxtermoccurrences 10000 +maxtokenlength 1000 fieldmatchmaxlength 1000000 ilscript[].doctype "annotationssimple" diff --git a/config-model/src/test/derived/arrays/ilscripts.cfg b/config-model/src/test/derived/arrays/ilscripts.cfg index 98cff642d9e..3f2dae48552 100644 --- a/config-model/src/test/derived/arrays/ilscripts.cfg +++ b/config-model/src/test/derived/arrays/ilscripts.cfg @@ -1,4 +1,5 @@ maxtermoccurrences 10000 +maxtokenlength 1000 fieldmatchmaxlength 1000000 ilscript[].doctype "arrays" ilscript[].docfield[] "tags" diff --git a/config-model/src/test/derived/attributeprefetch/ilscripts.cfg b/config-model/src/test/derived/attributeprefetch/ilscripts.cfg index dec054b33f0..5a3784f7cb9 100644 --- a/config-model/src/test/derived/attributeprefetch/ilscripts.cfg +++ b/config-model/src/test/derived/attributeprefetch/ilscripts.cfg @@ -1,4 +1,5 @@ maxtermoccurrences 10000 +maxtokenlength 1000 fieldmatchmaxlength 1000000 ilscript[].doctype "prefetch" ilscript[].docfield[] "singlebyte" diff --git a/config-model/src/test/derived/attributes/ilscripts.cfg b/config-model/src/test/derived/attributes/ilscripts.cfg index 6d3ef2799d9..58279759e5f 100644 --- a/config-model/src/test/derived/attributes/ilscripts.cfg +++ b/config-model/src/test/derived/attributes/ilscripts.cfg @@ -1,4 +1,5 @@ maxtermoccurrences 10000 +maxtokenlength 1000 fieldmatchmaxlength 1000000 ilscript[].doctype "attributes" ilscript[].docfield[] "a1" diff --git a/config-model/src/test/derived/bolding_dynamic_summary/ilscripts.cfg b/config-model/src/test/derived/bolding_dynamic_summary/ilscripts.cfg index c20c321ebcf..0b925da4778 100644 --- a/config-model/src/test/derived/bolding_dynamic_summary/ilscripts.cfg +++ b/config-model/src/test/derived/bolding_dynamic_summary/ilscripts.cfg @@ -1,4 +1,5 @@ maxtermoccurrences 10000 +maxtokenlength 1000 fieldmatchmaxlength 1000000 ilscript[].doctype "test" ilscript[].docfield[] "str_1" diff --git a/config-model/src/test/derived/complex/ilscripts.cfg b/config-model/src/test/derived/complex/ilscripts.cfg index 4405d2fda40..7d025e15703 100644 --- a/config-model/src/test/derived/complex/ilscripts.cfg +++ b/config-model/src/test/derived/complex/ilscripts.cfg @@ -1,4 +1,5 @@ maxtermoccurrences 10000 +maxtokenlength 1000 fieldmatchmaxlength 1000000 ilscript[].doctype "complex" ilscript[].docfield[] "title" diff --git a/config-model/src/test/derived/emptydefault/ilscripts.cfg b/config-model/src/test/derived/emptydefault/ilscripts.cfg index e4242153bce..bbb8e5c556c 100644 --- a/config-model/src/test/derived/emptydefault/ilscripts.cfg +++ b/config-model/src/test/derived/emptydefault/ilscripts.cfg @@ -1,4 +1,5 @@ maxtermoccurrences 10000 +maxtokenlength 1000 fieldmatchmaxlength 1000000 ilscript[].doctype "emptydefault" ilscript[].docfield[] "one" diff --git a/config-model/src/test/derived/exactmatch/ilscripts.cfg b/config-model/src/test/derived/exactmatch/ilscripts.cfg index 21dfbd1371b..1d1bd6d5e8a 100644 --- a/config-model/src/test/derived/exactmatch/ilscripts.cfg +++ b/config-model/src/test/derived/exactmatch/ilscripts.cfg @@ -1,4 +1,5 @@ maxtermoccurrences 10000 +maxtokenlength 1000 fieldmatchmaxlength 1000000 ilscript[].doctype "exactmatch" ilscript[].docfield[] "tag" diff --git a/config-model/src/test/derived/hnsw_index/ilscripts.cfg b/config-model/src/test/derived/hnsw_index/ilscripts.cfg index e48f116f468..c811b93c3df 100644 --- a/config-model/src/test/derived/hnsw_index/ilscripts.cfg +++ b/config-model/src/test/derived/hnsw_index/ilscripts.cfg @@ -1,4 +1,5 @@ maxtermoccurrences 10000 +maxtokenlength 1000 fieldmatchmaxlength 1000000 ilscript[].doctype "test" ilscript[].docfield[] "t1" diff --git a/config-model/src/test/derived/id/ilscripts.cfg b/config-model/src/test/derived/id/ilscripts.cfg index d3ab29f6cd8..121e305059e 100644 --- a/config-model/src/test/derived/id/ilscripts.cfg +++ b/config-model/src/test/derived/id/ilscripts.cfg @@ -1,4 +1,5 @@ maxtermoccurrences 10000 +maxtokenlength 1000 fieldmatchmaxlength 1000000 ilscript[].doctype "id" ilscript[].docfield[] "uri" diff --git a/config-model/src/test/derived/indexswitches/ilscripts.cfg b/config-model/src/test/derived/indexswitches/ilscripts.cfg index 472c1f95cb0..454f675c0a2 100644 --- a/config-model/src/test/derived/indexswitches/ilscripts.cfg +++ b/config-model/src/test/derived/indexswitches/ilscripts.cfg @@ -1,4 +1,5 @@ maxtermoccurrences 10000 +maxtokenlength 1000 fieldmatchmaxlength 1000000 ilscript[].doctype "indexswitches" ilscript[].docfield[] "title" diff --git a/config-model/src/test/derived/inheritance/ilscripts.cfg b/config-model/src/test/derived/inheritance/ilscripts.cfg index d4c804773f0..c966f32a502 100644 --- a/config-model/src/test/derived/inheritance/ilscripts.cfg +++ b/config-model/src/test/derived/inheritance/ilscripts.cfg @@ -1,4 +1,5 @@ maxtermoccurrences 10000 +maxtokenlength 1000 fieldmatchmaxlength 1000000 ilscript[].doctype "child" ilscript[].docfield[] "onlygrandparent" diff --git a/config-model/src/test/derived/language/ilscripts.cfg b/config-model/src/test/derived/language/ilscripts.cfg index 1860f180839..d0abc08f1e0 100644 --- a/config-model/src/test/derived/language/ilscripts.cfg +++ b/config-model/src/test/derived/language/ilscripts.cfg @@ -1,4 +1,5 @@ maxtermoccurrences 10000 +maxtokenlength 1000 fieldmatchmaxlength 1000000 ilscript[].doctype "language" ilscript[].docfield[] "language" diff --git a/config-model/src/test/derived/lowercase/ilscripts.cfg b/config-model/src/test/derived/lowercase/ilscripts.cfg index 8ba4bfa3349..49515e50df4 100644 --- a/config-model/src/test/derived/lowercase/ilscripts.cfg +++ b/config-model/src/test/derived/lowercase/ilscripts.cfg @@ -1,4 +1,5 @@ maxtermoccurrences 10000 +maxtokenlength 1000 fieldmatchmaxlength 1000000 ilscript[].doctype "lowercase" ilscript[].docfield[] "single_field_source" diff --git a/config-model/src/test/derived/multiplesummaries/ilscripts.cfg b/config-model/src/test/derived/multiplesummaries/ilscripts.cfg index 0cdf921de25..4a6de4154f8 100644 --- a/config-model/src/test/derived/multiplesummaries/ilscripts.cfg +++ b/config-model/src/test/derived/multiplesummaries/ilscripts.cfg @@ -1,4 +1,5 @@ maxtermoccurrences 10000 +maxtokenlength 1000 fieldmatchmaxlength 1000000 ilscript[].doctype "multiplesummaries" ilscript[].docfield[] "a" diff --git a/config-model/src/test/derived/music/ilscripts.cfg b/config-model/src/test/derived/music/ilscripts.cfg index f90cdb15baa..f79e8824b69 100644 --- a/config-model/src/test/derived/music/ilscripts.cfg +++ b/config-model/src/test/derived/music/ilscripts.cfg @@ -1,4 +1,5 @@ maxtermoccurrences 10000 +maxtokenlength 1000 fieldmatchmaxlength 1000000 ilscript[].doctype "music" ilscript[].docfield[] "bgndata" diff --git a/config-model/src/test/derived/newrank/ilscripts.cfg b/config-model/src/test/derived/newrank/ilscripts.cfg index b02e09a0496..487d2fca902 100644 --- a/config-model/src/test/derived/newrank/ilscripts.cfg +++ b/config-model/src/test/derived/newrank/ilscripts.cfg @@ -1,4 +1,5 @@ maxtermoccurrences 10000 +maxtokenlength 1000 fieldmatchmaxlength 1000000 ilscript[].doctype "newrank" ilscript[].docfield[] "bgndata" diff --git a/config-model/src/test/derived/orderilscripts/ilscripts.cfg b/config-model/src/test/derived/orderilscripts/ilscripts.cfg index 0ed1589af0a..4918e23efc6 100644 --- a/config-model/src/test/derived/orderilscripts/ilscripts.cfg +++ b/config-model/src/test/derived/orderilscripts/ilscripts.cfg @@ -1,4 +1,5 @@ maxtermoccurrences 10000 +maxtokenlength 1000 fieldmatchmaxlength 1000000 ilscript[].doctype "orderilscripts" ilscript[].docfield[] "foo" diff --git a/config-model/src/test/derived/position_array/ilscripts.cfg b/config-model/src/test/derived/position_array/ilscripts.cfg index ecafbc4a025..3f7611b25d8 100644 --- a/config-model/src/test/derived/position_array/ilscripts.cfg +++ b/config-model/src/test/derived/position_array/ilscripts.cfg @@ -1,4 +1,5 @@ maxtermoccurrences 10000 +maxtokenlength 1000 fieldmatchmaxlength 1000000 ilscript[].doctype "position_array" ilscript[].docfield[] "pos" diff --git a/config-model/src/test/derived/position_attribute/ilscripts.cfg b/config-model/src/test/derived/position_attribute/ilscripts.cfg index d2fc8503ce5..fbd1a293418 100644 --- a/config-model/src/test/derived/position_attribute/ilscripts.cfg +++ b/config-model/src/test/derived/position_attribute/ilscripts.cfg @@ -1,4 +1,5 @@ maxtermoccurrences 10000 +maxtokenlength 1000 fieldmatchmaxlength 1000000 ilscript[].doctype "position_attribute" ilscript[].docfield[] "pos" diff --git a/config-model/src/test/derived/position_extra/ilscripts.cfg b/config-model/src/test/derived/position_extra/ilscripts.cfg index a86dcec92ec..4645798723c 100644 --- a/config-model/src/test/derived/position_extra/ilscripts.cfg +++ b/config-model/src/test/derived/position_extra/ilscripts.cfg @@ -1,4 +1,5 @@ maxtermoccurrences 10000 +maxtokenlength 1000 fieldmatchmaxlength 1000000 ilscript[].doctype "position_extra" ilscript[].docfield[] "pos_str" diff --git a/config-model/src/test/derived/prefixexactattribute/ilscripts.cfg b/config-model/src/test/derived/prefixexactattribute/ilscripts.cfg index 40c7843a0a4..2d1904cf9d8 100644 --- a/config-model/src/test/derived/prefixexactattribute/ilscripts.cfg +++ b/config-model/src/test/derived/prefixexactattribute/ilscripts.cfg @@ -1,4 +1,5 @@ maxtermoccurrences 10000 +maxtokenlength 1000 fieldmatchmaxlength 1000000 ilscript[].doctype "prefixexactattribute" ilscript[].docfield[] "indexfield0" diff --git a/config-model/src/test/derived/ranktypes/ilscripts.cfg b/config-model/src/test/derived/ranktypes/ilscripts.cfg index adcd2f70c70..22526d1aa23 100644 --- a/config-model/src/test/derived/ranktypes/ilscripts.cfg +++ b/config-model/src/test/derived/ranktypes/ilscripts.cfg @@ -1,4 +1,5 @@ maxtermoccurrences 10000 +maxtokenlength 1000 fieldmatchmaxlength 1000000 ilscript[].doctype "ranktypes" ilscript[].docfield[] "title" diff --git a/config-model/src/test/derived/schemainheritance/ilscripts.cfg b/config-model/src/test/derived/schemainheritance/ilscripts.cfg index f7324920fe7..b1ba947f1dc 100644 --- a/config-model/src/test/derived/schemainheritance/ilscripts.cfg +++ b/config-model/src/test/derived/schemainheritance/ilscripts.cfg @@ -1,4 +1,5 @@ maxtermoccurrences 10000 +maxtokenlength 1000 fieldmatchmaxlength 1000000 ilscript[].doctype "child" ilscript[].docfield[] "pf1" diff --git a/config-model/src/test/derived/structanyorder/ilscripts.cfg b/config-model/src/test/derived/structanyorder/ilscripts.cfg index c07f04b3021..a806bc1b712 100644 --- a/config-model/src/test/derived/structanyorder/ilscripts.cfg +++ b/config-model/src/test/derived/structanyorder/ilscripts.cfg @@ -1,4 +1,5 @@ maxtermoccurrences 10000 +maxtokenlength 1000 fieldmatchmaxlength 1000000 ilscript[].doctype "annotationsimplicitstruct" ilscript[].docfield[] "structfield" diff --git a/config-model/src/test/derived/tokenization/ilscripts.cfg b/config-model/src/test/derived/tokenization/ilscripts.cfg index c08b6a54c83..cad8ec81879 100644 --- a/config-model/src/test/derived/tokenization/ilscripts.cfg +++ b/config-model/src/test/derived/tokenization/ilscripts.cfg @@ -1,4 +1,5 @@ maxtermoccurrences 10000 +maxtokenlength 1000 fieldmatchmaxlength 1000000 ilscript[].doctype "tokenization" ilscript[].docfield[] "text" diff --git a/config-model/src/test/derived/types/ilscripts.cfg b/config-model/src/test/derived/types/ilscripts.cfg index 17bed90deb4..73befb221ce 100644 --- a/config-model/src/test/derived/types/ilscripts.cfg +++ b/config-model/src/test/derived/types/ilscripts.cfg @@ -1,4 +1,5 @@ maxtermoccurrences 10000 +maxtokenlength 1000 fieldmatchmaxlength 1000000 ilscript[].doctype "types" ilscript[].docfield[] "abyte" diff --git a/config-model/src/test/derived/uri_array/ilscripts.cfg b/config-model/src/test/derived/uri_array/ilscripts.cfg index 3dd97e5c11f..0dc87b513ce 100644 --- a/config-model/src/test/derived/uri_array/ilscripts.cfg +++ b/config-model/src/test/derived/uri_array/ilscripts.cfg @@ -1,4 +1,5 @@ maxtermoccurrences 10000 +maxtokenlength 1000 fieldmatchmaxlength 1000000 ilscript[].doctype "uri_array" ilscript[].docfield[] "my_uri" diff --git a/config-model/src/test/derived/uri_wset/ilscripts.cfg b/config-model/src/test/derived/uri_wset/ilscripts.cfg index 48e07ef9959..cc45ee5ad8f 100644 --- a/config-model/src/test/derived/uri_wset/ilscripts.cfg +++ b/config-model/src/test/derived/uri_wset/ilscripts.cfg @@ -1,4 +1,5 @@ maxtermoccurrences 10000 +maxtokenlength 1000 fieldmatchmaxlength 1000000 ilscript[].doctype "uri_wset" ilscript[].docfield[] "my_uri" diff --git a/configdefinitions/src/vespa/ilscripts.def b/configdefinitions/src/vespa/ilscripts.def index acb06abb755..7a286773564 100644 --- a/configdefinitions/src/vespa/ilscripts.def +++ b/configdefinitions/src/vespa/ilscripts.def @@ -3,6 +3,8 @@ namespace=vespa.configdefinition ## The maximum number of occurrences of a given term to index per field maxtermoccurrences int default=10000 +## The maximum number of characters for a token +maxtokenlength int default=1000 fieldmatchmaxlength int default=1000000 ilscript[].doctype string diff --git a/container-core/src/main/java/com/yahoo/processing/request/CompoundName.java b/container-core/src/main/java/com/yahoo/processing/request/CompoundName.java index b4536a1c56b..440df4f9be9 100644 --- a/container-core/src/main/java/com/yahoo/processing/request/CompoundName.java +++ b/container-core/src/main/java/com/yahoo/processing/request/CompoundName.java @@ -8,6 +8,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.Map; +import java.util.Objects; import static com.yahoo.text.Lowercase.toLowerCase; @@ -74,41 +75,52 @@ public final class CompoundName { * @param compounds the compounds of this name */ private CompoundName(String name, String [] compounds, boolean useCache) { - if (name == null) throw new NullPointerException("Name can not be null"); - - this.name = name; + this.name = Objects.requireNonNull(name, "Name can not be null"); this.lowerCasedName = toLowerCase(name); - if (compounds.length == 1 && compounds[0].isEmpty()) { - this.compounds = List.of(); - this.hashCode = 0; - rest = this; - first = this; + if (compounds.length == 1) { + if (compounds[0].isEmpty()) { + this.compounds = List.of(); + this.hashCode = 0; + rest = first = this; + return; + } + this.compounds = new ImmutableArrayList(compounds); + this.hashCode = this.compounds.hashCode(); + rest = first = empty; return; } - this.compounds = new ImmutableArrayList(compounds); - this.hashCode = this.compounds.hashCode(); - - if (compounds.length > 1) { - String restName = name.substring(compounds[0].length()+1); - if (useCache) { - rest = cache.computeIfAbsent(restName, (key) -> new CompoundName(key, Arrays.copyOfRange(compounds, 1, compounds.length), useCache)); - } else { - rest = new CompoundName(restName, Arrays.copyOfRange(compounds, 1, compounds.length), useCache); + CompoundName[] children = new CompoundName[compounds.length]; + for (int i = 0; i + 1 < children.length; i++) { + int start = 0, end = i == 0 ? -1 : children[0].name.length(); + for (int j = 0; j + i < children.length; j++) { + end += compounds[j + i].length() + 1; + if (end == start) throw new IllegalArgumentException("'" + name + "' is not a legal compound name. " + + "Consecutive, leading or trailing dots are not allowed."); + String subName = this.name.substring(start, end); + CompoundName cached = cache.get(subName); + children[j] = cached != null ? cached + : new CompoundName(subName, + this.lowerCasedName.substring(start, end), + Arrays.copyOfRange(compounds, j, j + i + 1), + i == 0 ? empty : children[j + 1], + i == 0 ? empty : children[j]); + if (useCache && cached == null) cache.put(subName, children[j]); + start += compounds[j].length() + 1; } - } else { - rest = empty; } + this.compounds = new ImmutableArrayList(compounds); + this.hashCode = this.compounds.hashCode(); + this.rest = children[1]; + this.first = children[0]; + } - if (compounds.length > 1) { - String firstName = name.substring(0, name.length() - (compounds[compounds.length-1].length()+1)); - if (useCache) { - first = cache.computeIfAbsent(firstName, (key) -> new CompoundName(key, Arrays.copyOfRange(compounds, 0, compounds.length-1), useCache)); - } else { - first = new CompoundName(firstName, Arrays.copyOfRange(compounds, 0, compounds.length-1), useCache); - } - } else { - first = empty; - } + private CompoundName(String name, String lowerCasedName, String[] compounds, CompoundName rest, CompoundName first) { + this.name = name; + this.lowerCasedName = lowerCasedName; + this.compounds = new ImmutableArrayList(compounds); + this.hashCode = this.compounds.hashCode(); + this.rest = rest; + this.first = first; } private static List<String> parse(String s) { diff --git a/container-core/src/test/java/com/yahoo/processing/request/CompoundNameTestCase.java b/container-core/src/test/java/com/yahoo/processing/request/CompoundNameTestCase.java index b5143f89c78..7523a68501f 100644 --- a/container-core/src/test/java/com/yahoo/processing/request/CompoundNameTestCase.java +++ b/container-core/src/test/java/com/yahoo/processing/request/CompoundNameTestCase.java @@ -13,7 +13,7 @@ import static org.junit.jupiter.api.Assertions.*; /** * Module local test of the basic property name building block. * - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + * @author Steinar Knutsen */ public class CompoundNameTestCase { @@ -30,22 +30,22 @@ public class CompoundNameTestCase { } @Test - final void testLast() { + void testLast() { assertEquals(NAME.substring(NAME.lastIndexOf('.') + 1), C_NAME.last()); } @Test - final void testFirst() { + void testFirst() { assertEquals(NAME.substring(0, NAME.indexOf('.')), C_NAME.first()); } @Test - final void testRest() { + void testRest() { verifyStrict(NAME.substring(NAME.indexOf('.') + 1), C_NAME.rest()); } @Test - final void testRestN() { + void testRestN() { verifyStrict("a.b.c.d.e", C_abcde.rest(0)); verifyStrict("b.c.d.e", C_abcde.rest(1)); verifyStrict("c.d.e", C_abcde.rest(2)); @@ -53,8 +53,9 @@ public class CompoundNameTestCase { verifyStrict("e", C_abcde.rest(4)); verifyStrict(CompoundName.empty, C_abcde.rest(5)); } + @Test - final void testFirstN() { + void testFirstN() { verifyStrict("a.b.c.d.e", C_abcde.first(5)); verifyStrict("a.b.c.d", C_abcde.first(4)); verifyStrict("a.b.c", C_abcde.first(3)); @@ -64,15 +65,32 @@ public class CompoundNameTestCase { } @Test - final void testPrefix() { - CompoundName abc = CompoundName.from("a.b.c"); - assertTrue(abc.hasPrefix(CompoundName.empty)); - assertTrue(abc.hasPrefix(CompoundName.from("a"))); - assertTrue(abc.hasPrefix(CompoundName.from("a.b"))); - assertTrue(abc.hasPrefix(CompoundName.from("a.b.c"))); + void testPrefix() { + CompoundName abcc = CompoundName.from("a.b.cc"); + assertTrue(abcc.hasPrefix(CompoundName.empty)); + assertTrue(abcc.hasPrefix(CompoundName.from("a"))); + assertTrue(abcc.hasPrefix(CompoundName.from("a.b"))); + assertTrue(abcc.hasPrefix(CompoundName.from("a.b.cc"))); - assertFalse(abc.hasPrefix(CompoundName.from("a.b.c.d"))); - assertFalse(abc.hasPrefix(CompoundName.from("a.b.d"))); + assertFalse(abcc.hasPrefix(CompoundName.from("a.b.c"))); + assertFalse(abcc.hasPrefix(CompoundName.from("a.b.c.d"))); + assertFalse(abcc.hasPrefix(CompoundName.from("a.b.d"))); + } + + @Test + void testIllegalCompound() { + assertEquals("'a.' is not a legal compound name. Names can not end with a dot.", + assertThrows(IllegalArgumentException.class, + () -> CompoundName.from("a.")) + .getMessage()); + assertEquals("'.b' is not a legal compound name. Consecutive, leading or trailing dots are not allowed.", + assertThrows(IllegalArgumentException.class, + () -> CompoundName.from(".b")) + .getMessage()); + assertEquals("'a..b' is not a legal compound name. Consecutive, leading or trailing dots are not allowed.", + assertThrows(IllegalArgumentException.class, + () -> CompoundName.from("a..b")) + .getMessage()); } @Test @@ -82,7 +100,7 @@ public class CompoundNameTestCase { } @Test - final void testSize() { + void testSize() { Splitter s = Splitter.on('.'); Iterable<String> i = s.split(NAME); int n = 0; @@ -93,23 +111,23 @@ public class CompoundNameTestCase { } @Test - final void testGet() { + void testGet() { String s = C_NAME.get(0); assertEquals(NAME.substring(0, NAME.indexOf('.')), s); } @Test - final void testIsCompound() { + void testIsCompound() { assertTrue(C_NAME.isCompound()); } @Test - final void testIsEmpty() { + void testIsEmpty() { assertFalse(C_NAME.isEmpty()); } @Test - final void testAsList() { + void testAsList() { List<String> l = C_NAME.asList(); Splitter peoplesFront = Splitter.on('.'); Iterable<String> answer = peoplesFront.split(NAME); @@ -121,7 +139,7 @@ public class CompoundNameTestCase { } @Test - final void testEqualsObject() { + void testEqualsObject() { assertNotEquals(C_NAME, NAME); assertNotEquals(C_NAME, null); verifyStrict(C_NAME, C_NAME); @@ -129,7 +147,7 @@ public class CompoundNameTestCase { } @Test - final void testEmptyNonEmpty() { + void testEmptyNonEmpty() { assertTrue(CompoundName.empty.isEmpty()); assertEquals(0, CompoundName.empty.size()); assertFalse(CompoundName.from("a").isEmpty()); @@ -140,7 +158,7 @@ public class CompoundNameTestCase { } @Test - final void testGetLowerCasedName() { + void testGetLowerCasedName() { assertEquals(Lowercase.toLowerCase(NAME), C_NAME.getLowerCasedName()); } @@ -223,4 +241,5 @@ public class CompoundNameTestCase { assertEquals("[one]", CompoundName.from("one").asList().toString()); assertEquals("[one, two, three]", CompoundName.from("one.two.three").asList().toString()); } + } diff --git a/container-search/src/main/java/com/yahoo/search/handler/Json2SingleLevelMap.java b/container-search/src/main/java/com/yahoo/search/handler/Json2SingleLevelMap.java index fdedbdc2fd9..01167be6b8b 100644 --- a/container-search/src/main/java/com/yahoo/search/handler/Json2SingleLevelMap.java +++ b/container-search/src/main/java/com/yahoo/search/handler/Json2SingleLevelMap.java @@ -64,8 +64,8 @@ class Json2SingleLevelMap { } void parse(Map<String, String> map, String parent) throws IOException { - for (parser.nextToken(); parser.currentToken() != JsonToken.END_OBJECT; parser.nextToken()) { - String fieldName = parent + parser.currentName(); + for (parser.nextToken(); parser.getCurrentToken() != JsonToken.END_OBJECT; parser.nextToken()) { + String fieldName = parent + parser.getCurrentName(); JsonToken token = parser.nextToken(); if ((token == JsonToken.VALUE_STRING) || (token == JsonToken.VALUE_NUMBER_FLOAT) || @@ -89,9 +89,9 @@ class Json2SingleLevelMap { } private String skipChildren(JsonParser parser, byte [] input) throws IOException { - JsonLocation start = parser.currentLocation(); + JsonLocation start = parser.getCurrentLocation(); parser.skipChildren(); - JsonLocation end = parser.currentLocation(); + JsonLocation end = parser.getCurrentLocation(); int offset = (int)start.getByteOffset() - 1; return new String(input, offset, (int)(end.getByteOffset() - offset), StandardCharsets.UTF_8); } diff --git a/container-search/src/test/java/com/yahoo/search/rendering/JsonRendererTestCase.java b/container-search/src/test/java/com/yahoo/search/rendering/JsonRendererTestCase.java index 611df6ad284..ffa6c82e941 100644 --- a/container-search/src/test/java/com/yahoo/search/rendering/JsonRendererTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/rendering/JsonRendererTestCase.java @@ -1532,7 +1532,7 @@ public class JsonRendererTestCase { + "}"; assertEquals( "Unexpected character ('a' (code 97)): was expecting comma to separate Object entries\n" + - " at [Source: (String)\"{ \"root\": { \"invalidvalue\": 1adsf, }}\"; line: 1, column: 40]", + " at [Source: (String)\"{ \"root\": { \"invalidvalue\": 1adsf, }}\"; line: 1, column: 41]", validateJSON(json)); } diff --git a/dependency-versions/pom.xml b/dependency-versions/pom.xml index d27318aaaf9..0876674e8c6 100644 --- a/dependency-versions/pom.xml +++ b/dependency-versions/pom.xml @@ -37,7 +37,7 @@ <guava.vespa.version>33.2.0-jre</guava.vespa.version> <guice.vespa.version>6.0.0</guice.vespa.version> <j2objc-annotations.vespa.version>3.0.0</j2objc-annotations.vespa.version> - <jackson2.vespa.version>2.17.1</jackson2.vespa.version> + <jackson2.vespa.version>2.16.2</jackson2.vespa.version> <jackson-databind.vespa.version>${jackson2.vespa.version}</jackson-databind.vespa.version> <jakarta.inject.vespa.version>2.0.1</jakarta.inject.vespa.version> <javax.activation-api.vespa.version>1.2.0</javax.activation-api.vespa.version> diff --git a/docprocs/src/main/java/com/yahoo/docprocs/indexing/ScriptManager.java b/docprocs/src/main/java/com/yahoo/docprocs/indexing/ScriptManager.java index 86b0a2e78ad..3088083912b 100644 --- a/docprocs/src/main/java/com/yahoo/docprocs/indexing/ScriptManager.java +++ b/docprocs/src/main/java/com/yahoo/docprocs/indexing/ScriptManager.java @@ -72,7 +72,7 @@ public class ScriptManager { Map<String, Map<String, DocumentScript>> documentFieldScripts = new HashMap<>(config.ilscript().size()); ScriptParserContext parserContext = new ScriptParserContext(linguistics, embedders); parserContext.getAnnotatorConfig().setMaxTermOccurrences(config.maxtermoccurrences()); - parserContext.getAnnotatorConfig().setMaxTokenLength(config.fieldmatchmaxlength()); + parserContext.getAnnotatorConfig().setMaxTokenizeLength(config.fieldmatchmaxlength()); for (IlscriptsConfig.Ilscript ilscript : config.ilscript()) { DocumentType documentType = docTypeMgr.getDocumentType(ilscript.doctype()); diff --git a/document/src/main/java/com/yahoo/document/json/JsonReader.java b/document/src/main/java/com/yahoo/document/json/JsonReader.java index 9c621c033bd..358c0cb65e4 100644 --- a/document/src/main/java/com/yahoo/document/json/JsonReader.java +++ b/document/src/main/java/com/yahoo/document/json/JsonReader.java @@ -105,7 +105,7 @@ public class JsonReader { String condition = null; ParsedDocumentOperation operation = null; while (JsonToken.END_OBJECT != parser.nextValue()) { - switch (parser.currentName()) { + switch (parser.getCurrentName()) { case FIELDS -> { documentParseInfo.fieldsBuffer = new LazyTokenBuffer(parser); VespaJsonDocumentReader vespaJsonDocumentReader = new VespaJsonDocumentReader(typeManager.getIgnoreUndefinedFields()); @@ -177,7 +177,7 @@ public class JsonReader { state = END_OF_FEED; throw new IllegalArgumentException(r); } - if (documentParseInfo.isEmpty()) { + if ( ! documentParseInfo.isPresent()) { state = END_OF_FEED; return null; } diff --git a/document/src/main/java/com/yahoo/document/json/LazyTokenBuffer.java b/document/src/main/java/com/yahoo/document/json/LazyTokenBuffer.java index 53ddacf6cc3..0fbdd0b28c7 100644 --- a/document/src/main/java/com/yahoo/document/json/LazyTokenBuffer.java +++ b/document/src/main/java/com/yahoo/document/json/LazyTokenBuffer.java @@ -33,7 +33,7 @@ public class LazyTokenBuffer extends TokenBuffer { public Supplier<Token> lookahead() { return new Supplier<>() { int localNesting = nesting(); - final Supplier<Token> buffered = LazyTokenBuffer.super.lookahead(); + Supplier<Token> buffered = LazyTokenBuffer.super.lookahead(); @Override public Token get() { if (localNesting == 0) return null; @@ -54,7 +54,7 @@ public class LazyTokenBuffer extends TokenBuffer { JsonToken token = parser.nextValue(); if (token == null) throw new IllegalStateException("no more JSON tokens"); - return new Token(token, parser.currentName(), parser.getText()); + return new Token(token, parser.getCurrentName(), parser.getText()); } catch (IOException e) { throw new IllegalArgumentException("failed reading document JSON", e); diff --git a/document/src/main/java/com/yahoo/document/json/TokenBuffer.java b/document/src/main/java/com/yahoo/document/json/TokenBuffer.java index c5c022370bf..3a48f71c4cd 100644 --- a/document/src/main/java/com/yahoo/document/json/TokenBuffer.java +++ b/document/src/main/java/com/yahoo/document/json/TokenBuffer.java @@ -99,7 +99,7 @@ public class TokenBuffer { } int addFromParser(JsonParser tokens) throws IOException { - add(tokens.currentToken(), tokens.currentName(), tokens.getText()); + add(tokens.currentToken(), tokens.getCurrentName(), tokens.getText()); return nestingOffset(tokens.currentToken()); } diff --git a/document/src/main/java/com/yahoo/document/json/document/DocumentParser.java b/document/src/main/java/com/yahoo/document/json/document/DocumentParser.java index c5bcd356c94..77e11dcf2a8 100644 --- a/document/src/main/java/com/yahoo/document/json/document/DocumentParser.java +++ b/document/src/main/java/com/yahoo/document/json/document/DocumentParser.java @@ -61,7 +61,7 @@ public class DocumentParser { private boolean parseOneItem(DocumentParseInfo documentParseInfo, boolean docIdAndOperationIsSetExternally) throws IOException { parser.nextValue(); processIndent(); - if (parser.currentName() == null) return false; + if (parser.getCurrentName() == null) return false; if (indentLevel == 1L) { handleIdentLevelOne(documentParseInfo, docIdAndOperationIsSetExternally); } else if (indentLevel == 2L) { @@ -85,18 +85,17 @@ public class DocumentParser { private void handleIdentLevelOne(DocumentParseInfo documentParseInfo, boolean docIdAndOperationIsSetExternally) throws IOException { - JsonToken currentToken = parser.currentToken(); - String currentName = parser.currentName(); + JsonToken currentToken = parser.getCurrentToken(); if ((currentToken == JsonToken.VALUE_TRUE || currentToken == JsonToken.VALUE_FALSE) && - CREATE_IF_NON_EXISTENT.equals(currentName)) { + CREATE_IF_NON_EXISTENT.equals(parser.getCurrentName())) { documentParseInfo.create = Optional.of(currentToken == JsonToken.VALUE_TRUE); - } else if (currentToken == JsonToken.VALUE_STRING && CONDITION.equals(currentName)) { + } else if (currentToken == JsonToken.VALUE_STRING && CONDITION.equals(parser.getCurrentName())) { documentParseInfo.condition = Optional.of(parser.getText()); } else if (currentToken == JsonToken.VALUE_STRING) { // Value is expected to be set in the header not in the document. Ignore any unknown field // as well. if (! docIdAndOperationIsSetExternally) { - documentParseInfo.operationType = operationNameToOperationType(currentName); + documentParseInfo.operationType = operationNameToOperationType(parser.getCurrentName()); documentParseInfo.documentId = new DocumentId(parser.getText()); } } @@ -105,7 +104,7 @@ public class DocumentParser { private void handleIdentLevelTwo(DocumentParseInfo documentParseInfo) { try { // "fields" opens a dictionary and is therefore on level two which might be surprising. - if (parser.currentToken() == JsonToken.START_OBJECT && FIELDS.equals(parser.currentName())) { + if (parser.currentToken() == JsonToken.START_OBJECT && FIELDS.equals(parser.getCurrentName())) { documentParseInfo.fieldsBuffer.bufferObject(parser); processIndent(); } diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/TokenizeExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/TokenizeExpression.java index b807ad4cb65..849bc075a64 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/TokenizeExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/TokenizeExpression.java @@ -66,7 +66,7 @@ public final class TokenizeExpression extends Expression { if (config.getStemMode() != StemMode.NONE) { ret.append(" stem:\""+config.getStemMode()+"\""); } - if (config.hasNonDefaultMaxTokenLength()) { + if (config.hasNonDefaultMaxTokenizeLength()) { ret.append(" max-length:" + config.getMaxTokenizeLength()); } if (config.hasNonDefaultMaxTermOccurrences()) { diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/linguistics/AnnotatorConfig.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/linguistics/AnnotatorConfig.java index 7b6f350d831..4e5ef0d90df 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/linguistics/AnnotatorConfig.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/linguistics/AnnotatorConfig.java @@ -82,7 +82,7 @@ public class AnnotatorConfig implements Cloneable { return this; } - public AnnotatorConfig setMaxTokenLength(int maxTokenizeLength) { + public AnnotatorConfig setMaxTokenizeLength(int maxTokenizeLength) { this.maxTokenizeLength = maxTokenizeLength; return this; } @@ -91,7 +91,7 @@ public class AnnotatorConfig implements Cloneable { return maxTokenizeLength; } - public boolean hasNonDefaultMaxTokenLength() { + public boolean hasNonDefaultMaxTokenizeLength() { return maxTokenizeLength != DEFAULT_MAX_TOKENIZE_LENGTH; } diff --git a/indexinglanguage/src/main/javacc/IndexingParser.jj b/indexinglanguage/src/main/javacc/IndexingParser.jj index 469d96ead60..77591d3e54e 100644 --- a/indexinglanguage/src/main/javacc/IndexingParser.jj +++ b/indexinglanguage/src/main/javacc/IndexingParser.jj @@ -689,7 +689,7 @@ AnnotatorConfig tokenizeCfg() : } { ( <STEM> ( <COLON> str = string() ) ? { val.setStemMode(str); } | - <MAX_LENGTH> <COLON> maxLength = integer() { val.setMaxTokenLength(maxLength); } | + <MAX_LENGTH> <COLON> maxLength = integer() { val.setMaxTokenizeLength(maxLength); } | <MAX_OCCURRENCES> <COLON> maxTermOccurrences = integer() { val.setMaxTermOccurrences(maxTermOccurrences); } | <NORMALIZE> { val.setRemoveAccents(true); } )+ { return val; } diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/linguistics/LinguisticsAnnotatorTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/linguistics/LinguisticsAnnotatorTestCase.java index 136e71564d8..461c915acef 100644 --- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/linguistics/LinguisticsAnnotatorTestCase.java +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/linguistics/LinguisticsAnnotatorTestCase.java @@ -194,7 +194,7 @@ public class LinguisticsAnnotatorTestCase { Linguistics linguistics = new SimpleLinguistics(); - LinguisticsAnnotator annotator = new LinguisticsAnnotator(linguistics, new AnnotatorConfig().setMaxTokenLength(12)); + LinguisticsAnnotator annotator = new LinguisticsAnnotator(linguistics, new AnnotatorConfig().setMaxTokenizeLength(12)); assertTrue(annotator.annotate(shortValue)); assertEquals(spanTree, shortValue.getSpanTree(SpanTrees.LINGUISTICS)); diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/MetricsParser.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/MetricsParser.java index 052b8425a45..0e33d7dbf2f 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/MetricsParser.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/MetricsParser.java @@ -53,8 +53,8 @@ public class MetricsParser { throw new IOException("Expected start of object, got " + parser.currentToken()); } - for (parser.nextToken(); parser.currentToken() != JsonToken.END_OBJECT; parser.nextToken()) { - String fieldName = parser.currentName(); + for (parser.nextToken(); parser.getCurrentToken() != JsonToken.END_OBJECT; parser.nextToken()) { + String fieldName = parser.getCurrentName(); JsonToken token = parser.nextToken(); if (fieldName.equals("metrics")) { parseMetrics(parser, consumer); @@ -67,12 +67,12 @@ public class MetricsParser { } static private Instant parseSnapshot(JsonParser parser) throws IOException { - if (parser.currentToken() != JsonToken.START_OBJECT) { + if (parser.getCurrentToken() != JsonToken.START_OBJECT) { throw new IOException("Expected start of 'snapshot' object, got " + parser.currentToken()); } Instant timestamp = Instant.now(); - for (parser.nextToken(); parser.currentToken() != JsonToken.END_OBJECT; parser.nextToken()) { - String fieldName = parser.currentName(); + for (parser.nextToken(); parser.getCurrentToken() != JsonToken.END_OBJECT; parser.nextToken()) { + String fieldName = parser.getCurrentName(); JsonToken token = parser.nextToken(); if (fieldName.equals("to")) { timestamp = Instant.ofEpochSecond(parser.getLongValue()); @@ -88,12 +88,12 @@ public class MetricsParser { // 'metrics' object with 'snapshot' and 'values' arrays static private void parseMetrics(JsonParser parser, Collector consumer) throws IOException { - if (parser.currentToken() != JsonToken.START_OBJECT) { + if (parser.getCurrentToken() != JsonToken.START_OBJECT) { throw new IOException("Expected start of 'metrics' object, got " + parser.currentToken()); } Instant timestamp = Instant.now(); - for (parser.nextToken(); parser.currentToken() != JsonToken.END_OBJECT; parser.nextToken()) { - String fieldName = parser.currentName(); + for (parser.nextToken(); parser.getCurrentToken() != JsonToken.END_OBJECT; parser.nextToken()) { + String fieldName = parser.getCurrentName(); JsonToken token = parser.nextToken(); if (fieldName.equals("snapshot")) { timestamp = parseSnapshot(parser); @@ -109,7 +109,7 @@ public class MetricsParser { // 'values' array static private void parseMetricValues(JsonParser parser, Instant timestamp, Collector consumer) throws IOException { - if (parser.currentToken() != JsonToken.START_ARRAY) { + if (parser.getCurrentToken() != JsonToken.START_ARRAY) { throw new IOException("Expected start of 'metrics:values' array, got " + parser.currentToken()); } @@ -126,8 +126,8 @@ public class MetricsParser { String description = ""; Map<DimensionId, String> dim = Map.of(); List<Map.Entry<String, Number>> values = List.of(); - for (parser.nextToken(); parser.currentToken() != JsonToken.END_OBJECT; parser.nextToken()) { - String fieldName = parser.currentName(); + for (parser.nextToken(); parser.getCurrentToken() != JsonToken.END_OBJECT; parser.nextToken()) { + String fieldName = parser.getCurrentName(); JsonToken token = parser.nextToken(); switch (fieldName) { case "name" -> name = parser.getText(); @@ -154,8 +154,8 @@ public class MetricsParser { Set<Dimension> dimensions = new HashSet<>(); - for (parser.nextToken(); parser.currentToken() != JsonToken.END_OBJECT; parser.nextToken()) { - String fieldName = parser.currentName(); + for (parser.nextToken(); parser.getCurrentToken() != JsonToken.END_OBJECT; parser.nextToken()) { + String fieldName = parser.getCurrentName(); JsonToken token = parser.nextToken(); if (token == JsonToken.VALUE_STRING){ @@ -180,16 +180,17 @@ public class MetricsParser { private static List<Map.Entry<String, Number>> parseValues(JsonParser parser) throws IOException { List<Map.Entry<String, Number>> metrics = new ArrayList<>(); - for (parser.nextToken(); parser.currentToken() != JsonToken.END_OBJECT; parser.nextToken()) { - String metricName = parser.currentName(); + for (parser.nextToken(); parser.getCurrentToken() != JsonToken.END_OBJECT; parser.nextToken()) { + String fieldName = parser.getCurrentName(); JsonToken token = parser.nextToken(); + String metricName = fieldName; if (token == JsonToken.VALUE_NUMBER_INT) { metrics.add(Map.entry(metricName, parser.getLongValue())); } else if (token == JsonToken.VALUE_NUMBER_FLOAT) { double value = parser.getValueAsDouble(); metrics.add(Map.entry(metricName, value == ZERO_DOUBLE ? ZERO_DOUBLE : value)); } else { - throw new IllegalArgumentException("Value for aggregator '" + metricName + "' is not a number"); + throw new IllegalArgumentException("Value for aggregator '" + fieldName + "' is not a number"); } } return metrics; diff --git a/model-integration/abi-spec.json b/model-integration/abi-spec.json index e7130d9c777..31f2b64d728 100644 --- a/model-integration/abi-spec.json +++ b/model-integration/abi-spec.json @@ -94,6 +94,7 @@ "public ai.vespa.llm.clients.LlmLocalClientConfig$Builder model(com.yahoo.config.ModelReference)", "public ai.vespa.llm.clients.LlmLocalClientConfig$Builder parallelRequests(int)", "public ai.vespa.llm.clients.LlmLocalClientConfig$Builder maxQueueSize(int)", + "public ai.vespa.llm.clients.LlmLocalClientConfig$Builder maxQueueWait(int)", "public ai.vespa.llm.clients.LlmLocalClientConfig$Builder useGpu(boolean)", "public ai.vespa.llm.clients.LlmLocalClientConfig$Builder gpuLayers(int)", "public ai.vespa.llm.clients.LlmLocalClientConfig$Builder threads(int)", @@ -139,6 +140,7 @@ "public java.nio.file.Path model()", "public int parallelRequests()", "public int maxQueueSize()", + "public int maxQueueWait()", "public boolean useGpu()", "public int gpuLayers()", "public int threads()", diff --git a/model-integration/src/main/java/ai/vespa/llm/clients/LocalLLM.java b/model-integration/src/main/java/ai/vespa/llm/clients/LocalLLM.java index aa7c071b93a..b6409b5466d 100644 --- a/model-integration/src/main/java/ai/vespa/llm/clients/LocalLLM.java +++ b/model-integration/src/main/java/ai/vespa/llm/clients/LocalLLM.java @@ -3,6 +3,7 @@ package ai.vespa.llm.clients; import ai.vespa.llm.InferenceParameters; import ai.vespa.llm.LanguageModel; +import ai.vespa.llm.LanguageModelException; import ai.vespa.llm.completion.Completion; import ai.vespa.llm.completion.Prompt; import com.yahoo.component.AbstractComponent; @@ -14,10 +15,14 @@ import java.util.ArrayList; import java.util.List; import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.SynchronousQueue; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.Consumer; import java.util.logging.Logger; @@ -29,14 +34,19 @@ import java.util.logging.Logger; public class LocalLLM extends AbstractComponent implements LanguageModel { private final static Logger logger = Logger.getLogger(LocalLLM.class.getName()); + + private final ScheduledExecutorService scheduler = Executors.newSingleThreadScheduledExecutor(); + private final LlamaModel model; private final ThreadPoolExecutor executor; + private final long queueTimeoutMilliseconds; private final int contextSize; private final int maxTokens; @Inject public LocalLLM(LlmLocalClientConfig config) { executor = createExecutor(config); + queueTimeoutMilliseconds = config.maxQueueWait(); // Maximum number of tokens to generate - need this since some models can just generate infinitely maxTokens = config.maxTokens(); @@ -74,6 +84,7 @@ public class LocalLLM extends AbstractComponent implements LanguageModel { logger.info("Closing LLM model..."); model.close(); executor.shutdownNow(); + scheduler.shutdownNow(); } @Override @@ -104,22 +115,39 @@ public class LocalLLM extends AbstractComponent implements LanguageModel { // Todo: more options? var completionFuture = new CompletableFuture<Completion.FinishReason>(); + var hasStarted = new AtomicBoolean(false); try { - executor.submit(() -> { + Future<?> future = executor.submit(() -> { + hasStarted.set(true); for (LlamaModel.Output output : model.generate(inferParams)) { consumer.accept(Completion.from(output.text, Completion.FinishReason.none)); } completionFuture.complete(Completion.FinishReason.stop); }); + + if (queueTimeoutMilliseconds > 0) { + scheduler.schedule(() -> { + if ( ! hasStarted.get()) { + future.cancel(false); + String error = rejectedExecutionReason("Rejected completion due to timeout waiting to start"); + completionFuture.completeExceptionally(new LanguageModelException(504, error)); + } + }, queueTimeoutMilliseconds, TimeUnit.MILLISECONDS); + } + } catch (RejectedExecutionException e) { // If we have too many requests (active + any waiting in queue), we reject the completion - int activeCount = executor.getActiveCount(); - int queueSize = executor.getQueue().size(); - String error = String.format("Rejected completion due to too many requests, " + - "%d active, %d in queue", activeCount, queueSize); + String error = rejectedExecutionReason("Rejected completion due to too many requests"); throw new RejectedExecutionException(error); } return completionFuture; } + private String rejectedExecutionReason(String prepend) { + int activeCount = executor.getActiveCount(); + int queueSize = executor.getQueue().size(); + return String.format("%s, %d active, %d in queue", prepend, activeCount, queueSize); + } + + } diff --git a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/lightgbm/LightGBMImporter.java b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/lightgbm/LightGBMImporter.java index 6a1e2f2562a..e1d2f8802a6 100644 --- a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/lightgbm/LightGBMImporter.java +++ b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/lightgbm/LightGBMImporter.java @@ -34,9 +34,9 @@ public class LightGBMImporter extends ModelImporter { private boolean probe(File modelFile) { try (JsonParser parser = Jackson.mapper().createParser(modelFile)) { while (parser.nextToken() != null) { - JsonToken token = parser.currentToken(); + JsonToken token = parser.getCurrentToken(); if (token == JsonToken.FIELD_NAME) { - if ("tree_info".equals(parser.currentName())) return true; + if ("tree_info".equals(parser.getCurrentName())) return true; } } return false; diff --git a/model-integration/src/main/resources/configdefinitions/llm-local-client.def b/model-integration/src/main/resources/configdefinitions/llm-local-client.def index 4823a53ec46..6b83ffd0751 100755 --- a/model-integration/src/main/resources/configdefinitions/llm-local-client.def +++ b/model-integration/src/main/resources/configdefinitions/llm-local-client.def @@ -8,7 +8,10 @@ model model parallelRequests int default=1 # Additional number of requests to put in queue for processing before starting to reject new requests -maxQueueSize int default=10 +maxQueueSize int default=100 + +# Max number of milliseoncds to wait in the queue before rejecting a request +maxQueueWait int default=10000 # Use GPU useGpu bool default=true @@ -24,6 +27,6 @@ threads int default=-1 # Context is divided between parallel requests. So for 10 parallel requests, each "slot" gets 1/10 of the context contextSize int default=4096 -# Maximum number of tokens to process in one request - overriden by inference parameters +# Maximum number of tokens to process in one request - overridden by inference parameters maxTokens int default=512 diff --git a/model-integration/src/test/java/ai/vespa/llm/clients/LocalLLMTest.java b/model-integration/src/test/java/ai/vespa/llm/clients/LocalLLMTest.java index a3b260f3fb5..4db1140d171 100644 --- a/model-integration/src/test/java/ai/vespa/llm/clients/LocalLLMTest.java +++ b/model-integration/src/test/java/ai/vespa/llm/clients/LocalLLMTest.java @@ -2,6 +2,7 @@ package ai.vespa.llm.clients; import ai.vespa.llm.InferenceParameters; +import ai.vespa.llm.LanguageModelException; import ai.vespa.llm.completion.Completion; import ai.vespa.llm.completion.Prompt; import ai.vespa.llm.completion.StringPrompt; @@ -96,7 +97,6 @@ public class LocalLLMTest { try { for (int i = 0; i < promptsToUse; i++) { final var seq = i; - completions.set(seq, new StringBuilder()); futures.set(seq, llm.completeAsync(StringPrompt.from(prompts.get(seq)), defaultOptions(), completion -> { completions.get(seq).append(completion.text()); @@ -122,8 +122,9 @@ public class LocalLLMTest { var prompts = testPrompts(); var promptsToUse = prompts.size(); var parallelRequests = 2; - var additionalQueue = 1; - // 7 should be rejected + var additionalQueue = 100; + var queueWaitTime = 10; + // 8 should be rejected due to queue wait time var futures = new ArrayList<CompletableFuture<Completion.FinishReason>>(Collections.nCopies(promptsToUse, null)); var completions = new ArrayList<StringBuilder>(Collections.nCopies(promptsToUse, null)); @@ -131,10 +132,12 @@ public class LocalLLMTest { var config = new LlmLocalClientConfig.Builder() .parallelRequests(parallelRequests) .maxQueueSize(additionalQueue) + .maxQueueWait(queueWaitTime) .model(ModelReference.valueOf(model)); var llm = new LocalLLM(config.build()); var rejected = new AtomicInteger(0); + var timedOut = new AtomicInteger(0); try { for (int i = 0; i < promptsToUse; i++) { final var seq = i; @@ -143,7 +146,14 @@ public class LocalLLMTest { try { var future = llm.completeAsync(StringPrompt.from(prompts.get(seq)), defaultOptions(), completion -> { completions.get(seq).append(completion.text()); - }).exceptionally(exception -> Completion.FinishReason.error); + }).exceptionally(exception -> { + if (exception instanceof LanguageModelException lme) { + if (lme.code() == 504) { + timedOut.incrementAndGet(); + } + } + return Completion.FinishReason.error; + }); futures.set(seq, future); } catch (RejectedExecutionException e) { rejected.incrementAndGet(); @@ -151,13 +161,14 @@ public class LocalLLMTest { } for (int i = 0; i < promptsToUse; i++) { if (futures.get(i) != null) { - assertNotEquals(futures.get(i).join(), Completion.FinishReason.error); + futures.get(i).join(); } } } finally { llm.deconstruct(); } - assertEquals(7, rejected.get()); + assertEquals(0, rejected.get()); + assertEquals(8, timedOut.get()); } private static InferenceParameters defaultOptions() { diff --git a/predicate-search-core/src/main/java/com/yahoo/search/predicate/PredicateQueryParser.java b/predicate-search-core/src/main/java/com/yahoo/search/predicate/PredicateQueryParser.java index 42b6195549e..09487506ffe 100644 --- a/predicate-search-core/src/main/java/com/yahoo/search/predicate/PredicateQueryParser.java +++ b/predicate-search-core/src/main/java/com/yahoo/search/predicate/PredicateQueryParser.java @@ -10,6 +10,7 @@ import java.util.Arrays; /** * Parses predicate queries from JSON. + * * Input JSON is assumed to have the following format: * { * "features": [ @@ -45,7 +46,7 @@ public class PredicateQueryParser { try (JsonParser parser = factory.createParser(json)) { skipToken(parser, JsonToken.START_OBJECT); while (parser.nextToken() != JsonToken.END_OBJECT) { - String fieldName = parser.currentName(); + String fieldName = parser.getCurrentName(); switch (fieldName) { case "features": parseFeatures(parser, JsonParser::getText, featureHandler); @@ -81,7 +82,7 @@ public class PredicateQueryParser { long subqueryBitmap = SubqueryBitmap.DEFAULT_VALUE; // Specifying subquery bitmap is optional. while (parser.nextToken() != JsonToken.END_OBJECT) { - String fieldName = parser.currentName(); + String fieldName = parser.getCurrentName(); skipToken(parser, JsonToken.VALUE_STRING, JsonToken.VALUE_NUMBER_INT); switch (fieldName) { case "k": @@ -99,11 +100,11 @@ public class PredicateQueryParser { } if (key == null) { throw new IllegalArgumentException( - String.format("Feature key is missing! (%s)", parser.currentLocation())); + String.format("Feature key is missing! (%s)", parser.getCurrentLocation())); } if (value == null) { throw new IllegalArgumentException( - String.format("Feature value is missing! (%s)", parser.currentLocation())); + String.format("Feature value is missing! (%s)", parser.getCurrentLocation())); } featureHandler.accept(key, value, subqueryBitmap); } @@ -113,7 +114,7 @@ public class PredicateQueryParser { if (Arrays.stream(expected).noneMatch(e -> e.equals(actual))) { throw new IllegalArgumentException( String.format("Expected a token in %s, got %s (%s).", - Arrays.toString(expected), actual, parser.currentTokenLocation())); + Arrays.toString(expected), actual, parser.getTokenLocation())); } } diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_allocator.cpp b/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_allocator.cpp index 0c986422be6..758d1336399 100644 --- a/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_allocator.cpp +++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_allocator.cpp @@ -210,7 +210,8 @@ private: } FlowStats calculate_flow_stats(uint32_t docid_limit) const override { double rel_est = abs_to_rel_est(_activeLids.size(), docid_limit); - return {rel_est, bitvector_cost(), bitvector_strict_cost(rel_est)}; + double do_not_make_me_strict = 1000.0; + return {rel_est, bitvector_cost(), do_not_make_me_strict * bitvector_strict_cost(rel_est)}; } SearchIterator::UP createLeafSearch(const TermFieldMatchDataArray &tfmda) const override diff --git a/searchlib/src/vespa/searchlib/attribute/attribute_blueprint_factory.cpp b/searchlib/src/vespa/searchlib/attribute/attribute_blueprint_factory.cpp index 5b17b491a20..70b86bf22a1 100644 --- a/searchlib/src/vespa/searchlib/attribute/attribute_blueprint_factory.cpp +++ b/searchlib/src/vespa/searchlib/attribute/attribute_blueprint_factory.cpp @@ -94,6 +94,7 @@ using search::queryeval::StrictHeapOrSearch; using search::queryeval::WeightedSetTermBlueprint; using search::queryeval::flow::btree_cost; using search::queryeval::flow::btree_strict_cost; +using search::queryeval::flow::estimate_when_unknown; using search::queryeval::flow::get_num_indirections; using search::queryeval::flow::lookup_cost; using search::queryeval::flow::lookup_strict_cost; @@ -150,10 +151,9 @@ public: search::queryeval::FlowStats calculate_flow_stats(uint32_t docid_limit) const override { if (_hit_estimate.is_unknown()) { // E.g. attributes without fast-search are not able to provide a hit estimate. - // In this case we just assume matching half of the document corpus. // In addition, matching is lookup based, and we are not able to skip documents efficiently when being strict. size_t indirections = get_num_indirections(_attr.getBasicType(), _attr.getCollectionType()); - return {0.5, lookup_cost(indirections), lookup_strict_cost(indirections)}; + return {estimate_when_unknown(), lookup_cost(indirections), lookup_strict_cost(indirections)}; } else { double rel_est = abs_to_rel_est(_hit_estimate.est_hits(), docid_limit); return {rel_est, btree_cost(rel_est), btree_strict_cost(rel_est)}; diff --git a/searchlib/src/vespa/searchlib/queryeval/blueprint.cpp b/searchlib/src/vespa/searchlib/queryeval/blueprint.cpp index 7334db4b716..cfa165be067 100644 --- a/searchlib/src/vespa/searchlib/queryeval/blueprint.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/blueprint.cpp @@ -1,14 +1,15 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "blueprint.h" -#include "leaf_blueprints.h" +#include "andnotsearch.h" +#include "andsearch.h" #include "emptysearch.h" -#include "full_search.h" #include "field_spec.hpp" -#include "andsearch.h" -#include "orsearch.h" -#include "andnotsearch.h" +#include "flow_tuning.h" +#include "full_search.h" +#include "leaf_blueprints.h" #include "matching_elements_search.h" +#include "orsearch.h" #include <vespa/searchlib/fef/termfieldmatchdataarray.h> #include <vespa/vespalib/objects/visit.hpp> #include <vespa/vespalib/objects/objectdumper.h> @@ -238,7 +239,7 @@ Blueprint::default_flow_stats(uint32_t docid_limit, uint32_t abs_est, size_t chi FlowStats Blueprint::default_flow_stats(size_t child_cnt) { - return {0.5, 1.0 + child_cnt, 1.0 + child_cnt}; + return {flow::estimate_when_unknown(), 1.0 + child_cnt, 1.0 + child_cnt}; } std::unique_ptr<MatchingElementsSearch> diff --git a/searchlib/src/vespa/searchlib/queryeval/flow_tuning.h b/searchlib/src/vespa/searchlib/queryeval/flow_tuning.h index 22faa920bc0..5ed61ef9fc8 100644 --- a/searchlib/src/vespa/searchlib/queryeval/flow_tuning.h +++ b/searchlib/src/vespa/searchlib/queryeval/flow_tuning.h @@ -60,6 +60,12 @@ inline size_t get_num_indirections(const attribute::BasicType& basic_type, return res; } +// Some blueprints are not able to provide a hit estimate (e.g. attributes without fast-search). +// In such cases the following estimate is used instead. In most cases this is an overestimate. +inline double estimate_when_unknown() { + return 0.1; +} + // Non-strict cost of lookup based matching in an attribute (not fast-search). // Test used: IteratorBenchmark::analyze_term_search_in_attributes_non_strict inline double lookup_cost(size_t num_indirections) { diff --git a/vespa-feed-client-api/src/main/java/ai/vespa/feed/client/JsonFeeder.java b/vespa-feed-client-api/src/main/java/ai/vespa/feed/client/JsonFeeder.java index 3111815b332..11fb6526210 100644 --- a/vespa-feed-client-api/src/main/java/ai/vespa/feed/client/JsonFeeder.java +++ b/vespa-feed-client-api/src/main/java/ai/vespa/feed/client/JsonFeeder.java @@ -414,7 +414,7 @@ public class JsonFeeder implements Closeable { abstract String getDocumentJson(long start, long end); OperationParseException parseException(String error) { - JsonLocation location = parser.currentLocation(); + JsonLocation location = parser.getTokenLocation(); return new OperationParseException(error + " at offset " + location.getByteOffset() + " (line " + location.getLineNr() + ", column " + location.getColumnNr() + ")"); } @@ -444,13 +444,13 @@ public class JsonFeeder implements Closeable { case "create": parameters = parameters.createIfNonExistent(readBoolean()); break; case "fields": { expect(START_OBJECT); - start = parser.currentTokenLocation().getByteOffset(); + start = parser.getTokenLocation().getByteOffset(); int depth = 1; while (depth > 0) switch (parser.nextToken()) { case START_OBJECT: ++depth; break; case END_OBJECT: --depth; break; } - end = parser.currentTokenLocation().getByteOffset() + 1; + end = parser.getTokenLocation().getByteOffset() + 1; break; } default: throw parseException("Unexpected field name '" + parser.getText() + "'"); @@ -470,7 +470,7 @@ public class JsonFeeder implements Closeable { if (end >= start) throw parseException("Illegal 'fields' object for remove operation"); else - start = end = parser.currentTokenLocation().getByteOffset(); // getDocumentJson advances buffer overwrite head. + start = end = parser.getTokenLocation().getByteOffset(); // getDocumentJson advances buffer overwrite head. } else if (end < start) throw parseException("No 'fields' object for document"); @@ -486,14 +486,14 @@ public class JsonFeeder implements Closeable { private void expect(JsonToken token) throws IOException { if (parser.nextToken() != token) - throw new OperationParseException("Expected '" + token + "' at offset " + parser.currentTokenLocation().getByteOffset() + + throw new OperationParseException("Expected '" + token + "' at offset " + parser.getTokenLocation().getByteOffset() + ", but found '" + parser.currentToken() + "' (" + parser.getText() + ")"); } private String readString() throws IOException { String value = parser.nextTextValue(); if (value == null) - throw new OperationParseException("Expected '" + JsonToken.VALUE_STRING + "' at offset " + parser.currentTokenLocation().getByteOffset() + + throw new OperationParseException("Expected '" + JsonToken.VALUE_STRING + "' at offset " + parser.getTokenLocation().getByteOffset() + ", but found '" + parser.currentToken() + "' (" + parser.getText() + ")"); return value; @@ -502,7 +502,7 @@ public class JsonFeeder implements Closeable { private boolean readBoolean() throws IOException { Boolean value = parser.nextBooleanValue(); if (value == null) - throw new OperationParseException("Expected '" + JsonToken.VALUE_FALSE + "' or '" + JsonToken.VALUE_TRUE + "' at offset " + parser.currentTokenLocation().getByteOffset() + + throw new OperationParseException("Expected '" + JsonToken.VALUE_FALSE + "' or '" + JsonToken.VALUE_TRUE + "' at offset " + parser.getTokenLocation().getByteOffset() + ", but found '" + parser.currentToken() + "' (" + parser.getText() + ")"); return value; diff --git a/vespa-feed-client/src/main/java/ai/vespa/feed/client/impl/HttpFeedClient.java b/vespa-feed-client/src/main/java/ai/vespa/feed/client/impl/HttpFeedClient.java index 9dd11113c0b..a30cfd5ec39 100644 --- a/vespa-feed-client/src/main/java/ai/vespa/feed/client/impl/HttpFeedClient.java +++ b/vespa-feed-client/src/main/java/ai/vespa/feed/client/impl/HttpFeedClient.java @@ -219,13 +219,13 @@ class HttpFeedClient implements FeedClient { throw new ResultParseException(documentId, "Expected 'trace' to be an array, but got '" + parser.currentToken() + "' in: " + new String(json, UTF_8)); - int start = (int) parser.currentTokenLocation().getByteOffset(); + int start = (int) parser.getTokenLocation().getByteOffset(); int depth = 1; while (depth > 0) switch (parser.nextToken()) { case START_ARRAY: ++depth; break; case END_ARRAY: --depth; break; } - int end = (int) parser.currentTokenLocation().getByteOffset() + 1; + int end = (int) parser.getTokenLocation().getByteOffset() + 1; trace = new String(json, start, end - start, UTF_8); break; default: diff --git a/vespajlib/src/test/java/com/yahoo/slime/JsonBenchmark.java b/vespajlib/src/test/java/com/yahoo/slime/JsonBenchmark.java index cccc9667e11..ee755a44010 100644 --- a/vespajlib/src/test/java/com/yahoo/slime/JsonBenchmark.java +++ b/vespajlib/src/test/java/com/yahoo/slime/JsonBenchmark.java @@ -43,7 +43,7 @@ public class JsonBenchmark { try (JsonParser jsonParser = jsonFactory.createParser(json)) { JsonToken array = jsonParser.nextToken(); for (JsonToken token = jsonParser.nextToken(); !JsonToken.END_ARRAY.equals(token); token = jsonParser.nextToken()) { - if (JsonToken.FIELD_NAME.equals(token) && "weight".equals(jsonParser.currentName())) { + if (JsonToken.FIELD_NAME.equals(token) && "weight".equals(jsonParser.getCurrentName())) { token = jsonParser.nextToken(); count += jsonParser.getLongValue(); } |