summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--config-model-api/abi-spec.json1
-rw-r--r--config-model-api/src/main/java/com/yahoo/config/application/api/ValidationOverrides.java39
-rw-r--r--config-model-api/src/test/java/com/yahoo/config/application/api/ValidationOverrideTest.java3
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/Index.java22
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/Search.java3
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/document/SDField.java6
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/document/Stemming.java11
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java1
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/application/validation/ValidationOverridesValidator.java29
-rw-r--r--config-model/src/test/java/com/yahoo/vespa/model/application/validation/ValidationOverridesValidatorTest.java92
-rw-r--r--container-search/src/main/java/com/yahoo/prelude/query/TermItem.java30
-rw-r--r--documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusAsyncSession.java16
-rw-r--r--indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/linguistics/LinguisticsAnnotator.java21
-rw-r--r--messagebus/src/main/java/com/yahoo/messagebus/Sequencer.java1
-rw-r--r--messagebus/src/main/java/com/yahoo/messagebus/SourceSession.java7
15 files changed, 190 insertions, 92 deletions
diff --git a/config-model-api/abi-spec.json b/config-model-api/abi-spec.json
index 735778f4d46..ba5f994ecd6 100644
--- a/config-model-api/abi-spec.json
+++ b/config-model-api/abi-spec.json
@@ -594,6 +594,7 @@
"public void invalid(com.yahoo.config.application.api.ValidationId, java.lang.String, java.time.Instant)",
"public boolean allows(java.lang.String, java.time.Instant)",
"public boolean allows(com.yahoo.config.application.api.ValidationId, java.time.Instant)",
+ "public boolean validate(java.time.Instant)",
"public java.lang.String xmlForm()",
"public static java.lang.String toAllowMessage(com.yahoo.config.application.api.ValidationId)",
"public static com.yahoo.config.application.api.ValidationOverrides fromXml(java.io.Reader)",
diff --git a/config-model-api/src/main/java/com/yahoo/config/application/api/ValidationOverrides.java b/config-model-api/src/main/java/com/yahoo/config/application/api/ValidationOverrides.java
index 3221df38d4f..42a47781251 100644
--- a/config-model-api/src/main/java/com/yahoo/config/application/api/ValidationOverrides.java
+++ b/config-model-api/src/main/java/com/yahoo/config/application/api/ValidationOverrides.java
@@ -74,14 +74,22 @@ public class ValidationOverrides {
/** Returns whether the given (assumed invalid) change is allowed by this at the moment */
public boolean allows(ValidationId validationId, Instant now) {
for (Allow override : overrides) {
- if (now.plus(Duration.ofDays(30)).isBefore(override.until))
- throw new IllegalArgumentException(override + " is too far in the future: Max 30 days is allowed");
if (override.allows(validationId, now))
return true;
}
return false;
}
+ /** Validates overrides (checks 'until' date') */
+ public boolean validate(Instant now) {
+ for (Allow override : overrides) {
+ if (now.plus(Duration.ofDays(30)).isBefore(override.until))
+ throw new IllegalArgumentException("validation-overrides is invalid: " + override +
+ " is too far in the future: Max 30 days is allowed");
+ }
+ return false;
+ }
+
/** Returns the XML form of this, or null if it was not created by fromXml, nor is empty */
public String xmlForm() { return xmlForm; }
@@ -116,23 +124,18 @@ public class ValidationOverrides {
public static ValidationOverrides fromXml(String xmlForm) {
if ( xmlForm.isEmpty()) return ValidationOverrides.empty;
- try {
- // Assume valid structure is ensured by schema validation
- Element root = XML.getDocument(xmlForm).getDocumentElement();
- List<ValidationOverrides.Allow> overrides = new ArrayList<>();
- for (Element allow : XML.getChildren(root, "allow")) {
- Instant until = LocalDate.parse(allow.getAttribute("until"), DateTimeFormatter.ISO_DATE)
- .atStartOfDay().atZone(ZoneOffset.UTC).toInstant()
- .plus(Duration.ofDays(1)); // Make the override valid *on* the "until" date
- Optional<ValidationId> validationId = ValidationId.from(XML.getValue(allow));
- // skip unknown ids as they may be valid for other model versions
- validationId.ifPresent(id -> overrides.add(new Allow(id, until)));
- }
- return new ValidationOverrides(overrides, xmlForm);
- }
- catch (IllegalArgumentException e) {
- throw new IllegalArgumentException("validation-overrides is invalid", e);
+ // Assume valid structure is ensured by schema validation
+ Element root = XML.getDocument(xmlForm).getDocumentElement();
+ List<ValidationOverrides.Allow> overrides = new ArrayList<>();
+ for (Element allow : XML.getChildren(root, "allow")) {
+ Instant until = LocalDate.parse(allow.getAttribute("until"), DateTimeFormatter.ISO_DATE)
+ .atStartOfDay().atZone(ZoneOffset.UTC).toInstant()
+ .plus(Duration.ofDays(1)); // Make the override valid *on* the "until" date
+ Optional<ValidationId> validationId = ValidationId.from(XML.getValue(allow));
+ // skip unknown ids as they may be valid for other model versions
+ validationId.ifPresent(id -> overrides.add(new Allow(id, until)));
}
+ return new ValidationOverrides(overrides, xmlForm);
}
/** A validation override which allows a particular change. Immutable. */
diff --git a/config-model-api/src/test/java/com/yahoo/config/application/api/ValidationOverrideTest.java b/config-model-api/src/test/java/com/yahoo/config/application/api/ValidationOverrideTest.java
index f7ef059c5f2..81adef1cfea 100644
--- a/config-model-api/src/test/java/com/yahoo/config/application/api/ValidationOverrideTest.java
+++ b/config-model-api/src/test/java/com/yahoo/config/application/api/ValidationOverrideTest.java
@@ -61,10 +61,11 @@ public class ValidationOverrideTest {
ValidationOverrides overrides = ValidationOverrides.fromXml(new StringReader(validationOverrides));
Instant now = ManualClock.at("2000-01-01T23:59:00");
overrides.allows("indexing-change", now);
+ overrides.validate(now);
Assert.fail("Expected validation interval override validation validation failure");
}
catch (IllegalArgumentException e) {
- Assert.assertEquals("allow 'indexing-change' until 2000-02-03T00:00:00Z is too far in the future: Max 30 days is allowed",
+ Assert.assertEquals("validation-overrides is invalid: allow 'indexing-change' until 2000-02-03T00:00:00Z is too far in the future: Max 30 days is allowed",
e.getMessage());
}
}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/Index.java b/config-model/src/main/java/com/yahoo/searchdefinition/Index.java
index 577639ead7a..212b985f7f1 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/Index.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/Index.java
@@ -53,9 +53,6 @@ public class Index implements Cloneable, Serializable {
*/
private Stemming stemming = null;
- /** Whether the content of this index is normalized */
- private boolean normalized = true;
-
private Type type = Type.VESPA;
/** The boolean index definition, if set */
@@ -126,20 +123,19 @@ public class Index implements Cloneable, Serializable {
if (o == null || getClass() != o.getClass()) return false;
Index index = (Index) o;
return prefix == index.prefix &&
- normalized == index.normalized &&
- interleavedFeatures == index.interleavedFeatures &&
- Objects.equals(name, index.name) &&
- rankType == index.rankType &&
- Objects.equals(aliases, index.aliases) &&
- stemming == index.stemming &&
- type == index.type &&
- Objects.equals(boolIndex, index.boolIndex) &&
- Objects.equals(hnswIndexParams, index.hnswIndexParams);
+ interleavedFeatures == index.interleavedFeatures &&
+ Objects.equals(name, index.name) &&
+ rankType == index.rankType &&
+ Objects.equals(aliases, index.aliases) &&
+ stemming == index.stemming &&
+ type == index.type &&
+ Objects.equals(boolIndex, index.boolIndex) &&
+ Objects.equals(hnswIndexParams, index.hnswIndexParams);
}
@Override
public int hashCode() {
- return Objects.hash(name, rankType, prefix, aliases, stemming, normalized, type, boolIndex, hnswIndexParams, interleavedFeatures);
+ return Objects.hash(name, rankType, prefix, aliases, stemming, type, boolIndex, hnswIndexParams, interleavedFeatures);
}
public String toString() {
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/Search.java b/config-model/src/main/java/com/yahoo/searchdefinition/Search.java
index 9ce1b8bb330..c89f6f09b77 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/Search.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/Search.java
@@ -23,7 +23,6 @@ import com.yahoo.vespa.model.AbstractService;
import java.io.Reader;
import java.util.ArrayList;
-import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.Iterator;
@@ -48,7 +47,7 @@ import java.util.stream.Stream;
public class Search implements ImmutableSearch {
private static final String SD_DOC_FIELD_NAME = "sddocname";
- private static final List<String> RESERVED_NAMES = Arrays.asList(
+ private static final List<String> RESERVED_NAMES = List.of(
"index", "index_url", "summary", "attribute", "select_input", "host", SummaryClass.DOCUMENT_ID_FIELD,
"position", "split_foreach", "tokenize", "if", "else", "switch", "case", SD_DOC_FIELD_NAME, "relevancy");
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/SDField.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/SDField.java
index cf7bef654b0..ac92207820c 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/document/SDField.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/SDField.java
@@ -482,7 +482,7 @@ public class SDField extends Field implements TypedKey, FieldOperationContainer,
public void setDataType(DataType type) {
if (type.equals(DataType.URI)) { // Different defaults, naturally
normalizing.inferLowercase();
- stemming=Stemming.NONE;
+ stemming = Stemming.NONE;
}
this.dataType = type;
if ( ! idOverride) {
@@ -672,7 +672,7 @@ public class SDField extends Field implements TypedKey, FieldOperationContainer,
*/
@Override
public Stemming getStemming(Search search) {
- if (stemming!=null)
+ if (stemming != null)
return stemming;
else
return search.getStemming();
@@ -687,7 +687,7 @@ public class SDField extends Field implements TypedKey, FieldOperationContainer,
* Sets how this field should be stemmed, or set to null to use the default.
*/
public void setStemming(Stemming stemming) {
- this.stemming=stemming;
+ this.stemming = stemming;
}
/** Returns an unmodifiable map of the summary fields defined in this */
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/Stemming.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/Stemming.java
index 8a7ee78ea91..cefad472df7 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/document/Stemming.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/Stemming.java
@@ -3,12 +3,10 @@ package com.yahoo.searchdefinition.document;
import com.yahoo.language.process.StemMode;
-import java.util.logging.Logger;
-
/**
- * <p>The stemming setting of a field. This describes how the search engine
+ * The stemming setting of a field. This describes how the search engine
* should transform content of this field into base forms (stems) to increase
- * recall (find "car" when you search for "cars" etc.).</p>
+ * recall (find "car" when you search for "cars" etc.).
*
* @author bratseth
*/
@@ -26,8 +24,6 @@ public enum Stemming {
/** index multiple stems */
MULTIPLE("multiple");
- private static Logger log=Logger.getLogger(Stemming.class.getName());
-
private final String name;
/**
@@ -36,7 +32,6 @@ public enum Stemming {
*
* @throws IllegalArgumentException if there is no stemming type with the given name
*/
- @SuppressWarnings("deprecation")
public static Stemming get(String stemmingName) {
try {
return Stemming.valueOf(stemmingName.toUpperCase());
@@ -51,11 +46,11 @@ public enum Stemming {
public String getName() { return name; }
+ @Override
public String toString() {
return "stemming " + name;
}
- @SuppressWarnings("deprecation")
public StemMode toStemMode() {
switch(this) {
case SHORTEST: return StemMode.SHORTEST;
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java
index 84c7a48a998..d59ad5f6252 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java
@@ -70,6 +70,7 @@ public class Validation {
new RankSetupValidator(validationParameters.ignoreValidationErrors()).validate(model, deployState);
new NoPrefixForIndexes().validate(model, deployState);
new DeploymentSpecValidator().validate(model, deployState);
+ new ValidationOverridesValidator().validate(model, deployState);
new RankingConstantsValidator().validate(model, deployState);
new SecretStoreValidator().validate(model, deployState);
new EndpointCertificateSecretsValidator().validate(model, deployState);
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ValidationOverridesValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ValidationOverridesValidator.java
new file mode 100644
index 00000000000..955d7924f86
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ValidationOverridesValidator.java
@@ -0,0 +1,29 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.application.validation;
+
+import com.yahoo.config.application.api.ValidationOverrides;
+import com.yahoo.config.model.deploy.DeployState;
+import com.yahoo.vespa.model.VespaModel;
+
+import java.io.Reader;
+import java.time.Duration;
+import java.util.Optional;
+
+/**
+ * Validate validation overrides (validation-overrides.xml). Done as a validator to make sure this is
+ * done when validating the mode and not when building the model
+ *
+ * @author hmusum
+ */
+public class ValidationOverridesValidator extends Validator {
+
+ @Override
+ public void validate(VespaModel model, DeployState deployState) {
+ Optional<Reader> overrides = deployState.getApplicationPackage().getValidationOverrides();
+ if (overrides.isEmpty()) return;
+
+ ValidationOverrides validationOverrides = ValidationOverrides.fromXml(overrides.get());
+ validationOverrides.validate(deployState.now());
+ }
+
+}
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ValidationOverridesValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ValidationOverridesValidatorTest.java
new file mode 100644
index 00000000000..90d3d80c032
--- /dev/null
+++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ValidationOverridesValidatorTest.java
@@ -0,0 +1,92 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.application.validation;
+
+import com.yahoo.config.model.NullConfigModelRegistry;
+import com.yahoo.config.model.api.ValidationParameters;
+import com.yahoo.config.model.deploy.DeployState;
+import com.yahoo.config.model.test.MockApplicationPackage;
+import com.yahoo.vespa.model.VespaModel;
+import org.junit.Test;
+import org.xml.sax.SAXException;
+
+import java.io.IOException;
+import java.time.Duration;
+import java.time.Instant;
+import java.time.ZoneId;
+import java.time.format.DateTimeFormatter;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+/**
+ * @author hmusum
+ */
+public class ValidationOverridesValidatorTest {
+
+ private static final DateTimeFormatter dateTimeFormatter = DateTimeFormatter.ofPattern("yyyy-MM-dd")
+ .withZone(ZoneId.systemDefault());
+
+ @Test
+ public void testValidationOverride() throws IOException, SAXException {
+ String tenDays = dateTimeFormatter.format(Instant.now().plus(Duration.ofDays(10)));
+
+ var validationOverridesXml = "<?xml version='1.0' encoding='UTF-8'?>\n" +
+ " <validation-overrides>\n" +
+ " <allow until='" + tenDays + "'>deployment-removal</allow>\n" +
+ " </validation-overrides>";
+
+ var deployState = createDeployState(validationOverridesXml);
+ VespaModel model = new VespaModel(new NullConfigModelRegistry(), deployState);
+ Validation.validate(model, new ValidationParameters(), deployState);
+ }
+
+ @Test
+ public void testFailsWhenValidationOverrideIsTooFarInFuture() {
+ Instant now = Instant.now();
+ String sixtyDays = dateTimeFormatter.format(now.plus(Duration.ofDays(60)));
+ String sixtyOneDays = dateTimeFormatter.format(now.plus(Duration.ofDays(61)));
+
+ var validationOverrides = "<?xml version='1.0' encoding='UTF-8'?>\n" +
+ "<validation-overrides>\n" +
+ " <allow until='" + sixtyDays + "'>deployment-removal</allow>\n" +
+ "</validation-overrides>";
+ assertValidationError("validation-overrides is invalid: allow 'deployment-removal' until " +
+ sixtyOneDays + "T00:00:00Z is too far in the future: Max 30 days is allowed", validationOverrides);
+ }
+
+ private static void assertValidationError(String message, String validationOverridesXml) {
+ try {
+ var deployState = createDeployState(validationOverridesXml);
+ VespaModel model = new VespaModel(new NullConfigModelRegistry(), deployState);
+ Validation.validate(model, new ValidationParameters(), deployState);
+ fail("Did not get expected exception");
+ } catch (IllegalArgumentException e) {
+ assertEquals(message, e.getMessage());
+ } catch (SAXException|IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ private static DeployState createDeployState(String validationOverridesXml) {
+ var services = "<services version='1.0'>" +
+ " <admin version='2.0'>" +
+ " <adminserver hostalias='node0' />" +
+ " </admin>" +
+ " <container id='default' version='1.0'>" +
+ " <search/>" +
+ " <nodes>" +
+ " <node hostalias='node0'/>" +
+ " </nodes>" +
+ " </container>" +
+ "</services>";
+
+ var app = new MockApplicationPackage.Builder()
+ .withValidationOverrides(validationOverridesXml)
+ .withServices(services)
+ .build();
+ var builder = new DeployState.Builder().applicationPackage(app);
+ return builder.build();
+ }
+
+}
+
diff --git a/container-search/src/main/java/com/yahoo/prelude/query/TermItem.java b/container-search/src/main/java/com/yahoo/prelude/query/TermItem.java
index 2c33e7a2630..5df0cd120b2 100644
--- a/container-search/src/main/java/com/yahoo/prelude/query/TermItem.java
+++ b/container-search/src/main/java/com/yahoo/prelude/query/TermItem.java
@@ -97,37 +97,19 @@ public abstract class TermItem extends SimpleIndexedItem implements BlockItem {
@Override
public int getTermCount() { return 1; }
- /**
- * This refers to whether accent removal is a meaningful and possible
- * operation for this word. It should be named "isTransformable" or similar,
- * but for historical reasons that is not the case. This method has nothing
- * to do with Unicode normalization.
- *
- * @return true if accent removal can/should be performed
- */
- public boolean isNormalizable() {
- return normalizable;
- }
+ /** Returns whether accent removal is a meaningful and possible operation for this word. */
+ public boolean isNormalizable() { return normalizable; }
/**
- * This refers to whether accent removal is a meaningful and possible
- * operation for this word. It should be named "isTransformable" or similar,
- * but for historical reasons that is not the case. This method has nothing
- * to do with Unicode normalization.
+ * Sets whether accent removal is a meaningful and possible operation for this word.
*
* @param normalizable set to true if accent removal can/should be performed
*/
- public void setNormalizable(boolean normalizable) {
- this.normalizable = normalizable;
- }
+ public void setNormalizable(boolean normalizable) { this.normalizable = normalizable; }
@Override
- public SegmentingRule getSegmentingRule() {
- return segmentingRule;
- }
+ public SegmentingRule getSegmentingRule() { return segmentingRule; }
- public void setSegmentingRule(SegmentingRule segmentingRule) {
- this.segmentingRule = segmentingRule;
- }
+ public void setSegmentingRule(SegmentingRule segmentingRule) { this.segmentingRule = segmentingRule; }
}
diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusAsyncSession.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusAsyncSession.java
index 1da6f8bb472..488c2040976 100644
--- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusAsyncSession.java
+++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusAsyncSession.java
@@ -67,16 +67,16 @@ public class MessageBusAsyncSession implements MessageBusSession, AsyncSession {
private final BlockingQueue<Response> responses = new LinkedBlockingQueue<>();
private final ThrottlePolicy throttlePolicy;
private final SourceSession session;
+ private final String routeForGet;
private String route;
- private String routeForGet;
private int traceLevel;
/**
* Creates a new async session running on message bus logic.
*
- * @param asyncParams Common asyncsession parameters, not used.
- * @param bus The message bus on which to run.
- * @param mbusParams Parameters concerning message bus configuration.
+ * @param asyncParams common asyncsession parameters, not used
+ * @param bus the message bus on which to run
+ * @param mbusParams parameters concerning message bus configuration
*/
MessageBusAsyncSession(AsyncParameters asyncParams, MessageBus bus, MessageBusParams mbusParams) {
this(asyncParams, bus, mbusParams, null);
@@ -85,10 +85,10 @@ public class MessageBusAsyncSession implements MessageBusSession, AsyncSession {
/**
* Creates a new async session running on message bus logic with a specified reply handler.
*
- * @param asyncParams Common asyncsession parameters, not used.
- * @param bus The message bus on which to run.
- * @param mbusParams Parameters concerning message bus configuration.
- * @param handler The external reply handler.
+ * @param asyncParams common asyncsession parameters, not used
+ * @param bus the message bus on which to run
+ * @param mbusParams parameters concerning message bus configuration
+ * @param handler the external reply handler
*/
MessageBusAsyncSession(AsyncParameters asyncParams, MessageBus bus, MessageBusParams mbusParams,
ReplyHandler handler) {
diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/linguistics/LinguisticsAnnotator.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/linguistics/LinguisticsAnnotator.java
index 81a5305a778..f6d2be796dc 100644
--- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/linguistics/LinguisticsAnnotator.java
+++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/linguistics/LinguisticsAnnotator.java
@@ -29,6 +29,7 @@ public class LinguisticsAnnotator {
private final AnnotatorConfig config;
private static class TermOccurrences {
+
final Map<String, Integer> termOccurrences = new HashMap<>();
final int maxOccurrences;
@@ -38,14 +39,13 @@ public class LinguisticsAnnotator {
boolean termCountBelowLimit(String term) {
String lowerCasedTerm = toLowerCase(term);
- int occurences = termOccurrences.getOrDefault(lowerCasedTerm, 0);
- if (occurences >= maxOccurrences) {
- return false;
- }
+ int occurrences = termOccurrences.getOrDefault(lowerCasedTerm, 0);
+ if (occurrences >= maxOccurrences) return false;
- termOccurrences.put(lowerCasedTerm, occurences + 1);
+ termOccurrences.put(lowerCasedTerm, occurrences + 1);
return true;
}
+
}
/**
@@ -69,16 +69,15 @@ public class LinguisticsAnnotator {
if (text.getSpanTree(SpanTrees.LINGUISTICS) != null) return true; // Already annotated with LINGUISTICS.
Tokenizer tokenizer = factory.getTokenizer();
- String input = (text.getString().length() <= config.getMaxTokenizeLength())
+ String input = (text.getString().length() <= config.getMaxTokenizeLength())
? text.getString()
: text.getString().substring(0, config.getMaxTokenizeLength());
Iterable<Token> tokens = tokenizer.tokenize(input, config.getLanguage(), config.getStemMode(),
config.getRemoveAccents());
TermOccurrences termOccurrences = new TermOccurrences(config.getMaxTermOccurrences());
SpanTree tree = new SpanTree(SpanTrees.LINGUISTICS);
- for (Token token : tokens) {
- addAnnotationSpan(text.getString(), tree.spanList(), tokenizer, token, config.getStemMode(), termOccurrences);
- }
+ for (Token token : tokens)
+ addAnnotationSpan(text.getString(), tree.spanList(), token, config.getStemMode(), termOccurrences);
if (tree.numAnnotations() == 0) return false;
text.setSpanTree(tree);
@@ -107,11 +106,11 @@ public class LinguisticsAnnotator {
}
}
- private static void addAnnotationSpan(String input, SpanList parent, Tokenizer tokenizer, Token token, StemMode mode, TermOccurrences termOccurrences) {
+ private static void addAnnotationSpan(String input, SpanList parent, Token token, StemMode mode, TermOccurrences termOccurrences) {
if ( ! token.isSpecialToken()) {
if (token.getNumComponents() > 0) {
for (int i = 0; i < token.getNumComponents(); ++i) {
- addAnnotationSpan(input, parent, tokenizer, token.getComponent(i), mode, termOccurrences);
+ addAnnotationSpan(input, parent, token.getComponent(i), mode, termOccurrences);
}
return;
}
diff --git a/messagebus/src/main/java/com/yahoo/messagebus/Sequencer.java b/messagebus/src/main/java/com/yahoo/messagebus/Sequencer.java
index 39fc9aa8314..fcef11b2e74 100644
--- a/messagebus/src/main/java/com/yahoo/messagebus/Sequencer.java
+++ b/messagebus/src/main/java/com/yahoo/messagebus/Sequencer.java
@@ -152,4 +152,5 @@ public class Sequencer implements MessageHandler, ReplyHandler {
ReplyHandler handler = reply.popHandler();
handler.handleReply(reply);
}
+
}
diff --git a/messagebus/src/main/java/com/yahoo/messagebus/SourceSession.java b/messagebus/src/main/java/com/yahoo/messagebus/SourceSession.java
index 204bf2f7906..248cbdca7a0 100644
--- a/messagebus/src/main/java/com/yahoo/messagebus/SourceSession.java
+++ b/messagebus/src/main/java/com/yahoo/messagebus/SourceSession.java
@@ -137,8 +137,7 @@ public final class SourceSession implements ReplyHandler, MessageBus.SendBlocked
private Result sendInternal(Message message) {
synchronized (lock) {
if (closed) {
- return new Result(ErrorCode.SEND_QUEUE_CLOSED,
- "Source session is closed.");
+ return new Result(ErrorCode.SEND_QUEUE_CLOSED, "Source session is closed.");
}
if (throttlePolicy != null && ! throttlePolicy.canSend(message, pendingCount)) {
return new Result(ErrorCode.SEND_QUEUE_FULL,
@@ -152,8 +151,8 @@ public final class SourceSession implements ReplyHandler, MessageBus.SendBlocked
}
if (message.getTrace().shouldTrace(TraceLevel.COMPONENT)) {
message.getTrace().trace(TraceLevel.COMPONENT,
- "Source session accepted a " + message.getApproxSize() + " byte message. " +
- pendingCount + " message(s) now pending.");
+ "Source session accepted a " + message.getApproxSize() + " byte message. " +
+ pendingCount + " message(s) now pending.");
}
message.pushHandler(this);
sequencer.handleMessage(message);