diff options
Diffstat (limited to 'container-search/src/main')
36 files changed, 349 insertions, 338 deletions
diff --git a/container-search/src/main/java/com/yahoo/prelude/Index.java b/container-search/src/main/java/com/yahoo/prelude/Index.java index 8915c4b42f0..306c7c80577 100644 --- a/container-search/src/main/java/com/yahoo/prelude/Index.java +++ b/container-search/src/main/java/com/yahoo/prelude/Index.java @@ -23,31 +23,11 @@ import java.util.Set; */ public class Index { - public static class Attribute { - - private boolean tokenizedContent = false; - public final String name; - - public Attribute(String name) { - this.name = name; - } - - public boolean isTokenizedContent() { - return tokenizedContent; - } - - public void setTokenizedContent(boolean tokenizedContent) { - this.tokenizedContent = tokenizedContent; - } - } - /** The null index - don't use this for name lookups */ public static final Index nullIndex = new Index("(null)"); private final String name; - private String type; // TODO: Parse to a type object; do not expose this as a string - private final List<String> aliases = new ArrayList<>(); // The state resulting from adding commands to this (using addCommand) diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastSearcher.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastSearcher.java index 6f360c8b456..33ebebd8d49 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastSearcher.java @@ -81,7 +81,7 @@ public class FastSearcher extends VespaBackEndSearcher { @Override public Result doSearch2(Query query, Execution execution) { - if (dispatcher.searchCluster().groupSize() == 1) + if (dispatcher.searchCluster().wantedGroupSize() == 1) forceSinglePassGrouping(query); try (SearchInvoker invoker = getSearchInvoker(query)) { Result result = invoker.search(query, execution); diff --git a/container-search/src/main/java/com/yahoo/prelude/query/Item.java b/container-search/src/main/java/com/yahoo/prelude/query/Item.java index c4978b2a378..467fbd3127c 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/Item.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/Item.java @@ -65,7 +65,7 @@ public abstract class Item implements Cloneable { public final int code; - private ItemType(int code) { + ItemType(int code) { this.code = code; } diff --git a/container-search/src/main/java/com/yahoo/prelude/query/QueryCanonicalizer.java b/container-search/src/main/java/com/yahoo/prelude/query/QueryCanonicalizer.java index 88bae76b26d..2bf20bf7c5a 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/QueryCanonicalizer.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/QueryCanonicalizer.java @@ -94,7 +94,7 @@ public class QueryCanonicalizer { if (composite instanceof RankItem || composite instanceof NotItem) { collapseLevels(composite, composite.getItemIterator()); // collapse the first item only } - else if (composite instanceof AndItem || composite instanceof OrItem) { + else if (composite instanceof AndItem || composite instanceof OrItem || composite instanceof WeakAndItem) { for (ListIterator<Item> i = composite.getItemIterator(); i.hasNext(); ) collapseLevels(composite, i); } @@ -106,10 +106,17 @@ public class QueryCanonicalizer { Item child = i.next(); if (child == null) return; if (child.getClass() != composite.getClass()) return; + if (child instanceof WeakAndItem && !equalWeakAndSettings((WeakAndItem)child, (WeakAndItem)composite)) return; i.remove(); moveChildren((CompositeItem) child, i); } - + + private static boolean equalWeakAndSettings(WeakAndItem a, WeakAndItem b) { + if ( ! a.getIndexName().equals(b.getIndexName())) return false; + if (a.getN() != b.getN()) return false; + return true; + } + private static void moveChildren(CompositeItem from, ListIterator<Item> toIterator) { for (ListIterator<Item> i = from.getItemIterator(); i.hasNext(); ) toIterator.add(i.next()); diff --git a/container-search/src/main/java/com/yahoo/prelude/query/WandItem.java b/container-search/src/main/java/com/yahoo/prelude/query/WandItem.java index 8cce8fb5720..c5679e113f1 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/WandItem.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/WandItem.java @@ -99,13 +99,10 @@ public class WandItem extends WeightedSetItem { protected void appendHeadingString(StringBuilder buffer) { buffer.append(getName()); buffer.append("("); - buffer.append(targetNumHits); - buffer.append(","); - buffer.append(scoreThreshold); - buffer.append(","); + buffer.append(targetNumHits).append(","); + buffer.append(scoreThreshold).append(","); buffer.append(thresholdBoostFactor); - buffer.append(")"); - buffer.append(" "); + buffer.append(") "); } @Override diff --git a/container-search/src/main/java/com/yahoo/prelude/query/WeakAndItem.java b/container-search/src/main/java/com/yahoo/prelude/query/WeakAndItem.java index 4fa2ed8b214..e8817a44133 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/WeakAndItem.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/WeakAndItem.java @@ -18,16 +18,20 @@ import java.nio.ByteBuffer; */ public final class WeakAndItem extends NonReducibleCompositeItem { + /** The default N used if none is specified: 100 */ + public static final int defaultN = 100; + private int n; private String index; private int scoreThreshold = 0; - public ItemType getItemType() { - return ItemType.WEAK_AND; + /** Creates a WAND item with default N */ + public WeakAndItem() { + this(defaultN); } - public String getName() { - return "WAND"; + public WeakAndItem(int N) { + this("", N); } /** @@ -42,50 +46,37 @@ public final class WeakAndItem extends NonReducibleCompositeItem { this.n = n; this.index = (index == null) ? "" : index; } - public WeakAndItem(int N) { - this("", N); - } - /** Sets the index name of all subitems of this */ + @Override + public ItemType getItemType() { return ItemType.WEAK_AND; } + + @Override + public String getName() { return "WEAKAND"; } + + @Override public void setIndexName(String index) { String toSet = (index == null) ? "" : index; super.setIndexName(toSet); this.index = toSet; } - public String getIndexName() { - return index; - } + public String getIndexName() { return index; } /** Appends the heading of this string - <code>[getName()]([limit]) </code> */ + @Override protected void appendHeadingString(StringBuilder buffer) { buffer.append(getName()); buffer.append("("); buffer.append(n); - buffer.append(")"); - buffer.append(" "); + buffer.append(") "); } - /** The default N used if none is specified: 100 */ - public static final int defaultN = 100; + public int getN() { return n; } - /** Creates a WAND item with default N */ - public WeakAndItem() { - this(defaultN); - } - - public int getN() { - return n; - } - - public void setN(int N) { - this.n = N; - } + public void setN(int N) { this.n = N; } @Deprecated // TODO: Remove on Vespa 8 - public int getScoreThreshold() { - return scoreThreshold; - } + public int getScoreThreshold() { return scoreThreshold; } /** * Noop. @@ -93,9 +84,7 @@ public final class WeakAndItem extends NonReducibleCompositeItem { * @deprecated has no effect */ @Deprecated // TODO: Remove on Vespa 8 - public void setScoreThreshold(int scoreThreshold) { - this.scoreThreshold = scoreThreshold; - } + public void setScoreThreshold(int scoreThreshold) { this.scoreThreshold = scoreThreshold; } @Override protected void encodeThis(ByteBuffer buffer) { @@ -111,9 +100,7 @@ public final class WeakAndItem extends NonReducibleCompositeItem { } @Override - public int hashCode() { - return super.hashCode() + 31 * n; - } + public int hashCode() { return super.hashCode() + 31 * n; } /** Returns whether this item is of the same class and contains the same state as the given item. */ @Override diff --git a/container-search/src/main/java/com/yahoo/prelude/query/parser/AdvancedParser.java b/container-search/src/main/java/com/yahoo/prelude/query/parser/AdvancedParser.java index 74a993b0413..8b878417912 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/parser/AdvancedParser.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/parser/AdvancedParser.java @@ -147,10 +147,10 @@ public class AdvancedParser extends StructuredParser { return equiv; } return topLevelItem; - } else if (isTheWord("wand", item)) { + } else if (isTheWord("wand", item) || isTheWord("weakand", item)) { int n = consumeNumericArgument(); if (n == 0) - n=WeakAndItem.defaultN; + n = WeakAndItem.defaultN; if (topLevelIsClosed || !(topLevelItem instanceof WeakAndItem) || n != ((WeakAndItem)topLevelItem).getN()) { WeakAndItem wand = new WeakAndItem(); wand.setN(n); diff --git a/container-search/src/main/java/com/yahoo/prelude/semantics/RuleBase.java b/container-search/src/main/java/com/yahoo/prelude/semantics/RuleBase.java index ccc9c1d2f8f..feab8faf898 100644 --- a/container-search/src/main/java/com/yahoo/prelude/semantics/RuleBase.java +++ b/container-search/src/main/java/com/yahoo/prelude/semantics/RuleBase.java @@ -26,43 +26,43 @@ public class RuleBase { private String source; /** The name of the automata file used, or null if none */ - protected String automataFileName=null; + protected String automataFileName = null; /** * True if this rule base is default. * The semantics of default is left to the surrounding framework */ - private boolean isDefault=false; + private boolean isDefault = false; - private List<ProductionRule> productionRules=new java.util.ArrayList<>(); + private final List<ProductionRule> productionRules = new java.util.ArrayList<>(); - private Map<String, NamedCondition> namedConditions=new java.util.LinkedHashMap<>(); + private Map<String, NamedCondition> namedConditions = new java.util.LinkedHashMap<>(); /** The analyzer used to do evaluations over this rule base */ - private RuleEngine analyzer=new RuleEngine(this); + private final RuleEngine analyzer = new RuleEngine(this); - private static final PhraseMatcher nullPhraseMatcher=PhraseMatcher.getNullMatcher(); + private static final PhraseMatcher nullPhraseMatcher = PhraseMatcher.getNullMatcher(); /** * The matcher using an automata to match terms and phrases prior to matching rules * or the null matcher if no matcher is used. */ - private PhraseMatcher phraseMatcher=nullPhraseMatcher; + private PhraseMatcher phraseMatcher = nullPhraseMatcher; /** * The names of the rule bases included indirectly or directly in this * Ordered by first to last included */ - private Set<String> includedNames=new java.util.LinkedHashSet<>(); + private final Set<String> includedNames = new java.util.LinkedHashSet<>(); /** * True if this uses an automata, even if an automata is not present right now. Useful to validate without * having automatas available */ - private boolean usesAutomata=false; + private boolean usesAutomata = false; /** Should we allow stemmed matches? */ - private boolean stemming=true; + private boolean stemming = true; /** Creates an empty rule base. TODO: Disallow */ public RuleBase() { @@ -82,8 +82,8 @@ public class RuleBase { * @throws ParseException if the rule file can not be parsed correctly * @throws RuleBaseException if the rule file contains inconsistencies */ - public static RuleBase createFromFile(String ruleFile,String automataFile) throws java.io.IOException, ParseException { - return new RuleImporter().importFile(ruleFile,automataFile); + public static RuleBase createFromFile(String ruleFile, String automataFile) throws java.io.IOException, ParseException { + return new RuleImporter().importFile(ruleFile, automataFile); } /** @@ -96,14 +96,14 @@ public class RuleBase { * @throws com.yahoo.prelude.semantics.parser.ParseException if the rule file can not be parsed correctly * @throws com.yahoo.prelude.semantics.RuleBaseException if the rule file contains inconsistencies */ - public static RuleBase createFromString(String name,String ruleString,String automataFile) throws java.io.IOException, ParseException { - RuleBase base=new RuleImporter().importString(ruleString,automataFile,new RuleBase()); + public static RuleBase createFromString(String name, String ruleString, String automataFile) throws java.io.IOException, ParseException { + RuleBase base = new RuleImporter().importString(ruleString, automataFile, new RuleBase()); base.setName(name); return base; } /** Set to true to enable stemmed matches. True by default */ - public void setStemming(boolean stemming) { this.stemming=stemming; } + public void setStemming(boolean stemming) { this.stemming = stemming; } /** Returns whether stemmed matches are allowed. True by default */ public boolean getStemming() { return stemming; } @@ -125,19 +125,19 @@ public class RuleBase { /** Rules are order based - they are included recursively depth first */ private void inlineIncluded() { // Re-add our own conditions last to - added later overrides - Map<String, NamedCondition> thisConditions=namedConditions; - namedConditions=new LinkedHashMap<>(); + Map<String, NamedCondition> thisConditions = namedConditions; + namedConditions = new LinkedHashMap<>(); - Set<RuleBase> included=new HashSet<>(); + Set<RuleBase> included = new HashSet<>(); included.add(this); - for (ListIterator<ProductionRule> i=productionRules.listIterator(); i.hasNext(); ) { - ProductionRule rule=i.next(); + for (ListIterator<ProductionRule> i = productionRules.listIterator(); i.hasNext(); ) { + ProductionRule rule = i.next(); if ( ! (rule instanceof IncludeDirective) ) continue; i.remove(); - RuleBase toInclude=((IncludeDirective)rule).getIncludedBase(); + RuleBase toInclude = ((IncludeDirective)rule).getIncludedBase(); if ( ! included.contains(toInclude)) - toInclude.inlineIn(this,i,included); + toInclude.inlineIn(this, i, included); } namedConditions.putAll(thisConditions); @@ -147,14 +147,14 @@ public class RuleBase { * Recursively include this and everything it includes into the given rule base. * Skips bases already included in this. */ - private void inlineIn(RuleBase receiver,ListIterator<ProductionRule> receiverRules,Set<RuleBase> included) { + private void inlineIn(RuleBase receiver, ListIterator<ProductionRule> receiverRules, Set<RuleBase> included) { if (included.contains(this)) return; included.add(this); - for (Iterator<ProductionRule> i=productionRules.iterator(); i.hasNext(); ) { - ProductionRule rule=i.next(); + for (Iterator<ProductionRule> i = productionRules.iterator(); i.hasNext(); ) { + ProductionRule rule = i.next(); if (rule instanceof IncludeDirective) - ((IncludeDirective)rule).getIncludedBase().inlineIn(receiver,receiverRules,included); + ((IncludeDirective)rule).getIncludedBase().inlineIn(receiver, receiverRules, included); else receiverRules.add(rule); } @@ -164,11 +164,11 @@ public class RuleBase { /** Adds a named condition which can be referenced by rules */ public void addCondition(NamedCondition namedCondition) { - namedConditions.put(namedCondition.getName(),namedCondition); + namedConditions.put(namedCondition.getName(), namedCondition); - Condition condition=namedCondition.getCondition(); - Condition superCondition=findIncludedCondition(namedCondition.getName()); - resolveSuper(condition,superCondition); + Condition condition = namedCondition.getCondition(); + Condition superCondition = findIncludedCondition(namedCondition.getName()); + resolveSuper(condition, superCondition); } private void resolveSuper(Condition condition,Condition superCondition) { @@ -176,24 +176,22 @@ public class RuleBase { ((SuperCondition)condition).setCondition(superCondition); } else if (condition instanceof CompositeCondition) { - for (Iterator<Condition> i=((CompositeCondition)condition).conditionIterator(); i.hasNext(); ) { - Condition subCondition=i.next(); - resolveSuper(subCondition,superCondition); + for (Iterator<Condition> i = ((CompositeCondition)condition).conditionIterator(); i.hasNext(); ) { + Condition subCondition = i.next(); + resolveSuper(subCondition, superCondition); } } } private Condition findIncludedCondition(String name) { - for (Iterator<ProductionRule> i=productionRules.iterator(); i.hasNext(); ) { - ProductionRule rule=i.next(); + for (Iterator<ProductionRule> i = productionRules.iterator(); i.hasNext(); ) { + ProductionRule rule = i.next(); if ( ! (rule instanceof IncludeDirective) ) continue; - RuleBase included=((IncludeDirective)rule).getIncludedBase(); - NamedCondition condition=included.getCondition(name); - if (condition!=null) return condition.getCondition(); + RuleBase included = ((IncludeDirective)rule).getIncludedBase(); + NamedCondition condition = included.getCondition(name); + if (condition != null) return condition.getCondition(); included.findIncludedCondition(name); - // FIXME: dead code commented out - // if (condition!=null) return condition.getCondition(); } return null; } @@ -212,8 +210,8 @@ public class RuleBase { * change, and then re-added */ public void setName(String name) { - Validator.ensureNotNull("Rule base name",name); - this.name=name; + Validator.ensureNotNull("Rule base name", name); + this.name = name; } /** Returns the name of this rule base. This is never null. */ @@ -230,27 +228,27 @@ public class RuleBase { if ( ! new File(automataFile).exists()) throw new IllegalArgumentException("Automata file '" + automataFile + "' " + "included in " + this + " not found"); - phraseMatcher=new PhraseMatcher(automataFile); + phraseMatcher = new PhraseMatcher(automataFile); phraseMatcher.setIgnorePluralForm(true); phraseMatcher.setMatchAll(true); phraseMatcher.setMatchPhraseItems(true); phraseMatcher.setMatchSingleItems(true); setPhraseMatcher(phraseMatcher); - this.automataFileName=automataFile; + this.automataFileName = automataFile; } /** Returns the name of the automata file used, or null if none */ public String getAutomataFile() { return automataFileName; } /** Sets whether this base is default, the semantics of default is left to the application */ - public void setDefault(boolean isDefault) { this.isDefault=isDefault; } + public void setDefault(boolean isDefault) { this.isDefault = isDefault; } /** Returns whether this base is default, the semantics of default is left to the application */ public boolean isDefault() { return isDefault; } /** Thread safely sets the phrase matcher to use in this, or null to not use a phrase matcher */ public synchronized void setPhraseMatcher(PhraseMatcher matcher) { - if (matcher==null) + if (matcher == null) this.phraseMatcher = nullPhraseMatcher; else this.phraseMatcher = matcher; @@ -282,7 +280,7 @@ public class RuleBase { * Set to truew if this uses an automata, even if an automata is not present right now. * Useful to validate without having automatas available */ - void setUsesAutomata(boolean usesAutomata) { this.usesAutomata=usesAutomata; } + void setUsesAutomata(boolean usesAutomata) { this.usesAutomata = usesAutomata; } // Note that included rules are added though a list iterator, not this */ public void addRule(ProductionRule productionRule) { @@ -399,13 +397,13 @@ public class RuleBase { public String toContentString() { StringBuilder b = new StringBuilder(); for (Iterator<ProductionRule> i = productionRules.iterator(); i.hasNext(); ) { - b.append(i.next().toString()); + b.append(i.next()); b.append("\n"); } b.append("\n"); b.append("\n"); for (Iterator<NamedCondition> i = namedConditions.values().iterator(); i.hasNext(); ) { - b.append(i.next().toString()); + b.append(i.next()); b.append("\n"); } return b.toString(); @@ -414,10 +412,10 @@ public class RuleBase { /** A placeholder for an included rule base until it is inlined */ private static class IncludeDirective extends ProductionRule { - private RuleBase includedBase; + private final RuleBase includedBase; public IncludeDirective(RuleBase ruleBase) { - this.includedBase=ruleBase; + this.includedBase = ruleBase; } public RuleBase getIncludedBase() { return includedBase; } @@ -425,7 +423,6 @@ public class RuleBase { /** Not used */ public String getSymbol() { return ""; } - } } diff --git a/container-search/src/main/java/com/yahoo/prelude/semantics/RuleBaseException.java b/container-search/src/main/java/com/yahoo/prelude/semantics/RuleBaseException.java index d851c2648d5..5108777d938 100644 --- a/container-search/src/main/java/com/yahoo/prelude/semantics/RuleBaseException.java +++ b/container-search/src/main/java/com/yahoo/prelude/semantics/RuleBaseException.java @@ -6,7 +6,6 @@ package com.yahoo.prelude.semantics; * * @author bratseth */ -@SuppressWarnings("serial") public class RuleBaseException extends RuntimeException { public RuleBaseException(String message) { @@ -14,7 +13,7 @@ public class RuleBaseException extends RuntimeException { } public RuleBaseException(String message,Exception cause) { - super(message,cause); + super(message, cause); } } diff --git a/container-search/src/main/java/com/yahoo/prelude/semantics/RuleImporter.java b/container-search/src/main/java/com/yahoo/prelude/semantics/RuleImporter.java index 754d14ddcb4..ac643469ab6 100644 --- a/container-search/src/main/java/com/yahoo/prelude/semantics/RuleImporter.java +++ b/container-search/src/main/java/com/yahoo/prelude/semantics/RuleImporter.java @@ -28,13 +28,13 @@ public class RuleImporter { * If this is set, imported rule bases are looked up in this config * otherwise, they are looked up as files */ - private SemanticRulesConfig config = null; + private SemanticRulesConfig config; /** * Ignore requests to read automata files. * Useful to validate rule bases without having automatas present */ - private boolean ignoreAutomatas = false; + private boolean ignoreAutomatas; /** * Ignore requests to include files. @@ -61,8 +61,8 @@ public class RuleImporter { } public RuleImporter(SemanticRulesConfig config, boolean ignoreAutomatas) { - this.config=config; - this.ignoreAutomatas=ignoreAutomatas; + this.config = config; + this.ignoreAutomatas = ignoreAutomatas; } public RuleImporter(SemanticRulesConfig config, boolean ignoreAutomatas, boolean ignoreIncludes) { @@ -79,7 +79,7 @@ public class RuleImporter { * @throws ParseException if the file does not contain a valid semantic rule set */ public RuleBase importFile(String fileName) throws IOException, ParseException { - return importFile(fileName,null); + return importFile(fileName, null); } /** @@ -90,8 +90,8 @@ public class RuleImporter { * @throws java.io.IOException if the file can not be read for some reason * @throws ParseException if the file does not contain a valid semantic rule set */ - public RuleBase importFile(String fileName,String automataFile) throws IOException, ParseException { - return importFile(fileName,automataFile,null); + public RuleBase importFile(String fileName, String automataFile) throws IOException, ParseException { + return importFile(fileName, automataFile, null); } /** @@ -99,27 +99,26 @@ public class RuleImporter { * * @param fileName the rule file to use * @param automataFile the automata file to use, or null to not use any - * @param ruleBase an existing rule base to import these rules into, or null - * to create a new + * @param ruleBase an existing rule base to import these rules into, or null to create a new * @throws java.io.IOException if the file can not be read for some reason * @throws ParseException if the file does not contain a valid semantic rule set */ - public RuleBase importFile(String fileName,String automataFile,RuleBase ruleBase) throws IOException, ParseException { - ruleBase=privateImportFile(fileName,automataFile,ruleBase); + public RuleBase importFile(String fileName, String automataFile, RuleBase ruleBase) throws IOException, ParseException { + ruleBase = privateImportFile(fileName, automataFile, ruleBase); ruleBase.initialize(); return ruleBase; } - public RuleBase privateImportFile(String fileName,String automataFile,RuleBase ruleBase) throws IOException, ParseException { - BufferedReader reader=null; + public RuleBase privateImportFile(String fileName, String automataFile, RuleBase ruleBase) throws IOException, ParseException { + BufferedReader reader = null; try { - reader= IOUtils.createReader(fileName, "utf-8"); - File file=new File(fileName); - String absoluteFileName=file.getAbsolutePath(); - if (ruleBase==null) - ruleBase=new RuleBase(); + reader = IOUtils.createReader(fileName, "utf-8"); + File file = new File(fileName); + String absoluteFileName = file.getAbsolutePath(); + if (ruleBase == null) + ruleBase = new RuleBase(); ruleBase.setName(stripLastName(file.getName())); - privateImportFromReader(reader,absoluteFileName,automataFile,ruleBase); + privateImportFromReader(reader, absoluteFileName, automataFile, ruleBase); return ruleBase; } finally { @@ -129,14 +128,14 @@ public class RuleImporter { /** Imports all the rule files (files ending by "sr") in the given directory */ public List<RuleBase> importDir(String ruleBaseDir) throws IOException, ParseException { - File ruleBaseDirFile=new File(ruleBaseDir); - if (!ruleBaseDirFile.exists()) + File ruleBaseDirFile = new File(ruleBaseDir); + if ( ! ruleBaseDirFile.exists()) throw new IOException("Rule base dir '" + ruleBaseDirFile.getAbsolutePath() + "' does not exist"); - File[] files=ruleBaseDirFile.listFiles(); + File[] files = ruleBaseDirFile.listFiles(); Arrays.sort(files); - List<RuleBase> ruleBases=new java.util.ArrayList<>(); + List<RuleBase> ruleBases = new java.util.ArrayList<>(); for (File file : files) { - if (!file.getName().endsWith(".sr")) continue; + if ( ! file.getName().endsWith(".sr")) continue; RuleBase base = importFile(file.getAbsolutePath()); ruleBases.add(base); } @@ -144,50 +143,50 @@ public class RuleImporter { } /** Read and include a rule base in another */ - public void include(String ruleBaseName,RuleBase ruleBase) throws java.io.IOException, ParseException { + public void include(String ruleBaseName, RuleBase ruleBase) throws java.io.IOException, ParseException { if (ignoreIncludes) return; RuleBase include; - if (config==null) { - include=privateImportFromDirectory(ruleBaseName,ruleBase); + if (config == null) { + include = privateImportFromDirectory(ruleBaseName, ruleBase); } else { - include=privateImportFromConfig(ruleBaseName); + include = privateImportFromConfig(ruleBaseName); } ruleBase.include(include); } /** Returns an unitialized rule base */ - private RuleBase privateImportFromDirectory(String ruleBaseName,RuleBase ruleBase) throws IOException, ParseException { + private RuleBase privateImportFromDirectory(String ruleBaseName, RuleBase ruleBase) throws IOException, ParseException { RuleBase include = new RuleBase(); - String includeDir=new File(ruleBase.getSource()).getParentFile().getAbsolutePath(); + String includeDir = new File(ruleBase.getSource()).getParentFile().getAbsolutePath(); if (!ruleBaseName.endsWith(".sr")) - ruleBaseName=ruleBaseName + ".sr"; - File importFile=new File(includeDir,ruleBaseName); - if (!importFile.exists()) + ruleBaseName = ruleBaseName + ".sr"; + File importFile = new File(includeDir, ruleBaseName); + if ( ! importFile.exists()) throw new IOException("No file named '" + shortenPath(importFile.getPath()) + "'"); - return privateImportFile(importFile.getPath(),null,include); + return privateImportFile(importFile.getPath(), null, include); } /** Returns an unitialized rule base */ private RuleBase privateImportFromConfig(String ruleBaseName) throws IOException, ParseException { - SemanticRulesConfig.Rulebase ruleBaseConfig=findRuleBaseConfig(config,ruleBaseName); - if (ruleBaseConfig==null) - ruleBaseConfig=findRuleBaseConfig(config,stripLastName(ruleBaseName)); - if (ruleBaseConfig==null) + SemanticRulesConfig.Rulebase ruleBaseConfig = findRuleBaseConfig(config,ruleBaseName); + if (ruleBaseConfig == null) + ruleBaseConfig = findRuleBaseConfig(config, stripLastName(ruleBaseName)); + if (ruleBaseConfig == null) throw new ParseException("Could not find included rule base '" + ruleBaseName + "'"); return privateImportConfig(ruleBaseConfig); } - private SemanticRulesConfig.Rulebase findRuleBaseConfig(SemanticRulesConfig config,String ruleBaseName) { + private SemanticRulesConfig.Rulebase findRuleBaseConfig(SemanticRulesConfig config, String ruleBaseName) { for (Object aRulebase : config.rulebase()) { - SemanticRulesConfig.Rulebase ruleBaseConfig = (SemanticRulesConfig.Rulebase) aRulebase; + SemanticRulesConfig.Rulebase ruleBaseConfig = (SemanticRulesConfig.Rulebase)aRulebase; if (ruleBaseConfig.name().equals(ruleBaseName)) return ruleBaseConfig; } return null; } - public void setAutomata(RuleBase base,String automata) { + public void setAutomata(RuleBase base, String automata) { if (ignoreAutomatas) base.setUsesAutomata(true); // Stop it from failing on automata condition references else @@ -195,9 +194,9 @@ public class RuleImporter { } static String stripLastName(String fileName) { - int lastDotIndex=fileName.lastIndexOf("."); - if (lastDotIndex<0) return fileName; - return fileName.substring(0,lastDotIndex); + int lastDotIndex = fileName.lastIndexOf("."); + if (lastDotIndex < 0) return fileName; + return fileName.substring(0, lastDotIndex); } public RuleBase importString(String string, String automataFile) throws IOException, ParseException { @@ -217,22 +216,23 @@ public class RuleImporter { } public RuleBase importConfig(SemanticRulesConfig.Rulebase ruleBaseConfig) throws IOException, ParseException { - RuleBase ruleBase=privateImportConfig(ruleBaseConfig); + RuleBase ruleBase = privateImportConfig(ruleBaseConfig); ruleBase.initialize(); return ruleBase; } /** Imports an unitialized rule base */ - public RuleBase privateImportConfig(SemanticRulesConfig.Rulebase ruleBaseConfig) throws IOException, ParseException { - if (config==null) throw new IllegalStateException("Must initialize with config if importing from config"); + public RuleBase privateImportConfig(SemanticRulesConfig.Rulebase ruleBaseConfig) throws ParseException { + if (config == null) throw new IllegalStateException("Must initialize with config if importing from config"); RuleBase ruleBase = new RuleBase(); ruleBase.setName(ruleBaseConfig.name()); - return privateImportFromReader(new StringReader(ruleBaseConfig.rules()),"semantic-rules.cfg", - ruleBaseConfig.automata(),ruleBase); + return privateImportFromReader(new StringReader(ruleBaseConfig.rules()), + "semantic-rules.cfg", + ruleBaseConfig.automata(),ruleBase); } - public RuleBase importFromReader(Reader reader,String sourceInfo,String automataFile) throws ParseException { - return importFromReader(reader,sourceInfo,automataFile,null); + public RuleBase importFromReader(Reader reader, String sourceInfo, String automataFile) throws ParseException { + return importFromReader(reader, sourceInfo, automataFile, null); } /** @@ -245,7 +245,7 @@ public class RuleImporter { * @throws ParseException if the reader contains illegal rule syntax */ public RuleBase importFromReader(Reader reader, String sourceName, String automataFile, RuleBase ruleBase) throws ParseException { - ruleBase=privateImportFromReader(reader, sourceName, automataFile,ruleBase); + ruleBase = privateImportFromReader(reader, sourceName, automataFile, ruleBase); ruleBase.initialize(); return ruleBase; } @@ -253,19 +253,19 @@ public class RuleImporter { /** Returns an unitialized rule base */ public RuleBase privateImportFromReader(Reader reader, String sourceName, String automataFile, RuleBase ruleBase) throws ParseException { try { - if (ruleBase==null) { - ruleBase=new RuleBase(); + if (ruleBase == null) { + ruleBase = new RuleBase(); if (sourceName == null) sourceName = "anonymous"; ruleBase.setName(sourceName); } - ruleBase.setSource(sourceName.replace('\\','/')); + ruleBase.setSource(sourceName.replace('\\', '/')); new SemanticsParser(reader).semanticRules(ruleBase, this); - if (automataFile!=null && !automataFile.isEmpty()) - ruleBase.setAutomataFile(automataFile.replace('\\','/')); + if (automataFile != null && !automataFile.isEmpty()) + ruleBase.setAutomataFile(automataFile.replace('\\', '/')); return ruleBase; } catch (Throwable t) { // also catches token mgr errors - ParseException p=new ParseException("Could not parse '" + shortenPath(sourceName) + "'"); + ParseException p = new ParseException("Could not parse '" + shortenPath(sourceName) + "'"); p.initCause(t); throw p; } @@ -277,8 +277,8 @@ public class RuleImporter { * (if rules/ is present, these rules are read from an applicatino package) */ private static String shortenPath(String path) { - int rulesIndex=path.indexOf("rules/"); - if (rulesIndex<0) return path; + int rulesIndex = path.indexOf("rules/"); + if (rulesIndex < 0) return path; return path.substring(rulesIndex); } diff --git a/container-search/src/main/java/com/yahoo/prelude/semantics/engine/Evaluation.java b/container-search/src/main/java/com/yahoo/prelude/semantics/engine/Evaluation.java index 07342a48916..989f3040cc7 100644 --- a/container-search/src/main/java/com/yahoo/prelude/semantics/engine/Evaluation.java +++ b/container-search/src/main/java/com/yahoo/prelude/semantics/engine/Evaluation.java @@ -244,12 +244,14 @@ public class Evaluation { * @param desiredParentType the desired type of the composite which contains item when this returns */ public void insertItem(Item item, CompositeItem parent, int index, TermType desiredParentType) { - if (parent == null) { // TODO: Accommodate for termtype in this case too - query.getModel().getQueryTree().setRoot(item); + if (isEmpty(parent)) { + CompositeItem newParent = (CompositeItem)desiredParentType.createItemClass(); + newParent.addItem(item); + query.getModel().getQueryTree().setRoot(newParent); return; } - if (parent.getItemCount()>0 && parent instanceof QueryTree && parent.getItem(0) instanceof CompositeItem) { + if (parent.getItemCount() > 0 && parent instanceof QueryTree && parent.getItem(0) instanceof CompositeItem) { // combine with the existing root instead parent = (CompositeItem)parent.getItem(0); if (index == 1) { // that means adding it after the existing root @@ -261,9 +263,23 @@ public class Evaluation { && equalIndexNameIfParentIsPhrase(item, parent)) { addItem(parent, index, item, desiredParentType); } - else { + else if (incompatible(desiredParentType, parent)) { insertIncompatibleItem(item, parent, query, desiredParentType); } + else { + insertIncompatibleItemAsParent(item, parent, query, desiredParentType); + } + } + + private boolean isEmpty(Item item) { + if (item == null) return true; + if (item instanceof QueryTree && ((QueryTree) item).isEmpty()) return true; + return false; + } + + /** Returns true if the desired type cannot have childCandidate as a child */ + private boolean incompatible(TermType desiredParentType, Item childCandidate) { + return desiredParentType == TermType.EQUIV && childCandidate.getItemType() != Item.ItemType.EQUIV; } private void addItem(CompositeItem parent, int index, Item item, TermType desiredParentType) { @@ -305,6 +321,17 @@ public class Evaluation { } private void insertIncompatibleItem(Item item, CompositeItem parent, Query query, TermType desiredParentType) { + CompositeItem newParent; + if (desiredParentType == TermType.DEFAULT) + newParent = new AndItem(); + else + newParent = (CompositeItem)desiredParentType.createItemClass(); + + newParent.addItem(item); + parent.addItem(newParent); + } + + private void insertIncompatibleItemAsParent(Item item, CompositeItem parent, Query query, TermType desiredParentType) { // Create new parent CompositeItem newParent; if (desiredParentType == TermType.DEFAULT) @@ -325,11 +352,7 @@ public class Evaluation { } else { - int parentIndex = 0; - if (parentsParent != null) { - parentIndex = parentsParent.getItemIndex(parent); - } - parentsParent.setItem(parentIndex, newParent); + parentsParent.setItem(parentsParent.getItemIndex(parent), newParent); } } diff --git a/container-search/src/main/java/com/yahoo/prelude/semantics/engine/RuleEvaluation.java b/container-search/src/main/java/com/yahoo/prelude/semantics/engine/RuleEvaluation.java index 29a781b1e68..09fcb6a424f 100644 --- a/container-search/src/main/java/com/yahoo/prelude/semantics/engine/RuleEvaluation.java +++ b/container-search/src/main/java/com/yahoo/prelude/semantics/engine/RuleEvaluation.java @@ -10,7 +10,7 @@ import com.yahoo.prelude.semantics.rule.ProductionRule; import java.util.*; /** - * A particular evalutation of a particular rule. + * A particular evaluation of a particular rule. * * @author bratseth */ @@ -23,7 +23,7 @@ public class RuleEvaluation { // Remember that whenever state is added to this class, you // must consider whether/how to make that state backtrackable - // by savinginformation in choicepoint.state + // by saving information in choicepoint.state /** The items to match in this evaluation */ private List<FlattenedItem> items; @@ -41,12 +41,12 @@ public class RuleEvaluation { private String currentContext; /** A list of referencedMatches */ - private List<ReferencedMatches> referencedMatchesList = new java.util.ArrayList<>(); + private final List<ReferencedMatches> referencedMatchesList = new java.util.ArrayList<>(); - private List<Match> nonreferencedMatches = new java.util.ArrayList<>(); + private final List<Match> nonreferencedMatches = new java.util.ArrayList<>(); /** The evaluation owning this */ - private Evaluation evaluation; + private final Evaluation evaluation; /** The choice points saved in this evaluation */ private Stack<Choicepoint> choicepoints = null; @@ -82,7 +82,7 @@ public class RuleEvaluation { choicepoints.clear(); } - public void setMatchReferences(Set<String> matchReferences) { this.matchReferences=matchReferences; } + public void setMatchReferences(Set<String> matchReferences) { this.matchReferences = matchReferences; } /** * <p>Calculates an id which is unique for each match (the totality of the matched terms) @@ -96,23 +96,23 @@ public class RuleEvaluation { * we add other kinds of conditions.</p> */ int calculateMatchDigest(ProductionRule rule) { - int matchDigest=rule.hashCode(); - int matchCounter=1; - for (Iterator<ReferencedMatches> i=referencedMatchesList.iterator(); i.hasNext(); ) { - ReferencedMatches matches=i.next(); - int termCounter=0; - for (Iterator<Match> j=matches.matchIterator(); j.hasNext(); ) { - Match match=j.next(); - matchDigest=7*matchDigest*matchCounter+ - 71*termCounter+ - match.hashCode(); + int matchDigest = rule.hashCode(); + int matchCounter = 1; + for (Iterator<ReferencedMatches> i = referencedMatchesList.iterator(); i.hasNext(); ) { + ReferencedMatches matches = i.next(); + int termCounter = 0; + for (Iterator<Match> j = matches.matchIterator(); j.hasNext(); ) { + Match match = j.next(); + matchDigest = 7 * matchDigest * matchCounter+ + 71 * termCounter + + match.hashCode(); termCounter++; } matchCounter++; } - for (Iterator<Match> i=nonreferencedMatches.iterator(); i.hasNext(); ) { - Match match=i.next(); - matchDigest=7*matchDigest*matchCounter+match.hashCode(); + for (Iterator<Match> i = nonreferencedMatches.iterator(); i.hasNext(); ) { + Match match = i.next(); + matchDigest = 7 * matchDigest * matchCounter + match.hashCode(); matchCounter++; } return matchDigest; @@ -139,7 +139,7 @@ public class RuleEvaluation { /** Sets the current position */ public void setPosition(int position) { - this.position=position; + this.position = position; } /** Returns the total number of items to match in this evaluation */ @@ -151,21 +151,21 @@ public class RuleEvaluation { public Object getValue() { return value; } /** Sets the last value returned by a condition in this evaluatiino, or null */ - public void setValue(Object value) { this.value=value; } + public void setValue(Object value) { this.value = value; } /** Returns whether we are evaluating inside a condition which inverts the truth value */ public boolean isInNegation() { return inNegation; } /** sets whether we are evaluating inside a condition which inverts the truth value */ - public void setInNegation(boolean inNegation) { this.inNegation=inNegation; } + public void setInNegation(boolean inNegation) { this.inNegation = inNegation; } /** Returns the current position into the terms this evaluates over */ public int getPosition() { return position; } /** Sets a new current label and returns the previous one */ public String setCurrentLabel(String currentLabel) { - String oldLabel=currentLabel; - this.currentLabel=currentLabel; + String oldLabel = currentLabel; + this.currentLabel = currentLabel; return oldLabel; } @@ -179,8 +179,8 @@ public class RuleEvaluation { public FlattenedItem next() { position++; - if (position>=items.size()) { - position=items.size(); + if (position >= items.size()) { + position = items.size(); return null; } @@ -189,17 +189,16 @@ public class RuleEvaluation { // TODO: Simplistic yet. Nedd to support context nesting etc. public void entering(String context) { - if (context==null) return; - if (matchReferences!=null && matchReferences.contains(context)) - currentContext=context; - + if (context == null) return; + if (matchReferences != null && matchReferences.contains(context)) + currentContext = context; } public void leaving(String context) { - if (context==null) return; - if (currentContext==null) return; + if (context == null) return; + if (currentContext == null) return; if (currentContext.equals(context)) - currentContext=null; + currentContext = null; } /** @@ -269,7 +268,7 @@ public class RuleEvaluation { * @param termType the kind of item to index, this decides the resulting structure */ public void insertItem(Item item, CompositeItem parent, int index, TermType termType) { - evaluation.insertItem(item,parent,index,termType); + evaluation.insertItem(item, parent, index, termType); } /** Returns a read-only view of the items of this */ @@ -282,7 +281,7 @@ public class RuleEvaluation { } public void trace(int level,String string) { - evaluation.trace(level,string); + evaluation.trace(level, string); } public int getTraceLevel() { @@ -304,24 +303,24 @@ public class RuleEvaluation { * @param create true to create this choicepoint if it is missing * @return the choicepoint, or null if not present, and create is false */ - public Choicepoint getChoicepoint(Condition condition,boolean create) { - if (choicepoints==null) { - if (!create) return null; - choicepoints=new java.util.Stack<>(); + public Choicepoint getChoicepoint(Condition condition, boolean create) { + if (choicepoints == null) { + if ( ! create) return null; + choicepoints = new java.util.Stack<>(); } Choicepoint choicepoint=lookupChoicepoint(condition); - if (choicepoint==null) { - if (!create) return null; - choicepoint=new Choicepoint(this,condition); + if (choicepoint == null) { + if ( ! create) return null; + choicepoint = new Choicepoint(this, condition); choicepoints.push(choicepoint); } return choicepoint; } private Choicepoint lookupChoicepoint(Condition condition) { - for (Iterator<Choicepoint> i=choicepoints.iterator(); i.hasNext(); ) { - Choicepoint choicepoint=i.next(); - if (condition==choicepoint.getCondition()) + for (Iterator<Choicepoint> i = choicepoints.iterator(); i.hasNext(); ) { + Choicepoint choicepoint = i.next(); + if (condition == choicepoint.getCondition()) return choicepoint; } return null; @@ -337,8 +336,8 @@ public class RuleEvaluation { /** Remove all the terms recognized by this match */ public void removeMatches(ReferencedMatches matches) { - for (Iterator<Match> i=matches.matchIterator(); i.hasNext(); ) { - Match match=i.next(); + for (Iterator<Match> i = matches.matchIterator(); i.hasNext(); ) { + Match match = i.next(); removeItemByIdentity(match.getItem()); } } diff --git a/container-search/src/main/java/com/yahoo/prelude/semantics/rule/ProductionList.java b/container-search/src/main/java/com/yahoo/prelude/semantics/rule/ProductionList.java index 5b973228b1f..49d15710cd7 100644 --- a/container-search/src/main/java/com/yahoo/prelude/semantics/rule/ProductionList.java +++ b/container-search/src/main/java/com/yahoo/prelude/semantics/rule/ProductionList.java @@ -15,10 +15,10 @@ import com.yahoo.prelude.semantics.engine.RuleEvaluation; */ public class ProductionList { - private List<Production> productions =new java.util.ArrayList<>(); + private final List<Production> productions = new java.util.ArrayList<>(); /** True to replace by the production, false to add it */ - private boolean replacing=true; + private boolean replacing = true; public void addProduction(Production term) { term.setReplacing(replacing); @@ -28,12 +28,12 @@ public class ProductionList { /** True to replace, false to add, default true */ void setReplacing(boolean replacing) { - for (Iterator<Production> i=productions.iterator(); i.hasNext(); ) { - Production production=i.next(); + for (Iterator<Production> i = productions.iterator(); i.hasNext(); ) { + Production production = i.next(); production.setReplacing(replacing); } - this.replacing=replacing; + this.replacing = replacing; } /** Returns an unmodifiable view of the productions in this */ @@ -42,22 +42,22 @@ public class ProductionList { public int getTermCount() { return productions.size(); } void addMatchReferences(Set<String> matchReferences) { - for (Iterator<Production> i=productions.iterator(); i.hasNext(); ) { - Production term=i.next(); + for (Iterator<Production> i = productions.iterator(); i.hasNext(); ) { + Production term = i.next(); term.addMatchReferences(matchReferences); } } public void produce(RuleEvaluation e) { - for (int i=0; i<productions.size(); i++) { - productions.get(i).produce(e,i); + for (int i = 0; i < productions.size(); i++) { + productions.get(i).produce(e, i); } } public String toString() { - StringBuilder buffer=new StringBuilder(); - for (Iterator<Production> i=productions.iterator(); i.hasNext(); ) { - buffer.append(i.next().toString()); + StringBuilder buffer = new StringBuilder(); + for (Iterator<Production> i = productions.iterator(); i.hasNext(); ) { + buffer.append(i.next()); if (i.hasNext()) buffer.append(" "); } diff --git a/container-search/src/main/java/com/yahoo/prelude/semantics/rule/ReplacingProductionRule.java b/container-search/src/main/java/com/yahoo/prelude/semantics/rule/ReplacingProductionRule.java index dfa423ec889..70151ea479b 100644 --- a/container-search/src/main/java/com/yahoo/prelude/semantics/rule/ReplacingProductionRule.java +++ b/container-search/src/main/java/com/yahoo/prelude/semantics/rule/ReplacingProductionRule.java @@ -14,7 +14,7 @@ public class ReplacingProductionRule extends ProductionRule { /** Carries out the production of this rule */ public void produce(RuleEvaluation e) { removeNonreferencedMatches(e); - if (e.getTraceLevel()>=5) { + if (e.getTraceLevel() >= 5) { e.trace(5,"Removed terms to get '" + e.getEvaluation().getQuery().getModel().getQueryTree().getRoot() + "', will add terms"); } super.produce(e); @@ -22,16 +22,16 @@ public class ReplacingProductionRule extends ProductionRule { /** Remove items until there's only one item left */ private void removeNonreferencedMatches(RuleEvaluation e) { - int itemCount=e.getEvaluation().getQuerySize(); + int itemCount = e.getEvaluation().getQuerySize(); // Remove items backwards to ease index handling - for (int i=e.getNonreferencedMatchCount()-1; i>=0; i--) { + for (int i = e.getNonreferencedMatchCount() - 1; i >= 0; i--) { // Ensure we don't produce an empty query - if (getProduction().getTermCount()==0 && itemCount==1) + if (getProduction().getTermCount() == 0 && itemCount == 1) break; itemCount--; - Match match=e.getNonreferencedMatch(i); + Match match = e.getNonreferencedMatch(i); match.getItem().getParent().removeItem(match.getPosition()); } } diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/CloseableInvoker.java b/container-search/src/main/java/com/yahoo/search/dispatch/CloseableInvoker.java index 515d6249fd8..9329f4a6819 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/CloseableInvoker.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/CloseableInvoker.java @@ -12,6 +12,7 @@ import java.util.function.BiConsumer; * @author ollivir */ public abstract class CloseableInvoker implements Closeable { + protected abstract void release(); private BiConsumer<Boolean, Long> teardown = null; @@ -35,4 +36,5 @@ public abstract class CloseableInvoker implements Closeable { } release(); } + } diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java b/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java index 626cf087aca..9b92a78a7c9 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java @@ -65,7 +65,7 @@ public class Dispatcher extends AbstractComponent { /** A model of the search cluster this dispatches to */ private final SearchCluster searchCluster; - private final ClusterMonitor clusterMonitor; + private final ClusterMonitor<Node> clusterMonitor; private final LoadBalancer loadBalancer; @@ -108,7 +108,7 @@ public class Dispatcher extends AbstractComponent { } /* Protected for simple mocking in tests. Beware that searchCluster is shutdown on in deconstruct() */ - protected Dispatcher(ClusterMonitor clusterMonitor, + protected Dispatcher(ClusterMonitor<Node> clusterMonitor, SearchCluster searchCluster, DispatchConfig dispatchConfig, InvokerFactory invokerFactory, @@ -125,12 +125,7 @@ public class Dispatcher extends AbstractComponent { this.metricContext = metric.createContext(null); this.maxHitsPerNode = dispatchConfig.maxHitsPerNode(); searchCluster.addMonitoring(clusterMonitor); - Thread warmup = new Thread(new Runnable() { - @Override - public void run() { - warmup(dispatchConfig.warmuptime()); - } - }); + Thread warmup = new Thread(() -> warmup(dispatchConfig.warmuptime())); warmup.start(); try { while ( ! searchCluster.hasInformationAboutAllNodes()) { @@ -139,20 +134,17 @@ public class Dispatcher extends AbstractComponent { warmup.join(); } catch (InterruptedException e) {} - /* - * No we have information from all nodes and a ping iteration has completed. - * Instead of waiting until next ping interval to update coverage and group state, - * we should compute the state ourselves, so that when the dispatcher is ready the state - * of its groups are also known. - */ + // Now we have information from all nodes and a ping iteration has completed. + // Instead of waiting until next ping interval to update coverage and group state, + // we should compute the state ourselves, so that when the dispatcher is ready the state + // of its groups are also known. searchCluster.pingIterationCompleted(); } - /* - Will run important code in order to trigger JIT compilation and avoid cold start issues. - Currently warms up lz4 compression code. + /** + * Will run important code in order to trigger JIT compilation and avoid cold start issues. + * Currently warms up lz4 compression code. */ - private static long warmup(double seconds) { return new Compressor().warmup(seconds); } @@ -164,7 +156,7 @@ public class Dispatcher extends AbstractComponent { @Override public void deconstruct() { - /* The clustermonitor must be shutdown first as it uses the invokerfactory through the searchCluster. */ + // The clustermonitor must be shutdown first as it uses the invokerfactory through the searchCluster. clusterMonitor.shutdown(); invokerFactory.release(); } @@ -212,7 +204,7 @@ public class Dispatcher extends AbstractComponent { return invokerFactory.createSearchInvoker(searcher, query, OptionalInt.empty(), - Arrays.asList(node), + List.of(node), true, maxHitsPerNode) .orElseThrow(() -> new IllegalStateException("Could not dispatch directly to " + node)); diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/FillInvoker.java b/container-search/src/main/java/com/yahoo/search/dispatch/FillInvoker.java index dd4c4494ac5..8b7714aaf3b 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/FillInvoker.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/FillInvoker.java @@ -10,7 +10,8 @@ import com.yahoo.search.Result; * @author ollivir */ public abstract class FillInvoker extends CloseableInvoker { - /** Retrieve document summaries for the unfilled hits in the given {@link Result} */ + + /** Retrieves document summaries for the unfilled hits in the given {@link Result} */ public void fill(Result result, String summaryClass) { sendFillRequest(result, summaryClass); getFillResults(result, summaryClass); @@ -19,4 +20,5 @@ public abstract class FillInvoker extends CloseableInvoker { protected abstract void getFillResults(Result result, String summaryClass); protected abstract void sendFillRequest(Result result, String summaryClass); + } diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/InterleavedSearchInvoker.java b/container-search/src/main/java/com/yahoo/search/dispatch/InterleavedSearchInvoker.java index 036592dcf23..adf7368faa2 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/InterleavedSearchInvoker.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/InterleavedSearchInvoker.java @@ -235,17 +235,6 @@ public class InterleavedSearchInvoker extends SearchInvoker implements ResponseM return nextAdaptive; } - private String dbg(LeanHit hit) { - var buf = new StringBuilder(); - buf.append("LeanHit["); - if (hit.hasSortData()) buf.append("hasSortData,"); - buf.append("relevance=").append(hit.getRelevance()); - buf.append(",partId=").append(hit.getPartId()); - buf.append(",distributionKey=").append(hit.getDistributionKey()); - buf.append("]"); - return buf.toString(); - } - private List<LeanHit> mergeResult(Result result, InvokerResult partialResult, List<LeanHit> current) { collectCoverage(partialResult.getResult().getCoverage(true)); @@ -382,4 +371,5 @@ public class InterleavedSearchInvoker extends SearchInvoker implements ResponseM // For testing Collection<SearchInvoker> invokers() { return invokers; } + } diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/InvokerFactory.java b/container-search/src/main/java/com/yahoo/search/dispatch/InvokerFactory.java index f65e0e43757..dcf052d28e6 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/InvokerFactory.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/InvokerFactory.java @@ -35,7 +35,7 @@ public abstract class InvokerFactory { public abstract FillInvoker createFillInvoker(VespaBackEndSearcher searcher, Result result); /** - * Create a {@link SearchInvoker} for a list of content nodes. + * Creates a {@link SearchInvoker} for a list of content nodes. * * @param searcher the searcher processing the query * @param query the search query being processed diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/InvokerResult.java b/container-search/src/main/java/com/yahoo/search/dispatch/InvokerResult.java index 94c347a6927..2723429c0cf 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/InvokerResult.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/InvokerResult.java @@ -12,14 +12,19 @@ import java.util.List; /** * Wraps a Result and a flat, skinny hit list + * + * @author baldersheim */ public class InvokerResult { + private final Result result; private final List<LeanHit> leanHits; + public InvokerResult(Result result) { this.result = result; this.leanHits = Collections.emptyList(); } + public InvokerResult(Query query, int expectedHits) { result = new Result(query); leanHits = new ArrayList<>(expectedHits); @@ -32,6 +37,7 @@ public class InvokerResult { public List<LeanHit> getLeanHits() { return leanHits; } + void complete() { Query query = result.getQuery(); Sorting sorting = query.getRanking().getSorting(); @@ -47,4 +53,5 @@ public class InvokerResult { } leanHits.clear(); } + } diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/LeanHit.java b/container-search/src/main/java/com/yahoo/search/dispatch/LeanHit.java index 8a90557fa3b..df8fb2f29fa 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/LeanHit.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/LeanHit.java @@ -4,7 +4,11 @@ package com.yahoo.search.dispatch; import java.util.Arrays; +/** + * @author baldersheim + */ public class LeanHit implements Comparable<LeanHit> { + private final byte [] gid; private final double relevance; private final byte [] sortData; @@ -21,6 +25,7 @@ public class LeanHit implements Comparable<LeanHit> { this.partId = partId; this.distributionKey = distributionKey; } + public double getRelevance() { return relevance; } public byte [] getGid() { return gid; } public byte [] getSortData() { return sortData; } @@ -49,4 +54,5 @@ public class LeanHit implements Comparable<LeanHit> { int vr = (int) right[i] & 0xFF; return vl - vr; } + } diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/LoadBalancer.java b/container-search/src/main/java/com/yahoo/search/dispatch/LoadBalancer.java index 05e1ea6e2f9..ebde2ffc611 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/LoadBalancer.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/LoadBalancer.java @@ -43,8 +43,8 @@ public class LoadBalancer { } /** - * Select and allocate the search cluster group which is to be used for the next search query. Callers <b>must</b> call - * {@link #releaseGroup} symmetrically for each taken allocation. + * Select and allocate the search cluster group which is to be used for the next search query. + * Callers <b>must</b> call {@link #releaseGroup} symmetrically for each taken allocation. * * @param rejectedGroups if not null, the load balancer will only return groups with IDs not in the set * @return the node group to target, or <i>empty</i> if the internal dispatch logic cannot be used @@ -76,7 +76,7 @@ public class LoadBalancer { synchronized (this) { for (GroupStatus sched : scoreboard) { if (sched.group.id() == group.id()) { - sched.release(success, (double) searchTimeMs / 1000.0); + sched.release(success, searchTimeMs / 1000.0); break; } } diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/ResponseMonitor.java b/container-search/src/main/java/com/yahoo/search/dispatch/ResponseMonitor.java index c2e81d43677..3ebd21fa18a 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/ResponseMonitor.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/ResponseMonitor.java @@ -9,5 +9,7 @@ package com.yahoo.search.dispatch; * @author ollivir */ public interface ResponseMonitor<T> { + void responseAvailable(T from); + } diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/SearchErrorInvoker.java b/container-search/src/main/java/com/yahoo/search/dispatch/SearchErrorInvoker.java index 256759360f7..7dbc2e98759 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/SearchErrorInvoker.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/SearchErrorInvoker.java @@ -61,4 +61,5 @@ public class SearchErrorInvoker extends SearchInvoker { protected void setMonitor(ResponseMonitor<SearchInvoker> monitor) { this.monitor = monitor; } + } diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/SearchPath.java b/container-search/src/main/java/com/yahoo/search/dispatch/SearchPath.java index 7937be50813..f6480f80c01 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/SearchPath.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/SearchPath.java @@ -28,15 +28,11 @@ import java.util.stream.IntStream; public class SearchPath { /** - * Parse the search path and select nodes from the given cluster based on it. + * Parses the search path and select nodes from the given cluster based on it. * - * @param searchPath - * unparsed search path expression (see: model.searchPath in Search - * API reference) - * @param cluster - * the search cluster from which nodes are selected - * @throws InvalidSearchPathException - * if the searchPath is malformed + * @param searchPath unparsed search path expression (see: model.searchPath in Search API reference) + * @param cluster the search cluster from which nodes are selected + * @throws InvalidSearchPathException if the searchPath is malformed * @return list of nodes chosen with the search path, or an empty list in which * case some other node selection logic should be used */ diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/TopKEstimator.java b/container-search/src/main/java/com/yahoo/search/dispatch/TopKEstimator.java index aef1ef2f498..315dfdd4320 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/TopKEstimator.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/TopKEstimator.java @@ -6,9 +6,11 @@ import org.apache.commons.math3.distribution.TDistribution; /** * Use StudentT distribution and estimate how many hits you need from each partition * to to get the globally top-k documents with the desired probability + * * @author baldersheim */ public class TopKEstimator { + private final TDistribution studentT; private final double defaultP; private final boolean estimate; @@ -19,9 +21,11 @@ public class TopKEstimator { private static boolean needEstimate(double p) { return (0.0 < p) && (p < 1.0); } + TopKEstimator(double freedom, double defaultProbability) { this(freedom, defaultProbability, 0.0); } + public TopKEstimator(double freedom, double defaultProbability, double skewFactor) { this.studentT = new TDistribution(null, freedom); defaultP = defaultProbability; @@ -32,36 +36,44 @@ public class TopKEstimator { defaultCumulativeProbability[i] = computeCumulativeProbability(i+MIN_N, defaultP); } } + private double inverseCumulativeProbability(int n, double p) { if (p == defaultP && (n >= MIN_N) && (n < defaultCumulativeProbability.length + MIN_N)) { return defaultCumulativeProbability[n - MIN_N]; } return computeCumulativeProbability(n, p); } + private double computeCumulativeProbability(int n, double p) { double p_inverse = 1 - (1 - p)/computeN(n); return studentT.inverseCumulativeProbability(p_inverse); } + private double computeN(double n) { double p_max = (1 + skewFactor)/n; return Math.max(1, 1/p_max); } + double estimateExactK(double k, int n_i, double p) { double n = computeN(n_i); double variance = k * 1/n * (1 - 1/n); return k/n + inverseCumulativeProbability(n_i, p) * Math.sqrt(variance); } + double estimateExactK(double k, int n) { return estimateExactK(k, n, defaultP); } + public int estimateK(int k, int n) { return (estimate && (n >= MIN_N)) ? Math.min(k, (int)Math.ceil(estimateExactK(k, n, defaultP))) : k; } + public int estimateK(int k, int n, double p) { return (needEstimate(p) && (n >= MIN_N)) ? Math.min(k, (int)Math.ceil(estimateExactK(k, n, p))) : k; } } + diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/ProtobufSerialization.java b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/ProtobufSerialization.java index 6dc01f34571..250524fadf2 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/ProtobufSerialization.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/ProtobufSerialization.java @@ -196,7 +196,7 @@ public class ProtobufSerialization { result.getResult().setTotalHitCount(protobuf.getTotalHitCount()); result.getResult().setCoverage(convertToCoverage(protobuf)); - var haveGrouping = protobuf.getGroupingBlob() != null && !protobuf.getGroupingBlob().isEmpty(); + var haveGrouping = ! protobuf.getGroupingBlob().isEmpty(); if (haveGrouping) { BufferSerializer buf = new BufferSerializer(new GrowableByteBuffer(protobuf.getGroupingBlob().asReadOnlyByteBuffer())); int cnt = buf.getInt(null); @@ -219,7 +219,7 @@ public class ProtobufSerialization { } var slimeTrace = protobuf.getSlimeTrace(); - if (slimeTrace != null && !slimeTrace.isEmpty()) { + if ( ! slimeTrace.isEmpty()) { var traces = new Value.ArrayValue(); traces.add(new SlimeAdapter(BinaryFormat.decode(slimeTrace.toByteArray()).get())); query.trace(traces, query.getTraceLevel()); diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcPingFactory.java b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcPingFactory.java index 7d9b3ca1034..01e3ec3ca2b 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcPingFactory.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcPingFactory.java @@ -8,12 +8,16 @@ import com.yahoo.search.dispatch.searchcluster.Pinger; import com.yahoo.search.dispatch.searchcluster.PongHandler; public class RpcPingFactory implements PingFactory { + private final RpcResourcePool rpcResourcePool; + public RpcPingFactory(RpcResourcePool rpcResourcePool) { this.rpcResourcePool = rpcResourcePool; } + @Override public Pinger createPinger(Node node, ClusterMonitor<Node> monitor, PongHandler pongHandler) { return new RpcPing(node, monitor, rpcResourcePool, pongHandler); } + } diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcProtobufFillInvoker.java b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcProtobufFillInvoker.java index 341b9b2bce3..8a17be8102e 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcProtobufFillInvoker.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcProtobufFillInvoker.java @@ -38,6 +38,7 @@ import java.util.logging.Logger; * @author ollivir */ public class RpcProtobufFillInvoker extends FillInvoker { + private static final String RPC_METHOD = "vespa.searchprotocol.getDocsums"; private static final Logger log = Logger.getLogger(RpcProtobufFillInvoker.class.getName()); diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcResourcePool.java b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcResourcePool.java index 746461630dd..c3d072b8db6 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcResourcePool.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcResourcePool.java @@ -26,6 +26,7 @@ import java.util.Random; * @author ollivir */ public class RpcResourcePool extends AbstractComponent { + /** The compression method which will be used with rpc dispatch. "lz4" (default) and "none" is supported. */ public final static CompoundName dispatchCompression = new CompoundName("dispatch.compression"); diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcSearchInvoker.java b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcSearchInvoker.java index 4c0b77207d5..20b11efb470 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcSearchInvoker.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcSearchInvoker.java @@ -102,9 +102,7 @@ public class RpcSearchInvoker extends SearchInvoker implements Client.ResponseRe ProtobufResponse protobufResponse = response.response().get(); CompressionType compression = CompressionType.valueOf(protobufResponse.compression()); byte[] payload = resourcePool.compressor().decompress(protobufResponse.compressedPayload(), compression, protobufResponse.uncompressedSize()); - var result = ProtobufSerialization.deserializeToSearchResult(payload, query, searcher, node.pathIndex(), node.key()); - - return result; + return ProtobufSerialization.deserializeToSearchResult(payload, query, searcher, node.pathIndex(), node.key()); } @Override diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/PingFactory.java b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/PingFactory.java index 2e07d8d61e6..3b9e9573367 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/PingFactory.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/PingFactory.java @@ -3,7 +3,9 @@ package com.yahoo.search.dispatch.searchcluster; import com.yahoo.search.cluster.ClusterMonitor; - +/** + * @author ollivir + */ public interface PingFactory { Pinger createPinger(Node node, ClusterMonitor<Node> monitor, PongHandler pongHandler); diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/Pinger.java b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/Pinger.java index b4a7ccbf98c..681a7d0af2c 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/Pinger.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/Pinger.java @@ -8,5 +8,7 @@ package com.yahoo.search.dispatch.searchcluster; * @author baldersheim */ public interface Pinger { + void ping(); + } diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/PongHandler.java b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/PongHandler.java index 1b39f14fd86..c39426e9d76 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/PongHandler.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/PongHandler.java @@ -9,5 +9,7 @@ import com.yahoo.prelude.Pong; * @author baldersheim */ public interface PongHandler { + void handle(Pong pong); + } diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java index 1897c0af8bc..b3b2c23e7dc 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java @@ -94,7 +94,7 @@ public class SearchCluster implements NodeManager<Node> { this(clusterId, dispatchConfig, 1, vipStatus, pingFactory); } - public void addMonitoring(ClusterMonitor clusterMonitor) { + public void addMonitoring(ClusterMonitor<Node> clusterMonitor) { for (var group : orderedGroups()) { for (var node : group.nodes()) clusterMonitor.add(node, true); @@ -158,8 +158,11 @@ public class SearchCluster implements NodeManager<Node> { } } - /** Returns the number of nodes per group - size()/groups.size() */ - public int groupSize() { + /** + * Returns the wanted number of nodes per group - size()/groups.size(). + * The actual node count for a given group may differ due to node retirements. + */ + public int wantedGroupSize() { if (groups().size() == 0) return size(); return size() / groups().size(); } @@ -305,9 +308,10 @@ public class SearchCluster implements NodeManager<Node> { // With just one group sufficient coverage may not be the same as full coverage, as the // group will always be marked sufficient for use. updateSufficientCoverage(group, true); - boolean fullCoverage = isGroupCoverageSufficient(group.workingNodes(), group.nodes().size(), group.getActiveDocuments(), - group.getActiveDocuments()); - trackGroupCoverageChanges(0, group, fullCoverage, group.getActiveDocuments()); + boolean sufficientCoverage = isGroupCoverageSufficient(group.workingNodes(), + group.getActiveDocuments(), + group.getActiveDocuments()); + trackGroupCoverageChanges(0, group, sufficientCoverage, group.getActiveDocuments()); } private void pingIterationCompletedMultipleGroups() { @@ -327,7 +331,7 @@ public class SearchCluster implements NodeManager<Node> { Group group = orderedGroups().get(i); long activeDocuments = activeDocumentsInGroup[i]; long averageDocumentsInOtherGroups = (sumOfActiveDocuments - activeDocuments) / (numGroups - 1); - boolean sufficientCoverage = isGroupCoverageSufficient(group.workingNodes(), group.nodes().size(), activeDocuments, averageDocumentsInOtherGroups); + boolean sufficientCoverage = isGroupCoverageSufficient(group.workingNodes(), activeDocuments, averageDocumentsInOtherGroups); anyGroupsSufficientCoverage = anyGroupsSufficientCoverage || sufficientCoverage; updateSufficientCoverage(group, sufficientCoverage); trackGroupCoverageChanges(i, group, sufficientCoverage, averageDocumentsInOtherGroups); @@ -349,23 +353,22 @@ public class SearchCluster implements NodeManager<Node> { } } - private boolean isGroupCoverageSufficient(int workingNodes, int nodesInGroup, long activeDocuments, long averageDocumentsInOtherGroups) { - boolean sufficientCoverage = true; + private boolean isGroupCoverageSufficient(int workingNodesInGroup, long activeDocuments, long averageDocumentsInOtherGroups) { + double documentCoverage = 100.0 * (double) activeDocuments / averageDocumentsInOtherGroups; - if (averageDocumentsInOtherGroups > 0) { - double coverage = 100.0 * (double) activeDocuments / averageDocumentsInOtherGroups; - sufficientCoverage = coverage >= dispatchConfig.minActivedocsPercentage(); - } - if (sufficientCoverage) { - sufficientCoverage = isGroupNodeCoverageSufficient(workingNodes, nodesInGroup); - } - return sufficientCoverage; + if (averageDocumentsInOtherGroups > 0 && documentCoverage < dispatchConfig.minActivedocsPercentage()) + return false; + + if ( ! isGroupNodeCoverageSufficient(workingNodesInGroup)) + return false; + + return true; } - private boolean isGroupNodeCoverageSufficient(int workingNodes, int nodesInGroup) { + private boolean isGroupNodeCoverageSufficient(int workingNodesInGroup) { int nodesAllowedDown = dispatchConfig.maxNodesDownPerGroup() - + (int) (((double) nodesInGroup * (100.0 - dispatchConfig.minGroupCoverage())) / 100.0); - return workingNodes + nodesAllowedDown >= nodesInGroup; + + (int) (((double) wantedGroupSize() * (100.0 - dispatchConfig.minGroupCoverage())) / 100.0); + return workingNodesInGroup + nodesAllowedDown >= wantedGroupSize(); } public boolean isGroupWellBalanced(OptionalInt groupId) { @@ -379,7 +382,7 @@ public class SearchCluster implements NodeManager<Node> { */ public boolean isPartialGroupCoverageSufficient(OptionalInt knownGroupId, List<Node> nodes) { if (orderedGroups().size() == 1) { - boolean sufficient = nodes.size() >= groupSize() - dispatchConfig.maxNodesDownPerGroup(); + boolean sufficient = nodes.size() >= wantedGroupSize() - dispatchConfig.maxNodesDownPerGroup(); return sufficient; } @@ -391,7 +394,6 @@ public class SearchCluster implements NodeManager<Node> { if (group == null) { return false; } - int nodesInGroup = group.nodes().size(); long sumOfActiveDocuments = 0; int otherGroups = 0; for (Group g : orderedGroups()) { @@ -405,7 +407,7 @@ public class SearchCluster implements NodeManager<Node> { activeDocuments += n.getActiveDocuments(); } long averageDocumentsInOtherGroups = sumOfActiveDocuments / otherGroups; - return isGroupCoverageSufficient(nodes.size(), nodesInGroup, activeDocuments, averageDocumentsInOtherGroups); + return isGroupCoverageSufficient(nodes.size(), activeDocuments, averageDocumentsInOtherGroups); } private void trackGroupCoverageChanges(int index, Group group, boolean fullCoverage, long averageDocuments) { @@ -413,19 +415,21 @@ public class SearchCluster implements NodeManager<Node> { boolean changed = group.isFullCoverageStatusChanged(fullCoverage); if (changed || (!fullCoverage && System.currentTimeMillis() > nextLogTime)) { nextLogTime = System.currentTimeMillis() + 30 * 1000; - int requiredNodes = groupSize() - dispatchConfig.maxNodesDownPerGroup(); + int requiredNodes = group.nodes().size() - dispatchConfig.maxNodesDownPerGroup(); if (fullCoverage) { - log.info(() -> String.format("Group %d is now good again (%d/%d active docs, coverage %d/%d)", - index, group.getActiveDocuments(), averageDocuments, group.workingNodes(), groupSize())); + log.info(() -> String.format("Cluster %s: Group %d is now good again (%d/%d active docs, coverage %d/%d)", + clusterId, index, group.getActiveDocuments(), averageDocuments, + group.workingNodes(), group.nodes().size())); } else { StringBuilder missing = new StringBuilder(); for (var node : group.nodes()) { if (node.isWorking() != Boolean.TRUE) { - missing.append('\n').append(node.toString()); + missing.append('\n').append(node); } } - log.warning(() -> String.format("Coverage of group %d is only %d/%d (requires %d) (%d/%d active docs) Failed nodes are:%s", - index, group.workingNodes(), groupSize(), requiredNodes, group.getActiveDocuments(), averageDocuments, missing.toString())); + log.warning(() -> String.format("Cluster %s: Coverage of group %d is only %d/%d (requires %d) (%d/%d active docs) Failed nodes are:%s", + clusterId, index, group.workingNodes(), group.nodes().size(), requiredNodes, + group.getActiveDocuments(), averageDocuments, missing)); } } } diff --git a/container-search/src/main/java/com/yahoo/search/querytransform/VespaLowercasingSearcher.java b/container-search/src/main/java/com/yahoo/search/querytransform/VespaLowercasingSearcher.java index 25488aa7bbc..3aa9e59003d 100644 --- a/container-search/src/main/java/com/yahoo/search/querytransform/VespaLowercasingSearcher.java +++ b/container-search/src/main/java/com/yahoo/search/querytransform/VespaLowercasingSearcher.java @@ -31,8 +31,7 @@ public class VespaLowercasingSearcher extends LowercasingSearcher { public boolean shouldLowercase(WordItem word, IndexFacts.Session indexFacts) { if (word.isLowercased()) return false; - Index index = indexFacts.getIndex(word.getIndexName()); - return index.isLowercase() || index.isAttribute(); + return indexFacts.getIndex(word.getIndexName()).isLowercase(); } @Override @@ -41,8 +40,7 @@ public class VespaLowercasingSearcher extends LowercasingSearcher { StringBuilder sb = new StringBuilder(); sb.append(commonPath).append(".").append(word.getIndexName()); - Index index = indexFacts.getIndex(sb.toString()); - return index.isLowercase() || index.isAttribute(); + return indexFacts.getIndex(sb.toString()).isLowercase(); } } |