diff options
author | Øyvind Grønnesby <oyving@verizonmedia.com> | 2019-10-08 13:00:36 +0200 |
---|---|---|
committer | Øyvind Grønnesby <oyving@verizonmedia.com> | 2019-10-08 13:00:36 +0200 |
commit | 948500c2a4d52aef59ec18706ae4b8d793afb060 (patch) | |
tree | ce305f94dbf4f90423255adc3b02fd441bcf211b /container-search | |
parent | 0d9892df0aa99e13ff83bfdeb3de8f1e8305bdcd (diff) | |
parent | 64d8a699b668a5ed1dae0b86bd88bd3a2b40e48c (diff) |
Merge remote-tracking branch 'origin/master' into ogronnesby/let-plugin-get-key-as-parameter
Conflicts:
hosted-api/src/main/java/ai/vespa/hosted/api/Properties.java
Diffstat (limited to 'container-search')
26 files changed, 408 insertions, 176 deletions
diff --git a/container-search/abi-spec.json b/container-search/abi-spec.json index facf894272d..edaa5b4b824 100644 --- a/container-search/abi-spec.json +++ b/container-search/abi-spec.json @@ -5884,6 +5884,62 @@ ], "fields": [] }, + "com.yahoo.search.query.profile.SubstituteString$Component": { + "superClass": "java.lang.Object", + "interfaces": [], + "attributes": [ + "public", + "abstract" + ], + "methods": [ + "public void <init>()", + "protected abstract java.lang.String getValue(java.util.Map, com.yahoo.processing.request.Properties)" + ], + "fields": [] + }, + "com.yahoo.search.query.profile.SubstituteString$PropertyComponent": { + "superClass": "com.yahoo.search.query.profile.SubstituteString$Component", + "interfaces": [], + "attributes": [ + "public", + "final" + ], + "methods": [ + "public void <init>(java.lang.String)", + "public java.lang.String getValue(java.util.Map, com.yahoo.processing.request.Properties)", + "public java.lang.String toString()" + ], + "fields": [] + }, + "com.yahoo.search.query.profile.SubstituteString$RelativePropertyComponent": { + "superClass": "com.yahoo.search.query.profile.SubstituteString$Component", + "interfaces": [], + "attributes": [ + "public", + "final" + ], + "methods": [ + "public void <init>(java.lang.String)", + "public java.lang.String getValue(java.util.Map, com.yahoo.processing.request.Properties)", + "public java.lang.String fieldName()", + "public java.lang.String toString()" + ], + "fields": [] + }, + "com.yahoo.search.query.profile.SubstituteString$StringComponent": { + "superClass": "com.yahoo.search.query.profile.SubstituteString$Component", + "interfaces": [], + "attributes": [ + "public", + "final" + ], + "methods": [ + "public void <init>(java.lang.String)", + "public java.lang.String getValue(java.util.Map, com.yahoo.processing.request.Properties)", + "public java.lang.String toString()" + ], + "fields": [] + }, "com.yahoo.search.query.profile.SubstituteString": { "superClass": "java.lang.Object", "interfaces": [], @@ -5892,7 +5948,11 @@ ], "methods": [ "public static com.yahoo.search.query.profile.SubstituteString create(java.lang.String)", + "public void <init>(java.util.List, java.lang.String)", + "public boolean hasRelative()", "public java.lang.String substitute(java.util.Map, com.yahoo.processing.request.Properties)", + "public java.util.List components()", + "public java.lang.String stringValue()", "public int hashCode()", "public boolean equals(java.lang.Object)", "public java.lang.String toString()" @@ -6004,8 +6064,9 @@ ], "methods": [ "public void <init>()", + "public java.lang.Object valueFor(com.yahoo.search.query.profile.DimensionBinding)", "public void add(java.lang.Object, com.yahoo.search.query.profile.DimensionBinding)", - "public com.yahoo.search.query.profile.compiled.DimensionalValue build()" + "public com.yahoo.search.query.profile.compiled.DimensionalValue build(java.util.Map)" ], "fields": [] }, diff --git a/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java b/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java index 1f621eb926c..0c8e564578b 100644 --- a/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java @@ -7,7 +7,6 @@ import com.yahoo.component.chain.dependencies.After; import com.yahoo.container.QrSearchersConfig; import com.yahoo.container.handler.VipStatus; import com.yahoo.jdisc.Metric; -import com.yahoo.net.HostName; import com.yahoo.prelude.IndexFacts; import com.yahoo.prelude.fastsearch.ClusterParams; import com.yahoo.prelude.fastsearch.DocumentdbInfoConfig; @@ -27,8 +26,6 @@ import com.yahoo.vespa.config.search.DispatchConfig; import com.yahoo.vespa.streamingvisitors.VdsStreamingSearcher; import org.apache.commons.lang.StringUtils; -import java.net.InetAddress; -import java.net.UnknownHostException; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -371,6 +368,10 @@ public class ClusterSearcher extends Searcher { } @Override - public void deconstruct() { } + public void deconstruct() { + if (server != null) { + server.shutDown(); + } + } } diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastSearcher.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastSearcher.java index b0b3a7800e9..9a4913b3840 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastSearcher.java @@ -173,4 +173,9 @@ public class FastSearcher extends VespaBackEndSearcher { return getLogger().isLoggable(Level.FINE); } + @Override + public void shutDown() { + super.shutDown(); + dispatcher.shutDown(); + } } diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/VespaBackEndSearcher.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/VespaBackEndSearcher.java index 8f4b49ac71e..bc3ac6cdef1 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/VespaBackEndSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/VespaBackEndSearcher.java @@ -392,4 +392,6 @@ public abstract class VespaBackEndSearcher extends PingableSearcher { return getLogger().isLoggable(Level.FINE); } + public void shutDown() { } + } diff --git a/container-search/src/main/java/com/yahoo/search/cluster/ClusterMonitor.java b/container-search/src/main/java/com/yahoo/search/cluster/ClusterMonitor.java index 22c7f59872c..a016f7d695c 100644 --- a/container-search/src/main/java/com/yahoo/search/cluster/ClusterMonitor.java +++ b/container-search/src/main/java/com/yahoo/search/cluster/ClusterMonitor.java @@ -9,7 +9,9 @@ import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.concurrent.Executor; +import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; import java.util.logging.Level; import java.util.logging.Logger; @@ -26,9 +28,9 @@ public class ClusterMonitor<T> { private static Logger log = Logger.getLogger(ClusterMonitor.class.getName()); - private NodeManager<T> nodeManager; + private final NodeManager<T> nodeManager; - private MonitorThread monitorThread; + private final MonitorThread monitorThread; private volatile boolean shutdown = false; @@ -119,28 +121,35 @@ public class ClusterMonitor<T> { } public void run() { - log.fine("Starting cluster monitor thread"); + log.info("Starting cluster monitor thread " + getName()); // Pings must happen in a separate thread from this to handle timeouts // By using a cached thread pool we ensured that 1) a single thread will be used // for all pings when there are no problems (important because it ensures that // any thread local connections are reused) 2) a new thread will be started to execute // new pings when a ping is not responding - Executor pingExecutor=Executors.newCachedThreadPool(ThreadFactoryFactory.getDaemonThreadFactory("search.ping")); + ExecutorService pingExecutor=Executors.newCachedThreadPool(ThreadFactoryFactory.getDaemonThreadFactory("search.ping")); while (!isInterrupted()) { try { Thread.sleep(configuration.getCheckInterval()); log.finest("Activating ping"); ping(pingExecutor); } - catch (Exception e) { + catch (Throwable e) { if (shutdown && e instanceof InterruptedException) { break; + } else if ( ! (e instanceof Exception) ) { + log.log(Level.WARNING,"Error in monitor thread, will quit", e); + break; } else { - log.log(Level.WARNING,"Error in monitor thread",e); + log.log(Level.WARNING,"Exception in monitor thread", e); } } } - log.fine("Stopped cluster monitor thread"); + pingExecutor.shutdown(); + try { + pingExecutor.awaitTermination(10, TimeUnit.SECONDS); + } catch (InterruptedException e) { } + log.info("Stopped cluster monitor thread " + getName()); } } diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java b/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java index 7369b33e82d..ddd319b7bcb 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java @@ -195,6 +195,10 @@ public class Dispatcher extends AbstractComponent { return Optional.empty(); } + public void shutDown() { + searchCluster.shutDown(); + } + private void emitDispatchMetric(Optional<SearchInvoker> invoker) { if (invoker.isEmpty()) { metric.add(FDISPATCH_METRIC, 1, metricContext); diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/Node.java b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/Node.java index b47f2fefa5b..09ad715b471 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/Node.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/Node.java @@ -16,17 +16,15 @@ public class Node { private final int key; private int pathIndex; private final String hostname; - private final int fs4port; - final int group; + private final int group; private final AtomicBoolean statusIsKnown = new AtomicBoolean(false); private final AtomicBoolean working = new AtomicBoolean(true); private final AtomicLong activeDocuments = new AtomicLong(0); - public Node(int key, String hostname, int fs4port, int group) { + public Node(int key, String hostname, int group) { this.key = key; this.hostname = hostname; - this.fs4port = fs4port; this.group = group; } @@ -41,14 +39,15 @@ public class Node { public String hostname() { return hostname; } - public int fs4port() { return fs4port; } - /** Returns the id of this group this node belongs to */ public int group() { return group; } public void setWorking(boolean working) { this.statusIsKnown.lazySet(true); this.working.lazySet(working); + if ( ! working ) { + activeDocuments.set(0); + } } /** Returns whether this node is currently responding to requests, or null if status is not known */ @@ -57,17 +56,17 @@ public class Node { } /** Updates the active documents on this node */ - public void setActiveDocuments(long activeDocuments) { + void setActiveDocuments(long activeDocuments) { this.activeDocuments.set(activeDocuments); } /** Returns the active documents on this node. If unknown, 0 is returned. */ - public long getActiveDocuments() { - return this.activeDocuments.get(); + long getActiveDocuments() { + return activeDocuments.get(); } @Override - public int hashCode() { return Objects.hash(hostname, fs4port); } + public int hashCode() { return Objects.hash(hostname, key, pathIndex, group); } @Override public boolean equals(Object o) { @@ -75,11 +74,15 @@ public class Node { if ( ! (o instanceof Node)) return false; Node other = (Node)o; if ( ! Objects.equals(this.hostname, other.hostname)) return false; - if ( ! Objects.equals(this.fs4port, other.fs4port)) return false; + if ( ! Objects.equals(this.key, other.key)) return false; + if ( ! Objects.equals(this.pathIndex, other.pathIndex)) return false; + if ( ! Objects.equals(this.group, other.group)) return false; + return true; } @Override - public String toString() { return "search node " + hostname + ":" + fs4port + " in group " + group; } + public String toString() { return "search node key = " + key + " hostname = "+ hostname + " path = " + pathIndex + " in group " + group + + " statusIsKnown = " + statusIsKnown.get() + " working = " + working.get() + " activeDocs = " + activeDocuments.get(); } } diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java index a55a970e8ff..3595a24ca92 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java @@ -46,6 +46,7 @@ public class SearchCluster implements NodeManager<Node> { private final ClusterMonitor<Node> clusterMonitor; private final VipStatus vipStatus; private PingFactory pingFactory; + private long nextLogTime = 0; /** * A search node on this local machine having the entire corpus, which we therefore @@ -73,7 +74,7 @@ public class SearchCluster implements NodeManager<Node> { } this.groups = groupsBuilder.build(); LinkedHashMap<Integer, Group> groupIntroductionOrder = new LinkedHashMap<>(); - nodes.forEach(node -> groupIntroductionOrder.put(node.group(), groups.get(node.group))); + nodes.forEach(node -> groupIntroductionOrder.put(node.group(), groups.get(node.group()))); this.orderedGroups = ImmutableList.<Group>builder().addAll(groupIntroductionOrder.values()).build(); // Index nodes by host @@ -91,6 +92,10 @@ public class SearchCluster implements NodeManager<Node> { this.clusterMonitor = new ClusterMonitor<>(this); } + public void shutDown() { + clusterMonitor.shutdown(); + } + public void startClusterMonitoring(PingFactory pingFactory) { this.pingFactory = pingFactory; @@ -141,7 +146,7 @@ public class SearchCluster implements NodeManager<Node> { } for (DispatchConfig.Node node : dispatchConfig.node()) { if (filter.test(node)) { - nodesBuilder.add(new Node(node.key(), node.host(), node.fs4port(), node.group())); + nodesBuilder.add(new Node(node.key(), node.host(), node.group())); } } return nodesBuilder.build(); @@ -409,14 +414,21 @@ public class SearchCluster implements NodeManager<Node> { private void trackGroupCoverageChanges(int index, Group group, boolean fullCoverage, long averageDocuments) { boolean changed = group.isFullCoverageStatusChanged(fullCoverage); - if (changed) { + if (changed || (!fullCoverage && System.currentTimeMillis() > nextLogTime)) { + nextLogTime = System.currentTimeMillis() + 30 * 1000; int requiredNodes = groupSize() - dispatchConfig.maxNodesDownPerGroup(); if (fullCoverage) { log.info(() -> String.format("Group %d is now good again (%d/%d active docs, coverage %d/%d)", index, group.getActiveDocuments(), averageDocuments, group.workingNodes(), groupSize())); } else { - log.warning(() -> String.format("Coverage of group %d is only %d/%d (requires %d)", - index, group.workingNodes(), groupSize(), requiredNodes)); + StringBuilder missing = new StringBuilder(); + for (var node : group.nodes()) { + if (node.isWorking() != Boolean.TRUE) { + missing.append('\n').append(node.toString()); + } + } + log.warning(() -> String.format("Coverage of group %d is only %d/%d (requires %d) (%d/%d active docs) Failed nodes are:%s", + index, group.workingNodes(), groupSize(), requiredNodes, group.getActiveDocuments(), averageDocuments, missing.toString())); } } } diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/DimensionBinding.java b/container-search/src/main/java/com/yahoo/search/query/profile/DimensionBinding.java index 0059b761734..1fc1e19e3ee 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profile/DimensionBinding.java +++ b/container-search/src/main/java/com/yahoo/search/query/profile/DimensionBinding.java @@ -21,7 +21,7 @@ public class DimensionBinding { private DimensionValues values; /** The binding from those dimensions to values, and possibly other values */ - private Map<String,String> context; + private Map<String, String> context; public static final DimensionBinding nullBinding = new DimensionBinding(Collections.unmodifiableList(Collections.emptyList()), DimensionValues.empty, null); @@ -195,11 +195,11 @@ public class DimensionBinding { @Override public String toString() { if (isInvalid()) return "Invalid DimensionBinding"; - if (dimensions==null) return "DimensionBinding []"; - StringBuilder b=new StringBuilder("DimensionBinding ["); - for (int i=0; i<dimensions.size(); i++) { + if (dimensions == null) return "DimensionBinding []"; + StringBuilder b = new StringBuilder("DimensionBinding ["); + for (int i = 0; i < dimensions.size(); i++) { b.append(dimensions.get(i)).append("=").append(values.get(i)); - if (i<dimensions.size()-1) + if (i < dimensions.size()-1) b.append(", "); } b.append("]"); diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfile.java b/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfile.java index acca2d403be..f5f6b2d2550 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfile.java +++ b/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfile.java @@ -102,7 +102,7 @@ public class QueryProfile extends FreezableSimpleComponent implements Cloneable */ public List<QueryProfile> inherited() { if (isFrozen()) return inherited; // Frozen profiles always have an unmodifiable, non-null list - if (inherited==null) return Collections.emptyList(); + if (inherited == null) return Collections.emptyList(); return Collections.unmodifiableList(inherited); } @@ -474,17 +474,17 @@ public class QueryProfile extends FreezableSimpleComponent implements Cloneable /** Returns this value, or its corresponding substitution string if it contains substitutions */ protected Object convertToSubstitutionString(Object value) { - if (value==null) return value; - if (value.getClass()!=String.class) return value; - SubstituteString substituteString=SubstituteString.create((String)value); - if (substituteString==null) return value; + if (value == null) return value; + if (value.getClass() != String.class) return value; + SubstituteString substituteString = SubstituteString.create((String)value); + if (substituteString == null) return value; return substituteString; } /** Returns the field description of this field, or null if it is not typed */ protected FieldDescription getFieldDescription(CompoundName name, DimensionBinding binding) { - FieldDescriptionQueryProfileVisitor visitor=new FieldDescriptionQueryProfileVisitor(name.asList()); - accept(visitor, binding,null); + FieldDescriptionQueryProfileVisitor visitor = new FieldDescriptionQueryProfileVisitor(name.asList()); + accept(visitor, binding, null); return visitor.result(); } @@ -493,23 +493,23 @@ public class QueryProfile extends FreezableSimpleComponent implements Cloneable * false if it is declared unoverridable (in instance or type), and null if this profile has no * opinion on the matter because the value is not set in this. */ - Boolean isLocalOverridable(String localName,DimensionBinding binding) { - if (localLookup(localName, binding)==null) return null; // Not set - Boolean isLocalInstanceOverridable=isLocalInstanceOverridable(localName); - if (isLocalInstanceOverridable!=null) + Boolean isLocalOverridable(String localName, DimensionBinding binding) { + if (localLookup(localName, binding) == null) return null; // Not set + Boolean isLocalInstanceOverridable = isLocalInstanceOverridable(localName); + if (isLocalInstanceOverridable != null) return isLocalInstanceOverridable.booleanValue(); - if (type!=null) return type.isOverridable(localName); + if (type != null) return type.isOverridable(localName); return true; } protected Boolean isLocalInstanceOverridable(String localName) { - if (overridable==null) return null; + if (overridable == null) return null; return overridable.get(localName); } - protected Object lookup(CompoundName name,boolean allowQueryProfileResult, DimensionBinding dimensionBinding) { - SingleValueQueryProfileVisitor visitor=new SingleValueQueryProfileVisitor(name.asList(),allowQueryProfileResult); - accept(visitor,dimensionBinding,null); + protected Object lookup(CompoundName name, boolean allowQueryProfileResult, DimensionBinding dimensionBinding) { + SingleValueQueryProfileVisitor visitor = new SingleValueQueryProfileVisitor(name.asList(), allowQueryProfileResult); + accept(visitor, dimensionBinding, null); return visitor.getResult(); } @@ -518,7 +518,7 @@ public class QueryProfile extends FreezableSimpleComponent implements Cloneable } void acceptAndEnter(String key, QueryProfileVisitor visitor,DimensionBinding dimensionBinding, QueryProfile owner) { - boolean allowContent=visitor.enter(key); + boolean allowContent = visitor.enter(key); accept(allowContent, visitor, dimensionBinding, owner); if (allowContent) visitor.leave(key); @@ -548,25 +548,25 @@ public class QueryProfile extends FreezableSimpleComponent implements Cloneable } protected void visitVariants(boolean allowContent,QueryProfileVisitor visitor,DimensionBinding dimensionBinding) { - if (getVariants()!=null) + if (getVariants() != null) getVariants().accept(allowContent, getType(), visitor, dimensionBinding); } protected void visitInherited(boolean allowContent,QueryProfileVisitor visitor,DimensionBinding dimensionBinding, QueryProfile owner) { - if (inherited==null) return; + if (inherited == null) return; for (QueryProfile inheritedProfile : inherited) { - inheritedProfile.accept(allowContent,visitor,dimensionBinding.createFor(inheritedProfile.getDimensions()), owner); + inheritedProfile.accept(allowContent, visitor, dimensionBinding.createFor(inheritedProfile.getDimensions()), owner); if (visitor.isDone()) return; } } private void visitContent(QueryProfileVisitor visitor,DimensionBinding dimensionBinding) { - String contentKey=visitor.getLocalKey(); + String contentKey = visitor.getLocalKey(); // Visit this' content - if (contentKey!=null) { // Get only the content of the current key - if (type!=null) - contentKey=type.unalias(contentKey); + if (contentKey != null) { // Get only the content of the current key + if (type != null) + contentKey = type.unalias(contentKey); visitor.acceptValue(contentKey, getContent(contentKey), dimensionBinding, this); } else { // get all content in this @@ -590,11 +590,11 @@ public class QueryProfile extends FreezableSimpleComponent implements Cloneable /** Sets the value of a node in <i>this</i> profile - the local name given must not be nested (contain dots) */ protected QueryProfile setLocalNode(String localName, Object value,QueryProfileType parentType, DimensionBinding dimensionBinding, QueryProfileRegistry registry) { - if (parentType!=null && type==null && !isFrozen()) - type=parentType; + if (parentType != null && type == null && ! isFrozen()) + type = parentType; - value=checkAndConvertAssignment(localName, value, registry); - localPut(localName,value,dimensionBinding); + value = checkAndConvertAssignment(localName, value, registry); + localPut(localName, value, dimensionBinding); return this; } @@ -605,20 +605,20 @@ public class QueryProfile extends FreezableSimpleComponent implements Cloneable */ static Object combineValues(Object newValue, Object existingValue) { if (newValue instanceof QueryProfile) { - QueryProfile newProfile=(QueryProfile)newValue; - if ( existingValue==null || ! (existingValue instanceof QueryProfile)) { + QueryProfile newProfile = (QueryProfile)newValue; + if ( existingValue == null || ! (existingValue instanceof QueryProfile)) { if (!isModifiable(newProfile)) - newProfile=new BackedOverridableQueryProfile(newProfile); // Make the query profile reference overridable - newProfile.value=existingValue; + newProfile = new BackedOverridableQueryProfile(newProfile); // Make the query profile reference overridable + newProfile.value = existingValue; return newProfile; } // if both are profiles: - return combineProfiles(newProfile,(QueryProfile)existingValue); + return combineProfiles(newProfile, (QueryProfile)existingValue); } else { if (existingValue instanceof QueryProfile) { // we need to set a non-leaf value on a query profile - QueryProfile existingProfile=(QueryProfile)existingValue; + QueryProfile existingProfile = (QueryProfile)existingValue; if (isModifiable(existingProfile)) { existingProfile.setValue(newValue); return null; @@ -636,16 +636,16 @@ public class QueryProfile extends FreezableSimpleComponent implements Cloneable } private static QueryProfile combineProfiles(QueryProfile newProfile,QueryProfile existingProfile) { - QueryProfile returnValue=null; + QueryProfile returnValue = null; QueryProfile existingModifiable; // Ensure the existing profile is modifiable - if (existingProfile.getClass()==QueryProfile.class) { + if (existingProfile.getClass() == QueryProfile.class) { existingModifiable = new BackedOverridableQueryProfile(existingProfile); - returnValue=existingModifiable; + returnValue = existingModifiable; } else { // is an overridable wrapper - existingModifiable=existingProfile; // May be used as-is + existingModifiable = existingProfile; // May be used as-is } // Make the existing profile inherit the new one @@ -655,7 +655,7 @@ public class QueryProfile extends FreezableSimpleComponent implements Cloneable existingModifiable.addInherited(newProfile); // Remove content from the existing which the new one does not allow overrides of - if (existingModifiable.content!=null) { + if (existingModifiable.content != null) { for (String key : existingModifiable.content.unmodifiableMap().keySet()) { if ( ! newProfile.isLocalOverridable(key, null)) { existingModifiable.content.remove(key); @@ -681,10 +681,10 @@ public class QueryProfile extends FreezableSimpleComponent implements Cloneable * @throws IllegalArgumentException if the assignment is illegal */ protected Object checkAndConvertAssignment(String localName, Object value, QueryProfileRegistry registry) { - if (type==null) return value; // no type checking + if (type == null) return value; // no type checking - FieldDescription fieldDescription=type.getField(localName); - if (fieldDescription==null) { + FieldDescription fieldDescription = type.getField(localName); + if (fieldDescription == null) { if (type.isStrict()) throw new IllegalArgumentException("'" + localName + "' is not declared in " + type + ", and the type is strict"); return value; @@ -710,8 +710,8 @@ public class QueryProfile extends FreezableSimpleComponent implements Cloneable /** Do a variant-aware content lookup in this */ protected Object localLookup(String name, DimensionBinding dimensionBinding) { Object node = null; - if ( variants != null && !dimensionBinding.isNull()) - node = variants.get(name,type,true,dimensionBinding); + if ( variants != null && ! dimensionBinding.isNull()) + node = variants.get(name,type,true, dimensionBinding); if (node == null) node = content == null ? null : content.get(name); return node; @@ -801,7 +801,7 @@ public class QueryProfile extends FreezableSimpleComponent implements Cloneable } /** Sets a value directly in this query profile (unless frozen) */ - private void localPut(String localName,Object value, DimensionBinding dimensionBinding) { + private void localPut(String localName, Object value, DimensionBinding dimensionBinding) { ensureNotFrozen(); if (type != null) @@ -813,17 +813,17 @@ public class QueryProfile extends FreezableSimpleComponent implements Cloneable if (dimensionBinding.isNull()) { Object combinedValue; if (value instanceof QueryProfile) - combinedValue = combineValues(value,content==null ? null : content.get(localName)); + combinedValue = combineValues(value, content == null ? null : content.get(localName)); else combinedValue = combineValues(value, localLookup(localName, dimensionBinding)); if (combinedValue!=null) - content.put(localName,combinedValue); + content.put(localName, combinedValue); } else { if (variants == null) variants = new QueryProfileVariants(dimensionBinding.getDimensions(), this); - variants.set(localName,dimensionBinding.getValues(),value); + variants.set(localName, dimensionBinding.getValues(), value); } } diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileCompiler.java b/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileCompiler.java index accef7ba154..fd2852fda60 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileCompiler.java +++ b/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileCompiler.java @@ -33,31 +33,36 @@ public class QueryProfileCompiler { } public static CompiledQueryProfile compile(QueryProfile in, CompiledQueryProfileRegistry registry) { - DimensionalMap.Builder<CompoundName, Object> values = new DimensionalMap.Builder<>(); - DimensionalMap.Builder<CompoundName, QueryProfileType> types = new DimensionalMap.Builder<>(); - DimensionalMap.Builder<CompoundName, Object> references = new DimensionalMap.Builder<>(); - DimensionalMap.Builder<CompoundName, Object> unoverridables = new DimensionalMap.Builder<>(); - - // Resolve values for each existing variant and combine into a single data structure - Set<DimensionBindingForPath> variants = collectVariants(CompoundName.empty, in, DimensionBinding.nullBinding); - variants.add(new DimensionBindingForPath(DimensionBinding.nullBinding, CompoundName.empty)); // if this contains no variants - log.fine(() -> "Compiling " + in.toString() + " having " + variants.size() + " variants"); - for (DimensionBindingForPath variant : variants) { - log.finer(() -> " Compiling variant " + variant); - for (Map.Entry<String, Object> entry : in.listValues(variant.path(), variant.binding().getContext(), null).entrySet()) { - values.put(variant.path().append(entry.getKey()), variant.binding(), entry.getValue()); + try { + DimensionalMap.Builder<CompoundName, Object> values = new DimensionalMap.Builder<>(); + DimensionalMap.Builder<CompoundName, QueryProfileType> types = new DimensionalMap.Builder<>(); + DimensionalMap.Builder<CompoundName, Object> references = new DimensionalMap.Builder<>(); + DimensionalMap.Builder<CompoundName, Object> unoverridables = new DimensionalMap.Builder<>(); + + // Resolve values for each existing variant and combine into a single data structure + Set<DimensionBindingForPath> variants = collectVariants(CompoundName.empty, in, DimensionBinding.nullBinding); + variants.add(new DimensionBindingForPath(DimensionBinding.nullBinding, CompoundName.empty)); // if this contains no variants + log.fine(() -> "Compiling " + in.toString() + " having " + variants.size() + " variants"); + for (DimensionBindingForPath variant : variants) { + log.finer(() -> " Compiling variant " + variant); + for (Map.Entry<String, Object> entry : in.listValues(variant.path(), variant.binding().getContext(), null).entrySet()) { + values.put(variant.path().append(entry.getKey()), variant.binding(), entry.getValue()); + } + for (Map.Entry<CompoundName, QueryProfileType> entry : in.listTypes(variant.path(), variant.binding().getContext()).entrySet()) + types.put(variant.path().append(entry.getKey()), variant.binding(), entry.getValue()); + for (CompoundName reference : in.listReferences(variant.path(), variant.binding().getContext())) + references.put(variant.path().append(reference), variant.binding(), Boolean.TRUE); // Used as a set; value is ignored + for (CompoundName name : in.listUnoverridable(variant.path(), variant.binding().getContext())) + unoverridables.put(variant.path().append(name), variant.binding(), Boolean.TRUE); // Used as a set; value is ignored } - for (Map.Entry<CompoundName, QueryProfileType> entry : in.listTypes(variant.path(), variant.binding().getContext()).entrySet()) - types.put(variant.path().append(entry.getKey()), variant.binding(), entry.getValue()); - for (CompoundName reference : in.listReferences(variant.path(), variant.binding().getContext())) - references.put(variant.path().append(reference), variant.binding(), Boolean.TRUE); // Used as a set; value is ignored - for (CompoundName name : in.listUnoverridable(variant.path(), variant.binding().getContext())) - unoverridables.put(variant.path().append(name), variant.binding(), Boolean.TRUE); // Used as a set; value is ignored - } - return new CompiledQueryProfile(in.getId(), in.getType(), - values.build(), types.build(), references.build(), unoverridables.build(), - registry); + return new CompiledQueryProfile(in.getId(), in.getType(), + values.build(), types.build(), references.build(), unoverridables.build(), + registry); + } + catch (IllegalArgumentException e) { + throw new IllegalArgumentException("Invalid " + in, e); + } } /** diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/SubstituteString.java b/container-search/src/main/java/com/yahoo/search/query/profile/SubstituteString.java index 3252f0f4662..446bb250856 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profile/SubstituteString.java +++ b/container-search/src/main/java/com/yahoo/search/query/profile/SubstituteString.java @@ -2,6 +2,7 @@ package com.yahoo.search.query.profile; import com.yahoo.processing.request.Properties; +import com.yahoo.search.query.profile.compiled.CompiledQueryProfile; import java.util.ArrayList; import java.util.List; @@ -22,6 +23,7 @@ public class SubstituteString { private final List<Component> components; private final String stringValue; + private final boolean hasRelative; /** * Returns a new SubstituteString if the given string contains substitutions, null otherwise. @@ -35,34 +37,48 @@ public class SubstituteString { int end = value.indexOf("}", start + 2); if (end < 0) throw new IllegalArgumentException("Unterminated value substitution '" + value.substring(start) + "'"); - String propertyName = value.substring(start+2,end); - if (propertyName.indexOf("%{") >= 0) + String propertyName = value.substring(start + 2, end); + if (propertyName.contains("%{")) throw new IllegalArgumentException("Unterminated value substitution '" + value.substring(start) + "'"); components.add(new StringComponent(value.substring(lastEnd, start))); - components.add(new PropertyComponent(propertyName)); - lastEnd = end+1; + if (propertyName.startsWith(".")) + components.add(new RelativePropertyComponent(propertyName.substring(1))); + else + components.add(new PropertyComponent(propertyName)); + lastEnd = end + 1; start = value.indexOf("%{", lastEnd); } components.add(new StringComponent(value.substring(lastEnd))); return new SubstituteString(components, value); } - private SubstituteString(List<Component> components, String stringValue) { + public SubstituteString(List<Component> components, String stringValue) { this.components = components; this.stringValue = stringValue; + this.hasRelative = components.stream().anyMatch(component -> component instanceof RelativePropertyComponent); } + /** Returns whether this has at least one relative component */ + public boolean hasRelative() { return hasRelative; } + /** - * Perform the substitution in this, by looking up in the given query profile, + * Perform the substitution in this, by looking up in the given properties, * and returns the resulting string + * + * @param context the content which is used to resolve profile variants when looking up substitution values + * @param substitution the properties in which values to be substituted are looked up */ public String substitute(Map<String, String> context, Properties substitution) { StringBuilder b = new StringBuilder(); for (Component component : components) - b.append(component.getValue(context,substitution)); + b.append(component.getValue(context, substitution)); return b.toString(); } + public List<Component> components() { return components; } + + public String stringValue() { return stringValue; } + @Override public int hashCode() { return stringValue.hashCode(); @@ -81,13 +97,13 @@ public class SubstituteString { return stringValue; } - private abstract static class Component { + public abstract static class Component { protected abstract String getValue(Map<String, String> context, Properties substitution); } - private final static class StringComponent extends Component { + public final static class StringComponent extends Component { private final String value; @@ -107,7 +123,7 @@ public class SubstituteString { } - private final static class PropertyComponent extends Component { + public final static class PropertyComponent extends Component { private final String propertyName; @@ -116,7 +132,7 @@ public class SubstituteString { } @Override - public String getValue(Map<String,String> context, Properties substitution) { + public String getValue(Map<String, String> context, Properties substitution) { Object value = substitution.get(propertyName, context, substitution); if (value == null) return ""; return String.valueOf(value); @@ -129,4 +145,30 @@ public class SubstituteString { } + /** + * A component where the value should be looked up in the profile containing the substitution field + * rather than globally + */ + public final static class RelativePropertyComponent extends Component { + + private final String fieldName; + + public RelativePropertyComponent(String fieldName) { + this.fieldName = fieldName; + } + + @Override + public String getValue(Map<String, String> context, Properties substitution) { + throw new IllegalStateException("Should be resolved during compilation"); + } + + public String fieldName() { return fieldName; } + + @Override + public String toString() { + return "%{" + fieldName + "}"; + } + + } + } diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/compiled/Binding.java b/container-search/src/main/java/com/yahoo/search/query/profile/compiled/Binding.java index d94d601f103..2774bd4ebf2 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profile/compiled/Binding.java +++ b/container-search/src/main/java/com/yahoo/search/query/profile/compiled/Binding.java @@ -34,7 +34,7 @@ public class Binding implements Comparable<Binding> { private final int hashCode; @SuppressWarnings("unchecked") - public static final Binding nullBinding= new Binding(Integer.MAX_VALUE, Collections.<String,String>emptyMap()); + public static final Binding nullBinding = new Binding(Integer.MAX_VALUE, Collections.<String,String>emptyMap()); public static Binding createFrom(DimensionBinding dimensionBinding) { if (dimensionBinding.getDimensions().size() > maxDimensions) diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/compiled/CompiledQueryProfile.java b/container-search/src/main/java/com/yahoo/search/query/profile/compiled/CompiledQueryProfile.java index d6e93701ca1..ea85a2be242 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profile/compiled/CompiledQueryProfile.java +++ b/container-search/src/main/java/com/yahoo/search/query/profile/compiled/CompiledQueryProfile.java @@ -45,7 +45,8 @@ public class CompiledQueryProfile extends AbstractComponent implements Cloneable /** * Creates a new query profile from an id. */ - public CompiledQueryProfile(ComponentId id, QueryProfileType type, + public CompiledQueryProfile(ComponentId id, + QueryProfileType type, DimensionalMap<CompoundName, Object> entries, DimensionalMap<CompoundName, QueryProfileType> types, DimensionalMap<CompoundName, Object> references, diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/compiled/DimensionalMap.java b/container-search/src/main/java/com/yahoo/search/query/profile/compiled/DimensionalMap.java index b4a1c66e4e0..2e8f5dcf91c 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profile/compiled/DimensionalMap.java +++ b/container-search/src/main/java/com/yahoo/search/query/profile/compiled/DimensionalMap.java @@ -58,7 +58,7 @@ public class DimensionalMap<KEY, VALUE> { public DimensionalMap<KEY, VALUE> build() { Map<KEY, DimensionalValue<VALUE>> map = new HashMap<>(); for (Map.Entry<KEY, DimensionalValue.Builder<VALUE>> entry : entries.entrySet()) { - map.put(entry.getKey(), entry.getValue().build()); + map.put(entry.getKey(), entry.getValue().build(entries)); } return new DimensionalMap<>(map); } diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/compiled/DimensionalValue.java b/container-search/src/main/java/com/yahoo/search/query/profile/compiled/DimensionalValue.java index 506472c97d1..50d0a2de46f 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profile/compiled/DimensionalValue.java +++ b/container-search/src/main/java/com/yahoo/search/query/profile/compiled/DimensionalValue.java @@ -1,7 +1,9 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.query.profile.compiled; +import com.yahoo.processing.request.CompoundName; import com.yahoo.search.query.profile.DimensionBinding; +import com.yahoo.search.query.profile.SubstituteString; import java.util.ArrayList; import java.util.Collections; @@ -58,6 +60,15 @@ public class DimensionalValue<VALUE> { /** The minimal set of variants needed to capture all values at this key */ private Map<VALUE, Value.Builder<VALUE>> buildableVariants = new HashMap<>(); + /** Returns the value for the given binding, or null if none */ + public VALUE valueFor(DimensionBinding variantBinding) { + for (var entry : buildableVariants.entrySet()) { + if (entry.getValue().variants.contains(variantBinding)) + return entry.getKey(); + } + return null; + } + public void add(VALUE value, DimensionBinding variantBinding) { // Note: We know we can index by the value because its possible types are constrained // to what query profiles allow: String, primitives and query profiles @@ -69,10 +80,10 @@ public class DimensionalValue<VALUE> { variant.addVariant(variantBinding); } - public DimensionalValue<VALUE> build() { + public DimensionalValue<VALUE> build(Map<?, DimensionalValue.Builder<VALUE>> entries) { List<Value> variants = new ArrayList<>(); for (Value.Builder buildableVariant : buildableVariants.values()) { - variants.addAll(buildableVariant.build()); + variants.addAll(buildableVariant.build(entries)); } return new DimensionalValue(variants); } @@ -139,14 +150,17 @@ public class DimensionalValue<VALUE> { } /** Build a separate value object for each dimension combination which has this value */ - public List<Value<VALUE>> build() { + public List<Value<VALUE>> build(Map<CompoundName, DimensionalValue.Builder<VALUE>> entries) { // Shortcut for efficiency of the normal case - if (variants.size()==1) - return Collections.singletonList(new Value<>(value, Binding.createFrom(variants.iterator().next()))); + if (variants.size() == 1) { + return Collections.singletonList(new Value<>(substituteIfRelative(value, variants.iterator().next(), entries), + Binding.createFrom(variants.iterator().next()))); + } List<Value<VALUE>> values = new ArrayList<>(variants.size()); - for (DimensionBinding variant : variants) - values.add(new Value<>(value, Binding.createFrom(variant))); + for (DimensionBinding variant : variants) { + values.add(new Value<>(substituteIfRelative(value, variant, entries), Binding.createFrom(variant))); + } return values; } @@ -154,6 +168,46 @@ public class DimensionalValue<VALUE> { return value; } + @SuppressWarnings("unchecked") + private VALUE substituteIfRelative(VALUE value, + DimensionBinding variant, + Map<CompoundName, DimensionalValue.Builder<VALUE>> entries) { + if (value instanceof SubstituteString) { + SubstituteString substitute = (SubstituteString)value; + if (substitute.hasRelative()) { + List<SubstituteString.Component> resolvedComponents = new ArrayList<>(substitute.components().size()); + for (SubstituteString.Component component : substitute.components()) { + if (component instanceof SubstituteString.RelativePropertyComponent) { + SubstituteString.RelativePropertyComponent relativeComponent = (SubstituteString.RelativePropertyComponent)component; + var substituteValues = lookupByLocalName(relativeComponent.fieldName(), entries); + if (substituteValues == null) + throw new IllegalArgumentException("Could not resolve local substitution '" + + relativeComponent.fieldName() + "' in variant " + + variant); + String resolved = substituteValues.valueFor(variant).toString(); + resolvedComponents.add(new SubstituteString.StringComponent(resolved)); + } + else { + resolvedComponents.add(component); + } + } + return (VALUE)new SubstituteString(resolvedComponents, substitute.stringValue()); + } + } + return value; + } + + private DimensionalValue.Builder<VALUE> lookupByLocalName(String localName, + Map<CompoundName, DimensionalValue.Builder<VALUE>> entries) { + for (var entry : entries.entrySet()) { + if (entry.getKey().last().equals(localName)) + return entry.getValue(); + } + return null; + } + } + } + } diff --git a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/FastSearcherTestCase.java b/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/FastSearcherTestCase.java index eb4d65693bb..4011611b049 100644 --- a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/FastSearcherTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/FastSearcherTestCase.java @@ -81,7 +81,7 @@ public class FastSearcherTestCase { @Test public void testSinglePassGroupingIsForcedWithSingleNodeGroups() { FastSearcher fastSearcher = new FastSearcher("container.0", - MockDispatcher.create(Collections.singletonList(new Node(0, "host0", 123, 0))), + MockDispatcher.create(Collections.singletonList(new Node(0, "host0", 0))), new SummaryParameters(null), new ClusterParams("testhittype"), documentdbInfoConfig); @@ -102,7 +102,7 @@ public class FastSearcherTestCase { @Test public void testSinglePassGroupingIsNotForcedWithSingleNodeGroups() { - MockDispatcher dispatcher = MockDispatcher.create(ImmutableList.of(new Node(0, "host0", 123, 0), new Node(2, "host1", 123, 0))); + MockDispatcher dispatcher = MockDispatcher.create(ImmutableList.of(new Node(0, "host0", 0), new Node(2, "host1", 0))); FastSearcher fastSearcher = new FastSearcher("container.0", dispatcher, diff --git a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/MockDispatcher.java b/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/MockDispatcher.java index 0aa91442712..4fbbd9dd936 100644 --- a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/MockDispatcher.java +++ b/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/MockDispatcher.java @@ -39,7 +39,6 @@ class MockDispatcher extends Dispatcher { for (Node node : nodes) { DispatchConfig.Node.Builder dispatchConfigNodeBuilder = new DispatchConfig.Node.Builder(); dispatchConfigNodeBuilder.host(node.hostname()); - dispatchConfigNodeBuilder.fs4port(node.fs4port()); dispatchConfigNodeBuilder.port(0); // Mandatory, but currently not used here dispatchConfigNodeBuilder.group(node.group()); dispatchConfigNodeBuilder.key(key++); // not used diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/DispatcherTest.java b/container-search/src/test/java/com/yahoo/search/dispatch/DispatcherTest.java index 310f536f961..3d544f5c114 100644 --- a/container-search/src/test/java/com/yahoo/search/dispatch/DispatcherTest.java +++ b/container-search/src/test/java/com/yahoo/search/dispatch/DispatcherTest.java @@ -70,7 +70,7 @@ public class DispatcherTest { SearchCluster cl = new MockSearchCluster("1", 0, 0) { @Override public Optional<Node> localCorpusDispatchTarget() { - return Optional.of(new Node(1, "test", 123, 1)); + return Optional.of(new Node(1, "test", 1)); } }; MockInvokerFactory invokerFactory = new MockInvokerFactory(cl, (n, a) -> true); diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/LoadBalancerTest.java b/container-search/src/test/java/com/yahoo/search/dispatch/LoadBalancerTest.java index 1ebf7940f25..0496194f8ed 100644 --- a/container-search/src/test/java/com/yahoo/search/dispatch/LoadBalancerTest.java +++ b/container-search/src/test/java/com/yahoo/search/dispatch/LoadBalancerTest.java @@ -28,7 +28,7 @@ import static org.junit.Assert.assertThat; public class LoadBalancerTest { @Test public void requireThatLoadBalancerServesSingleNodeSetups() { - Node n1 = new Node(0, "test-node1", 0, 0); + Node n1 = new Node(0, "test-node1", 0); SearchCluster cluster = new SearchCluster("a", createDispatchConfig(n1), 1, null); LoadBalancer lb = new LoadBalancer(cluster, true); @@ -41,8 +41,8 @@ public class LoadBalancerTest { @Test public void requireThatLoadBalancerServesMultiGroupSetups() { - Node n1 = new Node(0, "test-node1", 0, 0); - Node n2 = new Node(1, "test-node2", 1, 1); + Node n1 = new Node(0, "test-node1", 0); + Node n2 = new Node(1, "test-node2", 1); SearchCluster cluster = new SearchCluster("a", createDispatchConfig(n1, n2), 1, null); LoadBalancer lb = new LoadBalancer(cluster, true); @@ -55,10 +55,10 @@ public class LoadBalancerTest { @Test public void requireThatLoadBalancerServesClusteredGroups() { - Node n1 = new Node(0, "test-node1", 0, 0); - Node n2 = new Node(1, "test-node2", 1, 0); - Node n3 = new Node(0, "test-node3", 0, 1); - Node n4 = new Node(1, "test-node4", 1, 1); + Node n1 = new Node(0, "test-node1", 0); + Node n2 = new Node(1, "test-node2", 0); + Node n3 = new Node(0, "test-node3", 1); + Node n4 = new Node(1, "test-node4", 1); SearchCluster cluster = new SearchCluster("a", createDispatchConfig(n1, n2, n3, n4), 2, null); LoadBalancer lb = new LoadBalancer(cluster, true); @@ -68,8 +68,8 @@ public class LoadBalancerTest { @Test public void requireThatLoadBalancerReturnsDifferentGroups() { - Node n1 = new Node(0, "test-node1", 0, 0); - Node n2 = new Node(1, "test-node2", 1, 1); + Node n1 = new Node(0, "test-node1", 0); + Node n2 = new Node(1, "test-node2", 1); SearchCluster cluster = new SearchCluster("a", createDispatchConfig(n1, n2), 1, null); LoadBalancer lb = new LoadBalancer(cluster, true); diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/MockInvoker.java b/container-search/src/test/java/com/yahoo/search/dispatch/MockInvoker.java index 2fe434d6f3f..c5fbda7c2f5 100644 --- a/container-search/src/test/java/com/yahoo/search/dispatch/MockInvoker.java +++ b/container-search/src/test/java/com/yahoo/search/dispatch/MockInvoker.java @@ -19,7 +19,7 @@ class MockInvoker extends SearchInvoker { private List<Hit> hits; protected MockInvoker(int key, Coverage coverage) { - super(Optional.of(new Node(key, "?", 0, 0))); + super(Optional.of(new Node(key, "?", 0))); this.coverage = coverage; } diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/MockSearchCluster.java b/container-search/src/test/java/com/yahoo/search/dispatch/MockSearchCluster.java index e3ff54102d4..0bcc30d9b10 100644 --- a/container-search/src/test/java/com/yahoo/search/dispatch/MockSearchCluster.java +++ b/container-search/src/test/java/com/yahoo/search/dispatch/MockSearchCluster.java @@ -38,7 +38,7 @@ public class MockSearchCluster extends SearchCluster { for (int group = 0; group < groups; group++) { List<Node> nodes = new ArrayList<>(); for (int node = 0; node < nodesPerGroup; node++) { - Node n = new Node(dk, "host" + dk, -1, group); + Node n = new Node(dk, "host" + dk, group); n.setWorking(true); nodes.add(n); hostBuilder.put(n.hostname(), n); @@ -124,8 +124,9 @@ public class MockSearchCluster extends SearchCluster { builder.minWaitAfterCoverageFactor(0); builder.maxWaitAfterCoverageFactor(0.5); } + int port = 10000; for (Node n : nodes) { - builder.node(new DispatchConfig.Node.Builder().key(n.key()).host(n.hostname()).port(n.fs4port()).group(n.group())); + builder.node(new DispatchConfig.Node.Builder().key(n.key()).host(n.hostname()).port(port++).group(n.group())); } return new DispatchConfig(builder); } diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/rpc/RpcSearchInvokerTest.java b/container-search/src/test/java/com/yahoo/search/dispatch/rpc/RpcSearchInvokerTest.java index d629bd36bb1..c07bf119782 100644 --- a/container-search/src/test/java/com/yahoo/search/dispatch/rpc/RpcSearchInvokerTest.java +++ b/container-search/src/test/java/com/yahoo/search/dispatch/rpc/RpcSearchInvokerTest.java @@ -36,7 +36,7 @@ public class RpcSearchInvokerTest { var mockClient = parameterCollectorClient(compressionTypeHolder, payloadHolder, lengthHolder); var mockPool = new RpcResourcePool(ImmutableMap.of(7, mockClient.createConnection("foo", 123))); @SuppressWarnings("resource") - var invoker = new RpcSearchInvoker(mockSearcher(), new Node(7, "seven", 77, 1), mockPool); + var invoker = new RpcSearchInvoker(mockSearcher(), new Node(7, "seven", 1), mockPool); Query q = new Query("search/?query=test&hits=10&offset=3"); invoker.sendSearchRequest(q); diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/searchcluster/SearchClusterTest.java b/container-search/src/test/java/com/yahoo/search/dispatch/searchcluster/SearchClusterTest.java index f29d6ddf324..f42185e955f 100644 --- a/container-search/src/test/java/com/yahoo/search/dispatch/searchcluster/SearchClusterTest.java +++ b/container-search/src/test/java/com/yahoo/search/dispatch/searchcluster/SearchClusterTest.java @@ -63,7 +63,7 @@ public class SearchClusterTest { for (String name : nodeNames) { int key = nodes.size() % nodesPergroup; int group = nodes.size() / nodesPergroup; - nodes.add(new Node(key, name, 13333, group)); + nodes.add(new Node(key, name, group)); numDocsPerNode.add(new AtomicInteger(1)); pingCounts.add(new AtomicInteger(0)); } @@ -132,7 +132,7 @@ public class SearchClusterTest { @Override public Callable<Pong> createPinger(Node node, ClusterMonitor<Node> monitor) { - int index = node.group*numPerGroup + node.key(); + int index = node.group() * numPerGroup + node.key(); return new Pinger(activeDocs.get(index), pingCounts.get(index)); } } diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/test/QueryProfileSubstitutionTestCase.java b/container-search/src/test/java/com/yahoo/search/query/profile/test/QueryProfileSubstitutionTestCase.java index ca1447b475a..b3b83b9c07e 100644 --- a/container-search/src/test/java/com/yahoo/search/query/profile/test/QueryProfileSubstitutionTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/query/profile/test/QueryProfileSubstitutionTestCase.java @@ -17,26 +17,56 @@ import static org.junit.Assert.fail; public class QueryProfileSubstitutionTestCase { @Test + public void testSubstitutionOnly() { + QueryProfile p = new QueryProfile("test"); + p.set("message","%{world}", null); + p.set("world", "world", null); + assertEquals("world", p.compile(null).get("message")); + } + + @Test public void testSingleSubstitution() { QueryProfile p = new QueryProfile("test"); p.set("message","Hello %{world}!", null); p.set("world", "world", null); - assertEquals("Hello world!",p.compile(null).get("message")); + assertEquals("Hello world!", p.compile(null).get("message")); - QueryProfile p2=new QueryProfile("test2"); + QueryProfile p2 = new QueryProfile("test2"); p2.addInherited(p); p2.set("world", "universe", null); assertEquals("Hello universe!", p2.compile(null).get("message")); } @Test + public void testRelativeSubstitution() { + QueryProfile p = new QueryProfile("test"); + p.set("message","Hello %{.world}!", null); + p.set("world", "world", null); + assertEquals("Hello world!", p.compile(null).get("message")); + } + + @Test + public void testRelativeSubstitutionNotFound() { + try { + QueryProfile p = new QueryProfile("test"); + p.set("message", "Hello %{.world}!", null); + assertEquals("Hello world!", p.compile(null).get("message")); + fail("Expected exception"); + } + catch (IllegalArgumentException e) { + assertEquals("Invalid query profile 'test': Could not resolve local substitution 'world' in variant DimensionBinding []", + Exceptions.toMessageString(e)); + } + } + + @Test public void testMultipleSubstitutions() { - QueryProfile p=new QueryProfile("test"); + QueryProfile p = new QueryProfile("test"); p.set("message","%{greeting} %{entity}%{exclamation}", null); p.set("greeting","Hola", null); p.set("entity","local group", null); p.set("exclamation","?", null); - assertEquals("Hola local group?",p.compile(null).get("message")); + assertEquals("Hola local group?", p.compile(null).get("message")); QueryProfile p2 = new QueryProfile("test2"); p2.addInherited(p); diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/test/QueryProfileVariantsTestCase.java b/container-search/src/test/java/com/yahoo/search/query/profile/test/QueryProfileVariantsTestCase.java index d9bf4a1db97..3da4558d67c 100644 --- a/container-search/src/test/java/com/yahoo/search/query/profile/test/QueryProfileVariantsTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/query/profile/test/QueryProfileVariantsTestCase.java @@ -978,44 +978,47 @@ public class QueryProfileVariantsTestCase { @Test public void testQueryProfileReferencesWithSubstitution() { - QueryProfile main=new QueryProfile("main"); + QueryProfile main = new QueryProfile("main"); main.setDimensions(new String[] {"x1"}); - QueryProfile referencedMain=new QueryProfile("referencedMain"); + QueryProfile referencedMain = new QueryProfile("referencedMain"); referencedMain.set("r1","%{prefix}mainReferenced-r1", null); // In both referencedMain.set("r2","%{prefix}mainReferenced-r2", null); // Only in this - QueryProfile referencedVariant=new QueryProfile("referencedVariant"); + QueryProfile referencedVariant = new QueryProfile("referencedVariant"); referencedVariant.set("r1","%{prefix}variantReferenced-r1", null); // In both referencedVariant.set("r3","%{prefix}variantReferenced-r3", null); // Only in this + referencedVariant.set("inthis", "local value", null); + referencedVariant.set("r4","This has %{.inthis}", null); // Relative - main.set("a",referencedMain, null); - main.set("a",referencedVariant,new String[] {"x1"}, null); - main.set("prefix","mainPrefix:", null); - main.set("prefix","variantPrefix:",new String[] {"x1"}, null); + main.set("a", referencedMain, null); + main.set("a", referencedVariant,new String[] {"x1"}, null); + main.set("prefix", "mainPrefix:", null); + main.set("prefix", "variantPrefix:", new String[] {"x1"}, null); - Properties properties=new QueryProfileProperties(main.compile(null)); + Properties properties = new QueryProfileProperties(main.compile(null)); // No context - Map<String,Object> listed=properties.listProperties(); - assertEquals(3,listed.size()); - assertEquals("mainPrefix:mainReferenced-r1",listed.get("a.r1")); - assertEquals("mainPrefix:mainReferenced-r2",listed.get("a.r2")); + Map<String,Object> listed = properties.listProperties(); + assertEquals(3, listed.size()); + assertEquals("mainPrefix:mainReferenced-r1", listed.get("a.r1")); + assertEquals("mainPrefix:mainReferenced-r2", listed.get("a.r2")); // Context x=x1 - listed=properties.listProperties(toMap(main,new String[] {"x1"})); - assertEquals(4,listed.size()); - assertEquals("variantPrefix:variantReferenced-r1",listed.get("a.r1")); - assertEquals("variantPrefix:mainReferenced-r2",listed.get("a.r2")); - assertEquals("variantPrefix:variantReferenced-r3",listed.get("a.r3")); + listed = properties.listProperties(toMap(main, new String[] {"x1"})); + assertEquals(6, listed.size()); + assertEquals("variantPrefix:variantReferenced-r1", listed.get("a.r1")); + assertEquals("variantPrefix:mainReferenced-r2", listed.get("a.r2")); + assertEquals("variantPrefix:variantReferenced-r3", listed.get("a.r3")); + assertEquals("This has local value", listed.get("a.r4")); } @Test public void testNewsCase1() { - QueryProfile shortcuts=new QueryProfile("shortcuts"); - shortcuts.setDimensions(new String[] {"custid_1","custid_2","custid_3","custid_4","custid_5","custid_6"}); - shortcuts.set("testout","outside", null); - shortcuts.set("test.out","dotoutside", null); - shortcuts.set("testin","inside",new String[] {"yahoo","ca","sc"}, null); - shortcuts.set("test.in","dotinside",new String[] {"yahoo","ca","sc"}, null); + QueryProfile shortcuts = new QueryProfile("shortcuts"); + shortcuts.setDimensions(new String[] {"custid_1", "custid_2", "custid_3", "custid_4", "custid_5", "custid_6"}); + shortcuts.set("testout", "outside", null); + shortcuts.set("test.out", "dotoutside", null); + shortcuts.set("testin", "inside", new String[] {"yahoo","ca","sc"}, null); + shortcuts.set("test.in", "dotinside", new String[] {"yahoo","ca","sc"}, null); QueryProfile profile=new QueryProfile("default"); profile.setDimensions(new String[] {"custid_1","custid_2","custid_3","custid_4","custid_5","custid_6"}); @@ -1024,10 +1027,10 @@ public class QueryProfileVariantsTestCase { profile.freeze(); Query query = new Query(HttpRequest.createTestRequest("?query=test&custid_1=yahoo&custid_2=ca&custid_3=sc", Method.GET), profile.compile(null)); - assertEquals("outside",query.properties().get("testout")); - assertEquals("dotoutside",query.properties().get("test.out")); - assertEquals("inside",query.properties().get("testin")); - assertEquals("dotinside",query.properties().get("test.in")); + assertEquals("outside", query.properties().get("testout")); + assertEquals("dotoutside", query.properties().get("test.out")); + assertEquals("inside", query.properties().get("testin")); + assertEquals("dotinside", query.properties().get("test.in")); } @Test |