diff options
author | Jon Bratseth <bratseth@yahoo-inc.com> | 2016-06-15 23:09:44 +0200 |
---|---|---|
committer | Jon Bratseth <bratseth@yahoo-inc.com> | 2016-06-15 23:09:44 +0200 |
commit | 72231250ed81e10d66bfe70701e64fa5fe50f712 (patch) | |
tree | 2728bba1131a6f6e5bdf95afec7d7ff9358dac50 /container-search/src/main/java/com/yahoo/search/query/profile |
Publish
Diffstat (limited to 'container-search/src/main/java/com/yahoo/search/query/profile')
40 files changed, 5550 insertions, 0 deletions
diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/AllReferencesQueryProfileVisitor.java b/container-search/src/main/java/com/yahoo/search/query/profile/AllReferencesQueryProfileVisitor.java new file mode 100644 index 00000000000..393aba2b002 --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/query/profile/AllReferencesQueryProfileVisitor.java @@ -0,0 +1,40 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.query.profile; + +import com.yahoo.processing.request.CompoundName; +import com.yahoo.search.query.profile.types.FieldDescription; +import com.yahoo.search.query.profile.types.QueryProfileFieldType; +import com.yahoo.search.query.profile.types.QueryProfileType; + +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +/** + * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a> + */ +final class AllReferencesQueryProfileVisitor extends PrefixQueryProfileVisitor { + + /** A map of query profile types */ + private Set<CompoundName> references = new HashSet<>(); + + public AllReferencesQueryProfileVisitor(CompoundName prefix) { + super(prefix); + } + + @Override + public void onValue(String name, Object value, DimensionBinding binding, QueryProfile owner) {} + + @Override + public void onQueryProfileInsidePrefix(QueryProfile profile, DimensionBinding binding, QueryProfile owner) { + references.add(currentPrefix); + } + + /** Returns the values resulting from this visiting */ + public Set<CompoundName> getResult() { return references; } + + /** Returns false - we are not done until we have seen all */ + public boolean isDone() { return false; } + +} diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/AllTypesQueryProfileVisitor.java b/container-search/src/main/java/com/yahoo/search/query/profile/AllTypesQueryProfileVisitor.java new file mode 100644 index 00000000000..fb9638a958b --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/query/profile/AllTypesQueryProfileVisitor.java @@ -0,0 +1,51 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.query.profile; + +import com.yahoo.processing.request.CompoundName; +import com.yahoo.search.query.profile.types.FieldDescription; +import com.yahoo.search.query.profile.types.QueryProfileFieldType; +import com.yahoo.search.query.profile.types.QueryProfileType; + +import java.util.HashMap; +import java.util.Map; + +/** + * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a> + */ +final class AllTypesQueryProfileVisitor extends PrefixQueryProfileVisitor { + + /** A map of query profile types */ + private Map<CompoundName, QueryProfileType> types = new HashMap<>(); + + public AllTypesQueryProfileVisitor(CompoundName prefix) { + super(prefix); + } + + @Override + public void onValue(String name, Object value, DimensionBinding binding, QueryProfile owner) {} + + + @Override + public void onQueryProfileInsidePrefix(QueryProfile profile, DimensionBinding binding, QueryProfile owner) { + if (profile.getType() != null) + addReachableTypes(currentPrefix, profile.getType()); + } + + private void addReachableTypes(CompoundName name, QueryProfileType type) { + types.put(name, type); + for (FieldDescription fieldDescription : type.fields().values()) { + if ( ! (fieldDescription.getType() instanceof QueryProfileFieldType)) continue; + QueryProfileFieldType fieldType = (QueryProfileFieldType)fieldDescription.getType(); + if (fieldType.getQueryProfileType() !=null) { + addReachableTypes(name.append(fieldDescription.getName()), fieldType.getQueryProfileType()); + } + } + } + + /** Returns the values resulting from this visiting */ + public Map<CompoundName, QueryProfileType> getResult() { return types; } + + /** Returns false - we are not done until we have seen all */ + public boolean isDone() { return false; } + +} diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/AllUnoverridableQueryProfileVisitor.java b/container-search/src/main/java/com/yahoo/search/query/profile/AllUnoverridableQueryProfileVisitor.java new file mode 100644 index 00000000000..65c3480272e --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/query/profile/AllUnoverridableQueryProfileVisitor.java @@ -0,0 +1,45 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.query.profile; + +import com.yahoo.processing.request.CompoundName; + +import java.util.HashSet; +import java.util.Set; + +/** + * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a> + */ +final class AllUnoverridableQueryProfileVisitor extends PrefixQueryProfileVisitor { + + /** A map of query profile types */ + private Set<CompoundName> unoverridables = new HashSet<>(); + + public AllUnoverridableQueryProfileVisitor(CompoundName prefix) { + super(prefix); + } + + @Override + public void onValue(String name, Object value, DimensionBinding binding, QueryProfile owner) { + addUnoverridable(name, currentPrefix.append(name), binding, owner); + } + + @Override + public void onQueryProfileInsidePrefix(QueryProfile profile, DimensionBinding binding, QueryProfile owner) { + addUnoverridable(currentPrefix.last(), currentPrefix, binding, owner); + } + + private void addUnoverridable(String localName, CompoundName fullName, DimensionBinding binding, QueryProfile owner) { + if (owner == null) return; + + Boolean isOverridable = owner.isLocalOverridable(localName, binding); + if (isOverridable != null && ! isOverridable) + unoverridables.add(fullName); + } + + /** Returns the values resulting from this visiting */ + public Set<CompoundName> getResult() { return unoverridables; } + + /** Returns false - we are not done until we have seen all */ + public boolean isDone() { return false; } + +} diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/AllValuesQueryProfileVisitor.java b/container-search/src/main/java/com/yahoo/search/query/profile/AllValuesQueryProfileVisitor.java new file mode 100644 index 00000000000..bef5b00c51b --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/query/profile/AllValuesQueryProfileVisitor.java @@ -0,0 +1,44 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.query.profile; + +import com.yahoo.processing.request.CompoundName; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +/** + * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a> + */ +final class AllValuesQueryProfileVisitor extends PrefixQueryProfileVisitor { + + private Map<String,Object> values=new HashMap<>(); + + /* Lists all values starting at prefix */ + public AllValuesQueryProfileVisitor(CompoundName prefix) { + super(prefix); + } + + public @Override void onValue(String localName, Object value, DimensionBinding binding, QueryProfile owner) { + putValue(localName, value, values); + } + + public @Override void onQueryProfileInsidePrefix(QueryProfile profile, DimensionBinding binding, QueryProfile owner) { + putValue("", profile.getValue(), values); + } + + private final void putValue(String key, Object value, Map<String, Object> values) { + if (value == null) return; + CompoundName fullName = currentPrefix.append(key); + if (fullName.isEmpty()) return; // Avoid putting a non-leaf (subtree) root in the list + if (values.containsKey(fullName.toString())) return; // The first value encountered has priority + values.put(fullName.toString(), value); + } + + /** Returns the values resulting from this visiting */ + public Map<String, Object> getResult() { return values; } + + /** Returns false - we are not done until we have seen all */ + public boolean isDone() { return false; } + +} diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/BackedOverridableQueryProfile.java b/container-search/src/main/java/com/yahoo/search/query/profile/BackedOverridableQueryProfile.java new file mode 100644 index 00000000000..71b27c6da63 --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/query/profile/BackedOverridableQueryProfile.java @@ -0,0 +1,139 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.query.profile; + +import com.yahoo.processing.request.CompoundName; +import com.yahoo.protect.Validator; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * <p>A wrapper of a query profile where overrides to the values in the referenced + * profile can be set.</p> + * + * <p>This is used to allow configured overrides (in a particular referencing profile) of a referenced query profile. + * + * <p>Properties which are defined as not overridable in the type (if any) of the referenced query profile + * cannot be set.</p> + * + * @author bratseth + */ +public class BackedOverridableQueryProfile extends OverridableQueryProfile implements Cloneable { + + /** The backing read only query profile, or null if this is not backed */ + private QueryProfile backingProfile; + + /** + * Creates an overridable profile from the given backing profile. The backing profile will never be + * written to. + * + * @param backingProfile the backing profile, which is assumed read only, never null + */ + public BackedOverridableQueryProfile(QueryProfile backingProfile) { + Validator.ensureNotNull("An overridable query profile must be backed by a real query profile",backingProfile); + setType(backingProfile.getType()); + this.backingProfile=backingProfile; + } + + @Override + public synchronized void freeze() { + super.freeze(); + backingProfile.freeze(); + } + + @Override + protected Object localLookup(String localName, DimensionBinding dimensionBinding) { + Object valueInThis=super.localLookup(localName,dimensionBinding); + if (valueInThis!=null) return valueInThis; + return backingProfile.localLookup(localName,dimensionBinding); + } + + protected Boolean isLocalInstanceOverridable(String localName) { + Boolean valueInThis=super.isLocalInstanceOverridable(localName); + if (valueInThis!=null) return valueInThis; + return backingProfile.isLocalInstanceOverridable(localName); + } + + @Override + protected QueryProfile createSubProfile(String name,DimensionBinding dimensionBinding) { + Object backing=backingProfile.lookup(new CompoundName(name),true,dimensionBinding.createFor(backingProfile.getDimensions())); + if (backing!=null && backing instanceof QueryProfile) + return new BackedOverridableQueryProfile((QueryProfile)backing); + else + return new OverridableQueryProfile(); // Nothing is set in this branch, so nothing to override, but need override checking + } + + /** Returns a clone of this which can be independently overridden, but which refers to the same backing profile */ + @Override + public BackedOverridableQueryProfile clone() { + BackedOverridableQueryProfile clone=(BackedOverridableQueryProfile)super.clone(); + return clone; + } + + /** Returns the query profile backing this */ + public QueryProfile getBacking() { return backingProfile; } + + @Override + public void addInherited(QueryProfile inherited) { + backingProfile.addInherited(inherited); + } + + void addInheritedHere(QueryProfile inherited) { + super.addInherited(inherited); + } + + @Override + protected void visitVariants(boolean allowContent,QueryProfileVisitor visitor,DimensionBinding dimensionBinding) { + super.visitVariants(allowContent, visitor, dimensionBinding); + if (visitor.isDone()) return; + backingProfile.visitVariants(allowContent, visitor, dimensionBinding); + } + + @Override + protected void visitInherited(boolean allowContent,QueryProfileVisitor visitor,DimensionBinding dimensionBinding, QueryProfile owner) { + super.visitInherited(allowContent,visitor,dimensionBinding, owner); + if (visitor.isDone()) return; + backingProfile.visitInherited(allowContent,visitor,dimensionBinding,owner); + } + + /** Returns a value from the content of this: The value in this, or the value from the backing if not set in this */ + protected Object getContent(String localKey) { + Object value=super.getContent(localKey); + if (value!=null) return value; + return backingProfile.getContent(localKey); + } + + /** + * Returns all the content from this: + * All the values in this, and all values in the backing where an overriding value is not set in this + */ + @Override + protected Map<String,Object> getContent() { + Map<String,Object> thisContent=super.getContent(); + Map<String,Object> backingContent=backingProfile.getContent(); + if (thisContent.isEmpty()) return backingContent; // Shortcut + if (backingContent.isEmpty()) return thisContent; // Shortcut + Map<String,Object> content=new HashMap<>(backingContent); + content.putAll(thisContent); + return content; + } + + @Override + public String toString() { + return "overridable wrapper of " + backingProfile.toString(); + } + + @Override + public boolean isExplicit() { + return backingProfile.isExplicit(); + } + + @Override + public List<String> getDimensions() { + List<String> dimensions=super.getDimensions(); + if (dimensions!=null) return dimensions; + return backingProfile.getDimensions(); + } + +} diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/CopyOnWriteContent.java b/container-search/src/main/java/com/yahoo/search/query/profile/CopyOnWriteContent.java new file mode 100644 index 00000000000..3c02677b676 --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/query/profile/CopyOnWriteContent.java @@ -0,0 +1,159 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.query.profile; + +import com.yahoo.component.provider.FreezableClass; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +/** + * A HashMap wrapper which can be cloned without copying the wrapped map. + * Copying of the map is deferred until there is a write access to the wrapped map. + * This may be frozen, at which point no further modifications are allowed. + * Note that <b>until</b> this is cloned, the internal map may be both read and written. + * + * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a> + */ +public class CopyOnWriteContent extends FreezableClass implements Cloneable { + + // TODO: Now that we used CompiledQueryProfiles at runtime we can remove this + + // Possible states: + // WRITABLE: The map can be freely modified - it is only used by this + // -> !isFrozen() && (map!=null || unmodifiableMap==null) + // COPYONWRITE: The map is referred by at least one clone - further modification must cause a copy + // -> !isFrozen() && (map==null && unmodifiableMap!=null) + // FROZEN: No further changes are allowed to the state of this, ever + // -> isFrozen() + + // Possible start states: + // WRITABLE: When created using the public constructor + // COPYONWRITE: When created by cloning + + // Possible state transitions: + // WRITABLE->COPYONWRITE: When this is cloned + // COPYONWRITE->WRITABLE: When a clone is written to + // (COPYONWRITE,WRITABLE)->FROZEN: When a profile is frozen + + /** The modifiable content of this. Null if this is empty or if this is not in the WRITABLE state */ + private Map<String,Object> map=null; + /** + * If map is non-null this is either null (not instantiated yet) or an unmodifiable wrapper of map, + * if map is null this is either null (this is empty) or a reference to the map of the content this was cloned from + */ + private Map<String,Object> unmodifiableMap =null; + + /** Create a WRITABLE, empty instance */ + public CopyOnWriteContent() { + } + + /** Create a COPYONWRITE instance with some initial state */ + private static CopyOnWriteContent createInCopyOnWriteState(Map<String,Object> unmodifiableMap) { + CopyOnWriteContent content=new CopyOnWriteContent(); + content.unmodifiableMap = unmodifiableMap; + return content; + } + + /** Create a WRITABLE instance with some initial state */ + private static CopyOnWriteContent createInWritableState(Map<String,Object> map) { + CopyOnWriteContent content=new CopyOnWriteContent(); + content.map = map; + return content; + } + + @Override + public void freeze() { + // Freeze this + if (unmodifiableMap==null) + unmodifiableMap= map!=null ? Collections.unmodifiableMap(map) : Collections.<String, Object>emptyMap(); + map=null; // just to keep the states simpler + + // Freeze content + for (Map.Entry<String,Object> entry : unmodifiableMap.entrySet()) { + if (entry.getValue() instanceof QueryProfile) + ((QueryProfile)entry.getValue()).freeze(); + } + super.freeze(); + } + + private boolean isEmpty() { + return (map==null || map.isEmpty()) && (unmodifiableMap ==null || unmodifiableMap.isEmpty()); + } + + private boolean isWritable() { + return !isFrozen() && (map!=null || unmodifiableMap==null); + } + + @Override + public CopyOnWriteContent clone() { + if (isEmpty()) return new CopyOnWriteContent(); // No referencing is necessary in this case + if (isDeepUnmodifiable(unmodifiableMap())) { + // Create an instance pointing to this and put both in the COPYONWRITE state + unmodifiableMap(); // Make sure we have an unmodifiable reference to the map below + map=null; // Put this into the COPYONWRITE state (unless it is already frozen, in which case this is a noop) + return createInCopyOnWriteState(unmodifiableMap()); + } + else { + // This contains query profiles, don't try to defer copying + return createInWritableState(deepClone(map)); + } + } + + private boolean isDeepUnmodifiable(Map<String,Object> map) { + for (Object value : map.values()) + if (value instanceof QueryProfile && !((QueryProfile)value).isFrozen()) return false; + return true; // all other values are primitives + } + + /** Deep clones a map - this handles all value types which can be found in a query profile */ + static Map<String,Object> deepClone(Map<String,Object> map) { + if (map==null) return null; + Map<String,Object> mapClone=new HashMap<>(map.size()); + for (Map.Entry<String,Object> entry : map.entrySet()) + mapClone.put(entry.getKey(),QueryProfile.cloneIfNecessary(entry.getValue())); + return mapClone; + } + + + //------- Content access ------------------------------------------------------- + + public Map<String,Object> unmodifiableMap() { + if (isEmpty()) return Collections.emptyMap(); + if (map==null) // in COPYONWRITE or FROZEN state + return unmodifiableMap; + // In WRITABLE state: Create unmodifiable wrapper if necessary and return it + if (unmodifiableMap==null) + unmodifiableMap=Collections.unmodifiableMap(map); + return unmodifiableMap; + } + + public Object get(String key) { + if (map!=null) return map.get(key); + if (unmodifiableMap!=null) return unmodifiableMap.get(key); + return null; + } + + public void put(String key,Object value) { + ensureNotFrozen(); + copyIfNotWritable(); + if (map==null) + map=new HashMap<>(); + map.put(key,value); + } + + public void remove(String key) { + ensureNotFrozen(); + copyIfNotWritable(); + if (map!=null) + map.remove(key); + } + + private void copyIfNotWritable() { + if (isWritable()) return; + // move from COPYONWRITE to WRITABLE state + map=new HashMap<>(unmodifiableMap); // deep clone is not necessary as this map is shallowly modifiable + unmodifiableMap=null; // will be created as needed + } + +} diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/DimensionBinding.java b/container-search/src/main/java/com/yahoo/search/query/profile/DimensionBinding.java new file mode 100644 index 00000000000..9adacee74af --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/query/profile/DimensionBinding.java @@ -0,0 +1,223 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.query.profile; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * An immutable, binding of a list of dimensions to dimension values + * + * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a> + */ +public class DimensionBinding { + + /** The dimensions of this */ + private List<String> dimensions=null; + + /** The values matching those dimensions */ + private DimensionValues values; + + /** The binding from those dimensions to values, and possibly other values */ + private Map<String,String> context; + + public static final DimensionBinding nullBinding = + new DimensionBinding(Collections.<String>unmodifiableList(Collections.<String>emptyList()), DimensionValues.empty, null); + + public static final DimensionBinding invalidBinding = + new DimensionBinding(Collections.<String>unmodifiableList(Collections.<String>emptyList()), DimensionValues.empty, null); + + /** Whether the value array contains only nulls */ + private boolean containsAllNulls; + + /** Creates a binding from a variant and a context. Any of the arguments may be null. */ + public static DimensionBinding createFrom(List<String> dimensions, Map<String,String> context) { + if (dimensions==null || dimensions.size()==0) { + if (context==null) return nullBinding; + if (dimensions==null) return new DimensionBinding(null,DimensionValues.empty,context); // Null, but must preserve context + } + + return new DimensionBinding(dimensions,extractDimensionValues(dimensions,context),context); + } + + /** Creates a binding from a variant and a context. Any of the arguments may be null. */ + public static DimensionBinding createFrom(List<String> dimensions, DimensionValues dimensionValues) { + if (dimensionValues==null || dimensionValues==DimensionValues.empty) return nullBinding; + if (dimensions==null) return new DimensionBinding(null,dimensionValues,null); // Null, but preserve raw material for creating a context later (in createFor) + + return new DimensionBinding(dimensions,dimensionValues,null); + } + + /** Returns a binding for a (possibly) new set of variants. Variants may be null, but not bindings */ + public DimensionBinding createFor(List<String> newDimensions) { + if (newDimensions==null) return this; // Note: Not necessarily null - if no new variants then keep the existing binding + // if (this.context==null && values.length==0) return nullBinding; // No data from which to create a non-null binding + if (this.dimensions==newDimensions) return this; // Avoid creating a new object if the dimensions are the same + + Map<String,String> context=this.context; + if (context==null) + context=this.values.asContext(this.dimensions !=null ? this.dimensions : newDimensions); + return new DimensionBinding(newDimensions,extractDimensionValues(newDimensions,context),context); + } + + /** + * Creates a dimension binding. The dimensions list given should be unmodifiable. + * The array will not be modified. The context is needed in order to convert this binding to another + * given another set of variant dimensions. + */ + private DimensionBinding(List<String> dimensions, DimensionValues values, Map<String,String> context) { + this.dimensions=dimensions; + this.values=values; + this.context = context; + containsAllNulls=values.isEmpty(); + } + + /** Returns a read-only list of the dimensions of this. This value is undefined if this isNull() */ + public List<String> getDimensions() { return dimensions; } + + /** Returns a context created from the dimensions and values of this */ + public Map<String,String> getContext() { + if (context !=null) return context; + context =values.asContext(dimensions); + return context; + } + + /** + * Returns the values for the dimensions of this. This value is undefined if this isEmpty() + * This array is always of the same length as the + * length of the dimension list - missing elements are represented as nulls. + * This is never null but may be empty. + */ + public DimensionValues getValues() { return values; } + + /** Returns true only if this binding is null (contains no values for its dimensions (if any) */ + public boolean isNull() { return dimensions==null || containsAllNulls; } + + /** + * Returns an array of the dimension values corresponding to the dimensions of this from the given context, + * in the corresponding order. The array is always of the same length as the number of dimensions. + * Dimensions which are not set in this context get a null value. + */ + private static DimensionValues extractDimensionValues(List<String> dimensions,Map<String,String> context) { + String[] dimensionValues=new String[dimensions.size()]; + if (context==null || context.size()==0) return DimensionValues.createFrom(dimensionValues); + for (int i=0; i<dimensions.size(); i++) + dimensionValues[i]=context.get(dimensions.get(i)); + return DimensionValues.createFrom(dimensionValues); + } + + /** + * Combines this binding with another if compatible. + * Two bindings are incompatible if + * <ul> + * <li>They contain a different value for the same key, or</li> + * <li>They contain the same pair of dimensions in a different order</li> + * </ul> + * + * @return the combined binding, or the special invalidBinding if these two bindings are incompatible + */ + public DimensionBinding combineWith(DimensionBinding binding) { + List<String> combinedDimensions = combineDimensions(getDimensions(), binding.getDimensions()); + if (combinedDimensions == null) return invalidBinding; + + // not runtime, so assume we don't need to preserve values outside the dimensions + Map<String, String> combinedValues = combineValues(getContext(), binding.getContext()); + if (combinedValues == null) return invalidBinding; + + return DimensionBinding.createFrom(combinedDimensions, combinedValues); + } + + /** + * Returns a combined list of dimensions from two separate lists, + * or null if they are incompatible. + * This is to combine two lists to one such that the partial order in both is preserved + * (or return null if impossible). + */ + private List<String> combineDimensions(List<String> d1, List<String> d2) { + List<String> combined = new ArrayList<>(); + int d1Index = 0, d2Index=0; + while (d1Index < d1.size() && d2Index < d2.size()) { + if (d1.get(d1Index).equals(d2.get(d2Index))) { // agreement on next element + combined.add(d1.get(d1Index)); + d1Index++; + d2Index++; + } + else if ( ! d2.contains(d1.get(d1Index))) { // next in d1 is independent from d2 + combined.add(d1.get(d1Index++)); + } + else if ( ! d1.contains(d2.get(d2Index))) { // next in d2 is independent from d1 + combined.add(d2.get(d2Index++)); + } + else { + return null; // no independent and no agreement + } + } + if (d1Index < d1.size()) + combined.addAll(d1.subList(d1Index, d1.size())); + else if (d2Index < d2.size()) + combined.addAll(d2.subList(d2Index, d2.size())); + + return combined; + } + + /** + * Returns a combined map of dimension values from two separate maps, + * or null if they are incompatible. + */ + private Map<String, String> combineValues(Map<String, String> m1, Map<String, String> m2) { + Map<String, String> combinedValues = new HashMap<>(m1); + for (Map.Entry<String, String> m2Entry : m2.entrySet()) { + if (m2Entry.getValue() == null) continue; + String m1Value = m1.get(m2Entry.getKey()); + if (m1Value != null && ! m1Value.equals(m2Entry.getValue())) + return null; // conflicting values of a key + combinedValues.put(m2Entry.getKey(), m2Entry.getValue()); + } + return combinedValues; + } + + private boolean intersects(List<String> l1, List<String> l2) { + for (String l1Item : l1) + if (l2.contains(l1Item)) + return true; + return false; + } + + /** + * Returns true if <code>this == invalidBinding</code> + */ + public boolean isInvalid() { return this == invalidBinding; } + + @Override + public String toString() { + if (isInvalid()) return "Invalid DimensionBinding"; + if (dimensions==null) return "DimensionBinding []"; + StringBuilder b=new StringBuilder("DimensionBinding ["); + for (int i=0; i<dimensions.size(); i++) { + b.append(dimensions.get(i)).append("=").append(values.get(i)); + if (i<dimensions.size()-1) + b.append(", "); + } + b.append("]"); + return b.toString(); + } + + /** Two bindings are equal if they contain the same dimensions and the same non-null values */ + @Override + public boolean equals(Object o) { + if (o==this) return true; + if (! (o instanceof DimensionBinding)) return false; + DimensionBinding other = (DimensionBinding)o; + if ( ! this.dimensions.equals(other.dimensions)) return false; + if ( ! this.values.equals(other.values)) return false; + return true; + } + + @Override + public int hashCode() { + return dimensions.hashCode() + 17 * values.hashCode(); + } + +} diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/DimensionValues.java b/container-search/src/main/java/com/yahoo/search/query/profile/DimensionValues.java new file mode 100644 index 00000000000..10435c4c6b5 --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/query/profile/DimensionValues.java @@ -0,0 +1,140 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.query.profile; + +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * An immutable set of dimension values. + * Note that this may contain more or fewer values than needed given a set of dimensions. + * Any missing values are treated as null. + */ +public class DimensionValues implements Comparable<DimensionValues> { + + private final String[] values; + + public static final DimensionValues empty=new DimensionValues(new String[] {}); + + public static DimensionValues createFrom(String[] values) { + if (values==null || values.length==0 || containsAllNulls(values)) return empty; + return new DimensionValues(values); + } + + /** + * Creates a set of dimension values, where the input array <b>must</b> be of + * the right size, and where no copying is done. + * + * @param values the dimension values. This need not be normalized to the right size. + * The input array is copied by this. + */ + private DimensionValues(String[] values) { + if (values==null) throw new NullPointerException("Dimension values cannot be null"); + this.values=Arrays.copyOf(values,values.length); + } + + /** Returns true if this is has the same value every place it has a value as the givenValues. */ + public boolean matches(DimensionValues givenValues) { + for (int i=0; i<this.size() || i<givenValues.size() ; i++) + if ( ! matches(this.get(i),givenValues.get(i))) + return false; + return true; + } + + private final boolean matches(String conditionString,String checkString) { + if (conditionString==null) return true; + return conditionString.equals(checkString); + } + + /** + * Implements the sort order of this which is based on specificity + * where dimensions to the left are more significant: + * -1 is returned if this is more specific than other, + * 1 is returned if other is more specific than this, + * 0 is returned if none is more specific than the other. + * <p> + * <b>Note:</b> This ordering is not consistent with equals - it returns 0 when the same dimensions + * are <i>set</i>, regardless of what they are set <i>to</i>. + */ + @Override + public int compareTo(DimensionValues other) { + for (int i=0; i<this.size() || i<other.size(); i++) { + if (get(i)!=null && other.get(i)==null) + return -1; + if (get(i)==null && other.get(i)!=null) + return 1; + } + return 0; + } + + /** Helper method which uses compareTo to return whether this is most specific */ + public boolean isMoreSpecificThan(DimensionValues other) { + return this.compareTo(other)<0; + } + + @Override + public boolean equals(Object o) { + if (this==o) return true; + if ( ! (o instanceof DimensionValues)) return false; + DimensionValues other=(DimensionValues)o; + for (int i=0; i<this.size() || i<other.size(); i++) { + if (get(i)==null) { + if (other.get(i)!=null) return false; + } + else { + if ( ! get(i).equals(other.get(i))) return false; + } + } + return true; + } + + @Override + public int hashCode() { + int hashCode = 0; + int i = 0; + for (String value : values) { + i++; + if (value != null) + hashCode += value.hashCode() * i; + } + return hashCode; + } + + @Override + public String toString() { return Arrays.toString(values); } + + public boolean isEmpty() { + return this==empty; + } + + private static boolean containsAllNulls(String[] values) { + for (String value : values) + if (value!=null) return false; + return true; + } + + public Map<String,String> asContext(List<String> dimensions) { + Map<String,String> context=new HashMap<>(); + if (dimensions==null) return context; + for (int i=0; i<dimensions.size(); i++) { + context.put(dimensions.get(i),get(i)); + } + return context; + } + + /** Returns the string at the given index, <b>or null if it has no value at this index.</b> */ + public String get(int index) { + if (index>=values.length) return null; + return values[index]; + } + + /** Returns the number of values in this (some of which may be null) */ + public int size() { return values.length; } + + /** Returns copy of the values in this in an array */ + public String[] getValues() { + return Arrays.copyOf(values,values.length); + } + +} diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/DumpTool.java b/container-search/src/main/java/com/yahoo/search/query/profile/DumpTool.java new file mode 100644 index 00000000000..b9d631cdd10 --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/query/profile/DumpTool.java @@ -0,0 +1,89 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.query.profile; + +import java.io.File; +import java.util.Map; + +import com.yahoo.yolean.Exceptions; +import com.yahoo.search.Query; +import com.yahoo.search.query.profile.config.QueryProfileXMLReader; + +/** + * A standalone tool for dumping query profile properties + * + * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a> + */ +public class DumpTool { + + /** Creates and returns a dump from some parameters */ + public String resolveAndDump(String... args) { + if (args.length==0 || args[0].startsWith("-")) { + StringBuilder result=new StringBuilder(); + result.append("Dumps all resolved query profile properties for a set of dimension values\n"); + result.append("USAGE: dump [query-profile] [dir]? [parameters]?\n"); + result.append(" and [query-profile] is the name of the query profile to dump the values of\n"); + result.append(" and [dir] is a path to an application package or query profile directory. Default: current dir\n"); + result.append(" and [parameters] is the http request encoded dimension keys used during resolving. Default: none\n"); + result.append("Examples:\n"); + result.append(" dump default\n"); + result.append(" - dumps the 'default' profile non-variant values in the current dir\n"); + result.append(" dump default x=x1&y=y1\n"); + result.append(" - dumps the 'default' profile resolved with dimensions values x=x1 and y=y1 in the current dir\n"); + result.append(" dump default myapppackage\n"); + result.append(" - dumps the 'default' profile non-variant values in myapppackage/search/query-profiles\n"); + result.append(" dump default dev/myprofiles x=x1&y=y1\n"); + result.append(" - dumps the 'default' profile resolved with dimensions values x=x1 and y=y1 in dev/myprofiles\n"); + return result.toString(); + } + + // Find what the arguments means + if (args.length>=3) { + return dump(args[0],args[1],args[2]); + } + else if (args.length==2) { + if (args[1].indexOf("=")>=0) + return dump(args[0],"",args[1]); + else + return dump(args[0],args[1],""); + } + else { // args.length=1 + return dump(args[0],"",""); + } + } + + private String dump(String profileName,String dir,String parameters) { + // Import profiles + if (dir.isEmpty()) + dir="."; + File dirInAppPackage=new File(dir,"search/query-profiles"); + if (dirInAppPackage.exists()) + dir=dirInAppPackage.getPath(); + QueryProfileXMLReader reader = new QueryProfileXMLReader(); + QueryProfileRegistry registry = reader.read(dir); + registry.freeze(); + + // Dump (through query to get wiring & parameter parsing done easily) + Query query = new Query("?" + parameters, registry.compile().findQueryProfile(profileName)); + Map<String,Object> properties=query.properties().listProperties(); + + // Create result + StringBuilder b=new StringBuilder(); + for (Map.Entry<String,Object> property : properties.entrySet()) { + b.append(property.getKey()); + b.append("="); + b.append(property.getValue().toString()); + b.append("\n"); + } + return b.toString(); + } + + public static void main(String... args) { + try { + System.out.print(new DumpTool().resolveAndDump(args)); + } + catch (Exception e) { + System.err.println(Exceptions.toMessageString(e)); + } + } + +} diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/FieldDescriptionQueryProfileVisitor.java b/container-search/src/main/java/com/yahoo/search/query/profile/FieldDescriptionQueryProfileVisitor.java new file mode 100644 index 00000000000..73c0fcd2cb1 --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/query/profile/FieldDescriptionQueryProfileVisitor.java @@ -0,0 +1,70 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.query.profile; + +import com.yahoo.search.query.profile.types.FieldDescription; + +import java.util.List; + +/** + * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a> + */ +final class FieldDescriptionQueryProfileVisitor extends QueryProfileVisitor { + + /** The result, or null if none */ + private FieldDescription result = null; + + private final List<String> name; + + private int nameIndex=-1; + + private boolean enteringContent=false; + + public FieldDescriptionQueryProfileVisitor(List<String> name) { + this.name=name; + } + + @Override + public String getLocalKey() { + return name.get(nameIndex); + } + + @Override + public boolean enter(String name) { + if (nameIndex+2<this.name.size()) { + nameIndex++; + enteringContent=true; + } + else { + enteringContent=false; + } + return enteringContent; + } + + @Override + public void leave(String name) { + nameIndex--; + } + + @Override + public void onValue(String name,Object value, DimensionBinding binding, QueryProfile owner) { + } + + @Override + public void onQueryProfile(QueryProfile profile, DimensionBinding binding, QueryProfile owner) { + if (enteringContent) return; // not at leaf query profile + if (profile.getType() == null) return; + result = profile.getType().getField(name.get(name.size()-1)); + } + + @Override + public boolean isDone() { + return result != null; + } + + public FieldDescription result() { return result; } + + @Override + public String toString() { + return "a query profile type visitor (hash " + hashCode() + ") with current value " + result; + } +} diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/ModelObjectMap.java b/container-search/src/main/java/com/yahoo/search/query/profile/ModelObjectMap.java new file mode 100644 index 00000000000..242c551f876 --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/query/profile/ModelObjectMap.java @@ -0,0 +1,26 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.query.profile; + +import com.yahoo.processing.request.CompoundName; +import com.yahoo.search.query.profile.types.FieldType; +import com.yahoo.search.query.properties.PropertyMap; + +/** + * A map which stores all types which cannot be stored in a query profile + * that is rich model objects. + * <p> + * This map will deep copy not only the model object map, but also each + * clonable member in the map. + * + * @author bratseth + */ +public class ModelObjectMap extends PropertyMap { + + /** Returns true if the class of the value is not acceptable as a query profile value */ + @Override + protected boolean shouldSet(CompoundName name,Object value) { + if (value==null) return true; + return FieldType.fromClass(value.getClass())==null; + } + +} diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/OverridableQueryProfile.java b/container-search/src/main/java/com/yahoo/search/query/profile/OverridableQueryProfile.java new file mode 100644 index 00000000000..5d0bffa1ea8 --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/query/profile/OverridableQueryProfile.java @@ -0,0 +1,51 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.query.profile; + +import com.yahoo.component.ComponentId; +import com.yahoo.processing.request.CompoundName; +import com.yahoo.search.query.profile.types.QueryProfileType; + +/** + * A regular query profile which knows it is storing overrides (not configured profiles) + * and that implements override legality checking. + * + * @author bratseth + */ +public class OverridableQueryProfile extends QueryProfile { + + private static final String simpleClassName = OverridableQueryProfile.class.getSimpleName(); + + /** Creates an unbacked overridable query profile */ + protected OverridableQueryProfile() { + super(ComponentId.createAnonymousComponentId(simpleClassName)); + } + + @Override + protected Object checkAndConvertAssignment(String localName, Object inputValue, QueryProfileRegistry registry) { + Object value=super.checkAndConvertAssignment(localName, inputValue, registry); + if (value!=null && value.getClass() == QueryProfile.class) { // We are assigning a query profile - make it overridable + return new BackedOverridableQueryProfile((QueryProfile)value); + } + return value; + } + + @Override + protected QueryProfile createSubProfile(String name,DimensionBinding binding) { + return new OverridableQueryProfile(); // Nothing is set in this branch, so nothing to override, but need override checking + } + + /** Returns a clone of this which can be independently overridden */ + @Override + public OverridableQueryProfile clone() { + if (isFrozen()) return this; + OverridableQueryProfile clone=(OverridableQueryProfile)super.clone(); + clone.initId(ComponentId.createAnonymousComponentId(simpleClassName)); + return clone; + } + + @Override + public String toString() { + return "an overridable query profile with no backing"; + } + +} diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/PrefixQueryProfileVisitor.java b/container-search/src/main/java/com/yahoo/search/query/profile/PrefixQueryProfileVisitor.java new file mode 100644 index 00000000000..2a22d58d8b7 --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/query/profile/PrefixQueryProfileVisitor.java @@ -0,0 +1,63 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.query.profile; + +import com.yahoo.processing.request.CompoundName; + +/** + * A query profile visitor which keeps track of name prefixes and can skip values outside a given prefix + * + * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a> + */ +abstract class PrefixQueryProfileVisitor extends QueryProfileVisitor { + + /** Only call onValue/onQueryProfile for nodes having this prefix */ + private final CompoundName prefix; + + /** The current prefix, relative to prefix. */ + protected CompoundName currentPrefix = CompoundName.empty; + + private int prefixComponentIndex = -1; + + public PrefixQueryProfileVisitor(CompoundName prefix) { + if (prefix == null) + prefix = CompoundName.empty; + this.prefix = prefix; + } + + @Override + public final void onQueryProfile(QueryProfile profile, DimensionBinding binding, QueryProfile owner) { + if (prefixComponentIndex < prefix.size()) return; // Not in the prefix yet + onQueryProfileInsidePrefix(profile, binding, owner); + } + + protected abstract void onQueryProfileInsidePrefix(QueryProfile profile, DimensionBinding binding, QueryProfile owner); + + @Override + public final boolean enter(String name) { + prefixComponentIndex++; + if (prefixComponentIndex-1 < prefix.size()) return true; // we're in the given prefix, which should not be included in the name + currentPrefix = currentPrefix.append(name); + return true; + } + + @Override + public final void leave(String name) { + prefixComponentIndex--; + if (prefixComponentIndex < prefix.size()) return; // we're in the given prefix, which should not be included in the name + if ( ! name.isEmpty() && ! currentPrefix.isEmpty()) + currentPrefix = currentPrefix.first(currentPrefix.size() - 1); + } + + /** + * Returns the correct prefix component if we are still going down the prefix path, + * or null to get all if we are inside the prefix + */ + @Override + public String getLocalKey() { + if (prefixComponentIndex < prefix.size()) + return prefix.get(prefixComponentIndex); + else + return null; + } + +} diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfile.java b/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfile.java new file mode 100644 index 00000000000..55210717305 --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfile.java @@ -0,0 +1,835 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.query.profile; + +import com.google.common.collect.ImmutableList; +import com.yahoo.component.ComponentId; +import com.yahoo.component.provider.FreezableSimpleComponent; +import com.yahoo.processing.request.CompoundName; +import com.yahoo.processing.request.Properties; +import com.yahoo.search.query.profile.compiled.CompiledQueryProfile; +import com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry; +import com.yahoo.search.query.profile.types.FieldDescription; +import com.yahoo.search.query.profile.types.QueryProfileFieldType; +import com.yahoo.search.query.profile.types.QueryProfileType; + +import java.util.*; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * A query profile is a data container with an id and a class (type). More precisely, it contains + * <ul> + * <li>An id, on the form name:version, where the version is optional, and follows the same rules as for other search container components. + * <li>A class id referring to the class defining this profile (see Query Profile Classes below) + * <li>A (possibly empty) list of ids of inherited query profiles + * <li>A (possibly empty) list of declarative predicates over search request parameters which defines when this query profile is applicable (see Query Profile Selection below) + * <li>The data content, which consists of + * <ul> + * <li>named values + * <li>named references to other profiles + * </ul> + * </ul> + * + * This serves the purpose of an intermediate format between configuration and runtime structures - the runtime + * structure used is QueryProfileProperties. + * + * @author bratseth + */ +public class QueryProfile extends FreezableSimpleComponent implements Cloneable { + + /** Defines the permissible content of this, or null if any content is permissible */ + private QueryProfileType type=null; + + /** The value at this query profile - allows non-fields to have values, e.g a=value1, a.b=value2 */ + private Object value=null; + + /** The variants of this, or null if none */ + private QueryProfileVariants variants=null; + + /** The resolved variant dimensions of this, or null if none or not resolved yet (is resolved at freeze) */ + private List<String> resolvedDimensions=null; + + /** The query profiles inherited by this, or null if none */ + private List<QueryProfile> inherited=null; + + /** The content of this profile. The values may be primitives, substitutable strings or other query profiles */ + private CopyOnWriteContent content=new CopyOnWriteContent(); + + /** + * Field override settings: fieldName→OverrideValue. These overrides the override + * setting in the type (if any) of this field). If there are no query profile level settings, this is null. + */ + private Map<String,Boolean> overridable=null; + + /** + * Creates a new query profile from an id. + * The query profile can be modified freely (but not accessed) until it is {@link #freeze frozen}. + * At that point it becomes readable but unmodifiable, which it stays until it goes out of reference. + */ + public QueryProfile(ComponentId id) { + super(id); + if ( ! id.isAnonymous()) + validateName(id.getName()); + } + + /** Convenience shorthand for new QueryProfile(new ComponentId(idString)) */ + public QueryProfile(String idString) { + this(new ComponentId(idString)); + } + + // ----------------- Public API ------------------------------------------------------------------------------- + + // ----------------- Setters and getters + + /** Returns the type of this or null if it has no type */ + public QueryProfileType getType() { return type; } + + /** Sets the type of this, or set to null to not use any type checking in this profile */ + public void setType(QueryProfileType type) { this.type=type; } + + /** Returns the virtual variants of this, or null if none */ + public QueryProfileVariants getVariants() { return variants; } + + /** + * Returns the list of profiles inherited by this. + * Note that order matters for inherited profiles - variables are resolved depth first in the order found in + * the inherited list. This always returns an unmodifiable list - use addInherited to add. + */ + public List<QueryProfile> inherited() { + if (isFrozen()) return inherited; // Frozen profiles always have an unmodifiable, non-null list + if (inherited==null) return Collections.emptyList(); + return Collections.unmodifiableList(inherited); + } + + /** Adds a profile to the end of the inherited list of this. Throws an exception if this is frozen. */ + public void addInherited(QueryProfile profile) { + addInherited(profile,(DimensionValues)null); + } + + public final void addInherited(QueryProfile profile,String[] dimensionValues) { + addInherited(profile,DimensionValues.createFrom(dimensionValues)); + } + + /** Adds a profile to the end of the inherited list of this for the given variant. Throws an exception if this is frozen. */ + public void addInherited(QueryProfile profile, DimensionValues dimensionValues) { + ensureNotFrozen(); + + DimensionBinding dimensionBinding=DimensionBinding.createFrom(getDimensions(),dimensionValues); + if (dimensionBinding.isNull()) { + if (inherited==null) + inherited=new ArrayList<>(); + inherited.add(profile); + } + else { + if (variants==null) + variants=new QueryProfileVariants(dimensionBinding.getDimensions(), this); + variants.inherit(profile,dimensionBinding.getValues()); + } + } + + /** + * Returns the content fields declared in this (i.e not including those inherited) as a read-only map. + * @throws IllegalStateException if this is frozen + */ + public Map<String,Object> declaredContent() { + ensureNotFrozen(); + return content.unmodifiableMap(); + } + + /** + * Returns if the given field is declared explicitly as overridable or not in this or any <i>nested</i> profiles + * (i.e not including overridable settings <i>inherited</i> and from <i>types</i>). + * + * @param name the (possibly dotted) field name to return + * @param context the context in which the name is resolved, or null if none + * @return true/false if this is declared overridable/not overridable in this instance, null if it is not + * given any value is <i>this</i> profile instance + * @throws IllegalStateException if this is frozen + */ + public Boolean isDeclaredOverridable(String name, Map<String,String> context) { + return isDeclaredOverridable(new CompoundName(name),DimensionBinding.createFrom(getDimensions(),context)); + } + + /** Sets the dimensions over which this may vary. Note: This will erase any currently defined variants */ + public void setDimensions(String[] dimensions) { + ensureNotFrozen(); + variants=new QueryProfileVariants(dimensions, this); + } + + /** Returns the value set at this node, to allow non-leafs to have values. Returns null if none. */ + public Object getValue() { return value; } + + public void setValue(Object value) { + ensureNotFrozen(); + this.value=value; + } + + /** Returns the variant dimensions to be used in this - an unmodifiable list of dimension names */ + public List<String> getDimensions() { + if (isFrozen()) return resolvedDimensions; + if (variants!=null) return variants.getDimensions(); + if (inherited==null) return null; + for (QueryProfile inheritedProfile : inherited) { + List<String> inheritedDimensions=inheritedProfile.getDimensions(); + if (inheritedDimensions!=null) return inheritedDimensions; + } + return null; + } + + // ----------------- Query profile facade API + + /** + * Sets the overridability of a field in this profile, + * this overrides the corresponding setting in the type (if any) + */ + public final void setOverridable(String fieldName, boolean overridable, Map<String,String> context) { + setOverridable(new CompoundName(fieldName), overridable,DimensionBinding.createFrom(getDimensions(), context)); + } + + /** + * Return all objects that start with the given prefix path using no context. Use "" to list all. + * <p> + * For example, if {a.d => "a.d-value" ,a.e => "a.e-value", b.d => "b.d-value", then calling listValues("a") + * will return {"d" => "a.d-value","e" => "a.e-value"} + */ + public final Map<String, Object> listValues(String prefix) { return listValues(new CompoundName(prefix)); } + + /** + * Return all objects that start with the given prefix path using no context. Use "" to list all. + * <p> + * For example, if {a.d => "a.d-value" ,a.e => "a.e-value", b.d => "b.d-value", then calling listValues("a") + * will return {"d" => "a.d-value","e" => "a.e-value"} + */ + public final Map<String, Object> listValues(CompoundName prefix) { return listValues(prefix, null); } + + /** + * Return all objects that start with the given prefix path. Use "" to list all. + * <p> + * For example, if {a.d => "a.d-value" ,a.e => "a.e-value", b.d => "b.d-value", then calling listValues("a") + * will return {"d" => "a.d-value","e" => "a.e-value"} + */ + public final Map<String, Object> listValues(String prefix, Map<String,String> context) { + return listValues(new CompoundName(prefix), context); + } + + /** + * Return all objects that start with the given prefix path. Use "" to list all. + * <p> + * For example, if {a.d => "a.d-value" ,a.e => "a.e-value", b.d => "b.d-value", then calling listValues("a") + * will return {"d" => "a.d-value","e" => "a.e-value"} + */ + public final Map<String, Object> listValues(CompoundName prefix, Map<String,String> context) { + return listValues(prefix, context, null); + } + + /** + * Adds all objects that start with the given path prefix to the given value map. Use "" to list all. + * <p> + * For example, if {a.d => "a.d-value" ,a.e => "a.e-value", b.d => "b.d-value", then calling listValues("a") + * will return {"d" => "a.d-value","e" => "a.e-value"} + */ + public Map<String, Object> listValues(CompoundName prefix, Map<String, String> context, Properties substitution) { + DimensionBinding dimensionBinding=DimensionBinding.createFrom(getDimensions(),context); + + AllValuesQueryProfileVisitor visitor=new AllValuesQueryProfileVisitor(prefix); + accept(visitor,dimensionBinding, null); + Map<String,Object> values=visitor.getResult(); + + if (substitution==null) return values; + for (Map.Entry<String,Object> entry : values.entrySet()) { + if (entry.getValue().getClass()==String.class) continue; // Shortcut + if (entry.getValue() instanceof SubstituteString) + entry.setValue(((SubstituteString)entry.getValue()).substitute(context,substitution)); + } + return values; + } + + /** + * Lists types reachable from this, indexed by the prefix having that type. + * If this is itself typed, this' type will be included with an empty prefix + */ + Map<CompoundName, QueryProfileType> listTypes(CompoundName prefix, Map<String, String> context) { + DimensionBinding dimensionBinding = DimensionBinding.createFrom(getDimensions(), context); + AllTypesQueryProfileVisitor visitor = new AllTypesQueryProfileVisitor(prefix); + accept(visitor, dimensionBinding, null); + return visitor.getResult(); + } + + /** + * Lists references reachable from this. + */ + Set<CompoundName> listReferences(CompoundName prefix, Map<String, String> context) { + DimensionBinding dimensionBinding=DimensionBinding.createFrom(getDimensions(),context); + AllReferencesQueryProfileVisitor visitor=new AllReferencesQueryProfileVisitor(prefix); + accept(visitor,dimensionBinding,null); + return visitor.getResult(); + } + + /** + * Lists every entry (value or reference) reachable from this which is not overridable + */ + Set<CompoundName> listUnoverridable(CompoundName prefix, Map<String, String> context) { + DimensionBinding dimensionBinding = DimensionBinding.createFrom(getDimensions(),context); + AllUnoverridableQueryProfileVisitor visitor = new AllUnoverridableQueryProfileVisitor(prefix); + accept(visitor, dimensionBinding, null); + return visitor.getResult(); + } + + /** + * Returns a value from this query profile by resolving the given name: + * <ul> + * <li>The name up to the first dot is the value looked up in the value of this profile + * <li>The rest of the name (if any) is used as the name to look up in the referenced query profile + * </ul> + * + * If this name does not resolve <i>completely</i> into a value in this or any inherited profile, null is returned. + */ + public final Object get(String name) { return get(name,(Map<String,String>)null); } + + /** Returns a value from this using the given property context for resolution and using this for substitution */ + public final Object get(String name, Map<String,String> context) { + return get(name,context,null); + } + + /** Returns a value from this using the given dimensions for resolution */ + public final Object get(String name, String[] dimensionValues) { + return get(name,dimensionValues,null); + } + + public final Object get(String name, String[] dimensionValues, Properties substitution) { + return get(name,DimensionValues.createFrom(dimensionValues),substitution); + } + + /** Returns a value from this using the given dimensions for resolution */ + public final Object get(String name, DimensionValues dimensionValues, Properties substitution) { + return get(name,DimensionBinding.createFrom(getDimensions(),dimensionValues),substitution); + } + + public final Object get(String name, Map<String,String> context, Properties substitution) { + return get(name,DimensionBinding.createFrom(getDimensions(),context),substitution); + } + + public final Object get(CompoundName name, Map<String,String> context, Properties substitution) { + return get(name,DimensionBinding.createFrom(getDimensions(),context),substitution); + } + + final Object get(String name, DimensionBinding binding,Properties substitution) { + return get(new CompoundName(name),binding,substitution); + } + + final Object get(CompoundName name, DimensionBinding binding, Properties substitution) { + Object node=get(name,binding); + if (node!=null && node.getClass()==String.class) return node; // Shortcut + if (node instanceof SubstituteString) return ((SubstituteString)node).substitute(binding.getContext(),substitution); + return node; + } + + final Object get(CompoundName name,DimensionBinding dimensionBinding) { + return lookup(name,false,dimensionBinding); + } + + /** + * Returns the node at the position prescribed by the given name (without doing substitutions) - + * a primitive value, a substitutable string, a query profile, or null if not found. + */ + public final Object lookup(String name, Map<String,String> context) { + return lookup(new CompoundName(name),true,DimensionBinding.createFrom(getDimensions(),context)); + } + + /** Sets a value in this or any nested profile using null as context */ + public final void set(String name, Object value, QueryProfileRegistry registry) { + set(name,value,(Map<String,String>)null, registry); + } + + /** + * Sets a value in this or any nested profile. Any missing structure needed to set this will be created. + * If this value is already set, this will overwrite the previous value. + * + * @param name the name of the field, possibly a dotted name which will cause setting of a variable in a subprofile + * @param value the value to assign to the name, a primitive wrapper, string or a query profile + * @param context the context used to resolve where this value should be set, or null if none + * @throws IllegalArgumentException if the given name is illegal given the types of this or any nested query profile + * @throws IllegalStateException if this query profile is frozen + */ + public final void set(CompoundName name,Object value,Map<String,String> context, QueryProfileRegistry registry) { + set(name, value, DimensionBinding.createFrom(getDimensions(), context), registry); + } + + public final void set(String name,Object value,Map<String,String> context, QueryProfileRegistry registry) { + set(new CompoundName(name), value, DimensionBinding.createFrom(getDimensions(), context), registry); + } + + public final void set(String name,Object value,String[] dimensionValues, QueryProfileRegistry registry) { + set(name,value,DimensionValues.createFrom(dimensionValues), registry); + } + + /** + * Sets a value in this or any nested profile. Any missing structure needed to set this will be created. + * If this value is already set, this will overwrite the previous value. + * + * @param name the name of the field, possibly a dotted name which will cause setting of a variable in a subprofile + * @param value the value to assign to the name, a primitive wrapper, string or a query profile + * @param dimensionValues the dimension values - will be matched by order to the dimensions set in this - if this is + * shorter or longer than the number of dimensions it will be adjusted as needed + * @param registry the registry used to resolve query profile references. If null is passed query profile references + * will cause an exception + * @throws IllegalArgumentException if the given name is illegal given the types of this or any nested query profile + * @throws IllegalStateException if this query profile is frozen + */ + public final void set(String name,Object value,DimensionValues dimensionValues, QueryProfileRegistry registry) { + set(new CompoundName(name), value, DimensionBinding.createFrom(getDimensions(), dimensionValues), registry); + } + + // ----------------- Misc + + public boolean isExplicit() { + return !getId().isAnonymous(); + } + + /** + * Switches this from write-only to read-only mode. + * This profile can never be modified again after this method returns. + * Calling this on an already frozen profile has no effect. + * <p> + * Calling this will also freeze any profiles inherited and referenced by this. + */ + // TODO: Remove/simplify as query profiles are not used at query time + public synchronized void freeze() { + if (isFrozen()) return; + + resolvedDimensions=getDimensions(); + + if (variants !=null) + variants.freeze(); + + if (inherited!=null) { + for (QueryProfile inheritedProfile : inherited) + inheritedProfile.freeze(); + } + + content.freeze(); + + inherited= inherited==null ? ImmutableList.of() : ImmutableList.copyOf(inherited); + + super.freeze(); + } + + @Override + public String toString() { + return "query profile '" + getId() + "'" + (type!=null ? " of type '" + type.getId() + "'" : ""); + } + + /** + * Returns a clone of this. The clone will not be frozen and will contain copied inherited and content collections + * pointing to the same values as this. + */ + @Override + public QueryProfile clone() { + if (isFrozen()) return this; + QueryProfile clone=(QueryProfile)super.clone(); + if (variants !=null) + clone.variants = variants.clone(); + if (inherited!=null) + clone.inherited=new ArrayList<>(inherited); + + if (this.content!=null) + clone.content=content.clone(); + + return clone; + } + + /** + * Clones a value of a type which may appear in a query profile if cloning is necessary (i.e if it is + * not immutable). Returns the input type otherwise. + */ + static Object cloneIfNecessary(Object object) { + if (object instanceof QueryProfile) return ((QueryProfile)object).clone(); + return object; // Other types are immutable + } + + /** Throws IllegalArgumentException if the given string is not a valid query profile name */ + public static void validateName(String name) { + Matcher nameMatcher=namePattern.matcher(name); + if ( ! nameMatcher.matches()) + throw new IllegalArgumentException("Illegal name '" + name + "'"); + } + + // ----------------- For subclass use -------------------------------------------------------------------- + + /** Override this to intercept all writes to this profile (or any nested profiles) */ + protected void set(CompoundName name, Object value, DimensionBinding binding, QueryProfileRegistry registry) { + try { + setNode(name, value, null, binding, registry); + } + catch (IllegalArgumentException e) { + throw new IllegalArgumentException("Could not set '" + name + "' to '" + value + "'",e); + } + } + + /** Returns this value, or its corresponding substitution string if it contains substitutions */ + protected Object convertToSubstitutionString(Object value) { + if (value==null) return value; + if (value.getClass()!=String.class) return value; + SubstituteString substituteString=SubstituteString.create((String)value); + if (substituteString==null) return value; + return substituteString; + } + + /** Returns the field description of this field, or null if it is not typed */ + protected FieldDescription getFieldDescription(CompoundName name, DimensionBinding binding) { + FieldDescriptionQueryProfileVisitor visitor=new FieldDescriptionQueryProfileVisitor(name.asList()); + accept(visitor, binding,null); + return visitor.result(); + } + + /** + * Returns true if this value is definitely overridable in this (set and not unoverridable), + * false if it is declared unoverridable (in instance or type), and null if this profile has no + * opinion on the matter because the value is not set in this. + */ + Boolean isLocalOverridable(String localName,DimensionBinding binding) { + if (localLookup(localName, binding)==null) return null; // Not set + Boolean isLocalInstanceOverridable=isLocalInstanceOverridable(localName); + if (isLocalInstanceOverridable!=null) + return isLocalInstanceOverridable.booleanValue(); + if (type!=null) return type.isOverridable(localName); + return true; + } + + protected Boolean isLocalInstanceOverridable(String localName) { + if (overridable==null) return null; + return overridable.get(localName); + } + + protected Object lookup(CompoundName name,boolean allowQueryProfileResult, DimensionBinding dimensionBinding) { + SingleValueQueryProfileVisitor visitor=new SingleValueQueryProfileVisitor(name.asList(),allowQueryProfileResult); + accept(visitor,dimensionBinding,null); + return visitor.getResult(); + } + + protected final void accept(QueryProfileVisitor visitor,DimensionBinding dimensionBinding, QueryProfile owner) { + acceptAndEnter("", visitor, dimensionBinding, owner); + } + + void acceptAndEnter(String key, QueryProfileVisitor visitor,DimensionBinding dimensionBinding, QueryProfile owner) { + boolean allowContent=visitor.enter(key); + accept(allowContent, visitor, dimensionBinding, owner); + if (allowContent) + visitor.leave(key); + } + + /** + * Visit the profiles and values referenced from this in order of decreasing precedence + * + * @param allowContent whether content in this should be visited + * @param visitor the visitor + * @param dimensionBinding the dimension binding to use + */ + final void accept(boolean allowContent,QueryProfileVisitor visitor, DimensionBinding dimensionBinding, QueryProfile owner) { + visitor.onQueryProfile(this, dimensionBinding, owner); + if (visitor.isDone()) return; + + visitVariants(allowContent,visitor,dimensionBinding); + if (visitor.isDone()) return; + + if (allowContent) { + visitContent(visitor,dimensionBinding); + if (visitor.isDone()) return; + } + + if (visitor.visitInherited()) + visitInherited(allowContent, visitor, dimensionBinding, owner); + } + + protected void visitVariants(boolean allowContent,QueryProfileVisitor visitor,DimensionBinding dimensionBinding) { + if (getVariants()!=null) + getVariants().accept(allowContent, getType(), visitor, dimensionBinding); + } + + protected void visitInherited(boolean allowContent,QueryProfileVisitor visitor,DimensionBinding dimensionBinding, QueryProfile owner) { + if (inherited==null) return; + for (QueryProfile inheritedProfile : inherited) { + inheritedProfile.accept(allowContent,visitor,dimensionBinding.createFor(inheritedProfile.getDimensions()), owner); + if (visitor.isDone()) return; + } + } + + private void visitContent(QueryProfileVisitor visitor,DimensionBinding dimensionBinding) { + String contentKey=visitor.getLocalKey(); + + // Visit this' content + if (contentKey!=null) { // Get only the content of the current key + if (type!=null) + contentKey=type.unalias(contentKey); + visitor.acceptValue(contentKey, getContent(contentKey), dimensionBinding, this); + } + else { // get all content in this + for (Map.Entry<String,Object> entry : getContent().entrySet()) { + visitor.acceptValue(entry.getKey(), entry.getValue(), dimensionBinding, this); + if (visitor.isDone()) return; + } + } + } + + /** Returns a value from the content of this, or null if not present */ + protected Object getContent(String key) { + return content.get(key); + } + + /** Returns all the content from this as an unmodifiable map */ + protected Map<String,Object> getContent() { + return content.unmodifiableMap(); + } + + /** Sets the value of a node in <i>this</i> profile - the local name given must not be nested (contain dots) */ + protected QueryProfile setLocalNode(String localName, Object value,QueryProfileType parentType, + DimensionBinding dimensionBinding, QueryProfileRegistry registry) { + if (parentType!=null && type==null && !isFrozen()) + type=parentType; + + value=checkAndConvertAssignment(localName, value, registry); + localPut(localName,value,dimensionBinding); + return this; + } + + /** + * Combines an existing and a new value for a query property key. + * Return the new object to add to the state of the owning profile (/variant), or null if no new value needs to + * be added (usually because the new value was added to the existing). + */ + static Object combineValues(Object newValue, Object existingValue) { + if (newValue instanceof QueryProfile) { + QueryProfile newProfile=(QueryProfile)newValue; + if ( existingValue==null || ! (existingValue instanceof QueryProfile)) { + if (!isModifiable(newProfile)) + newProfile=new BackedOverridableQueryProfile(newProfile); // Make the query profile reference overridable + newProfile.value=existingValue; + return newProfile; + } + + // if both are profiles: + return combineProfiles(newProfile,(QueryProfile)existingValue); + } + else { + if (existingValue instanceof QueryProfile) { // we need to set a non-leaf value on a query profile + QueryProfile existingProfile=(QueryProfile)existingValue; + if (isModifiable(existingProfile)) { + existingProfile.setValue(newValue); + return null; + } + else { + QueryProfile existingOverridable = new BackedOverridableQueryProfile((QueryProfile)existingValue); + existingOverridable.setValue(newValue); + return existingOverridable; + } + } + else { + return newValue; + } + } + } + + private static QueryProfile combineProfiles(QueryProfile newProfile,QueryProfile existingProfile) { + QueryProfile returnValue=null; + QueryProfile existingModifiable; + + // Ensure the existing profile is modifiable + if (existingProfile.getClass()==QueryProfile.class) { + existingModifiable = new BackedOverridableQueryProfile(existingProfile); + returnValue=existingModifiable; + } + else { // is an overridable wrapper + existingModifiable=existingProfile; // May be used as-is + } + + // Make the existing profile inherit the new one + if (existingModifiable instanceof BackedOverridableQueryProfile) + ((BackedOverridableQueryProfile)existingModifiable).addInheritedHere(newProfile); + else + existingModifiable.addInherited(newProfile); + + // Remove content from the existing which the new one does not allow overrides of + if (existingModifiable.content!=null) { + for (String key : existingModifiable.content.unmodifiableMap().keySet()) { + if ( ! newProfile.isLocalOverridable(key, null)) { + existingModifiable.content.remove(key); + } + } + } + + return returnValue; + } + + /** Returns whether the given profile may be modified from this profile */ + private static boolean isModifiable(QueryProfile profile) { + if (profile.isFrozen()) return false; + if ( ! profile.isExplicit()) return true; // Implicitly defined from this - ok to modify then + if (! (profile instanceof BackedOverridableQueryProfile)) return false; + return true; + } + + /** + * Converts to the type of the receiving field, if possible and necessary. + * + * @return the value to be assigned: the original or a converted value + * @throws IllegalArgumentException if the assignment is illegal + */ + protected Object checkAndConvertAssignment(String localName, Object value, QueryProfileRegistry registry) { + if (type==null) return value; // no type checking + + FieldDescription fieldDescription=type.getField(localName); + if (fieldDescription==null) { + if (type.isStrict()) + throw new IllegalArgumentException("'" + localName + "' is not declared in " + type + ", and the type is strict"); + return value; + } + + if (registry == null && (fieldDescription.getType() instanceof QueryProfileFieldType)) + throw new IllegalArgumentException("A registry was not passed: Query profile references is not supported"); + Object convertedValue = fieldDescription.getType().convertFrom(value, registry); + if (convertedValue == null) + throw new IllegalArgumentException("'" + value + "' is not a " + fieldDescription.getType().toInstanceDescription()); + return convertedValue; + } + + /** + * Looks up all inherited profiles and adds any that matches this name. + * This default implementation returns an empty profile. + */ + protected QueryProfile createSubProfile(String name,DimensionBinding dimensionBinding) { + QueryProfile queryProfile = new QueryProfile(ComponentId.createAnonymousComponentId(name)); + return queryProfile; + } + + /** Do a variant-aware content lookup in this */ + protected Object localLookup(String name, DimensionBinding dimensionBinding) { + Object node=null; + if ( variants!=null && !dimensionBinding.isNull()) + node=variants.get(name,type,true,dimensionBinding); + if (node==null) + node=content==null ? null : content.get(name); + return node; + } + + // ----------------- Private ---------------------------------------------------------------------------------- + + private Boolean isDeclaredOverridable(CompoundName name,DimensionBinding dimensionBinding) { + QueryProfile parent= lookupParentExact(name, true, dimensionBinding); + if (parent.overridable==null) return null; + return parent.overridable.get(name.last()); + } + + /** + * Sets the overridability of a field in this profile, + * this overrides the corresponding setting in the type (if any) + */ + private void setOverridable(CompoundName fieldName,boolean overridable,DimensionBinding dimensionBinding) { + QueryProfile parent= lookupParentExact(fieldName, true, dimensionBinding); + if (parent.overridable==null) + parent.overridable=new HashMap<>(); + parent.overridable.put(fieldName.last(),overridable); + } + + /** Sets a value to a (possibly non-local) node. The parent query profile holding the value is returned */ + private void setNode(CompoundName name, Object value, QueryProfileType parentType, + DimensionBinding dimensionBinding, QueryProfileRegistry registry) { + ensureNotFrozen(); + if (name.isCompound()) { + QueryProfile parent= getQueryProfileExact(name.first(), true, dimensionBinding); + parent.setNode(name.rest(), value,parentType, dimensionBinding.createFor(parent.getDimensions()), registry); + } + else { + setLocalNode(name.toString(), value,parentType, dimensionBinding, registry); + } + } + + /** + * Looks up and, if necessary, creates, the query profile which should hold the given local name portion of the + * given name. If the name contains no dots, this is returned. + * + * @param name the name of the variable to lookup the parent of + * @param create whether or not to create the parent if it is not present + * @return the parent, or null if not present and created is false + */ + private QueryProfile lookupParentExact(CompoundName name, boolean create, DimensionBinding dimensionBinding) { + CompoundName rest=name.rest(); + if (rest.isEmpty()) return this; + + QueryProfile topmostParent= getQueryProfileExact(name.first(), create, dimensionBinding); + if (topmostParent==null) return null; + return topmostParent.lookupParentExact(rest, create, dimensionBinding.createFor(topmostParent.getDimensions())); + } + + /** + * Returns a query profile from this by name + * + * @param localName the local name of the profile in this, this is never a compound + * @param create whether the profile should be created if missing + * @return the created profile, or null if not present, and create is false + */ + private QueryProfile getQueryProfileExact(String localName, boolean create, DimensionBinding dimensionBinding) { + Object node=localExactLookup(localName, dimensionBinding); + if (node!=null && node instanceof QueryProfile) { + return (QueryProfile)node; + } + if (!create) return null; + + QueryProfile queryProfile=createSubProfile(localName,dimensionBinding); + if (type!=null) { + Class<?> legalClass=type.getValueClass(localName); + if (legalClass==null || ! legalClass.isInstance(queryProfile)) + throw new RuntimeException("'" + localName + "' is not a legal query profile reference name in " + this); + queryProfile.setType(type.getType(localName)); + } + localPut(localName,queryProfile,dimensionBinding); + return queryProfile; + } + + /** Do a variant-aware content lookup in this - without looking in any wrapped content. But by matching variant bindings exactly only */ + private Object localExactLookup(String name,DimensionBinding dimensionBinding) { + if (dimensionBinding.isNull()) return content==null ? null : content.get(name); + if (variants==null) return null; + QueryProfileVariant variant=variants.getVariant(dimensionBinding.getValues(),false); + if (variant==null) return null; + return variant.values().get(name); + } + + /** Sets a value directly in this query profile (unless frozen) */ + private void localPut(String localName,Object value,DimensionBinding dimensionBinding) { + ensureNotFrozen(); + + if (type!=null) + localName=type.unalias(localName); + + validateName(localName); + value=convertToSubstitutionString(value); + + if (dimensionBinding.isNull()) { + Object combinedValue; + if (value instanceof QueryProfile) + combinedValue = combineValues(value,content==null ? null : content.get(localName)); + else + combinedValue = combineValues(value, localLookup(localName, dimensionBinding)); + + if (combinedValue!=null) + content.put(localName,combinedValue); + } + else { + if (variants==null) + variants=new QueryProfileVariants(dimensionBinding.getDimensions(), this); + variants.set(localName,dimensionBinding.getValues(),value); + } + } + + private static final Pattern namePattern=Pattern.compile("[$a-zA-Z_/][-$a-zA-Z0-9_/()]*"); + + /** + * Returns a compiled version of this which produces faster lookup times + * + * @param registry the registry this will be added to by the caller, or null if none + */ + public CompiledQueryProfile compile(CompiledQueryProfileRegistry registry) { + return QueryProfileCompiler.compile(this, registry); + } + +} diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileCompiler.java b/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileCompiler.java new file mode 100644 index 00000000000..795c7655dfb --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileCompiler.java @@ -0,0 +1,140 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.query.profile; + +import com.yahoo.processing.request.CompoundName; +import com.yahoo.search.query.profile.compiled.CompiledQueryProfile; +import com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry; +import com.yahoo.search.query.profile.compiled.DimensionalMap; +import com.yahoo.search.query.profile.types.QueryProfileType; + +import java.util.Collection; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.logging.Level; +import java.util.logging.Logger; + +/** + * Compile a set of query profiles into compiled profiles. + * + * @author bratseth + */ +public class QueryProfileCompiler { + + private static final Logger log = Logger.getLogger(QueryProfileCompiler.class.getName()); + + public static CompiledQueryProfileRegistry compile(QueryProfileRegistry input) { + CompiledQueryProfileRegistry output = new CompiledQueryProfileRegistry(input.getTypeRegistry()); + for (QueryProfile inputProfile : input.allComponents()) { + output.register(compile(inputProfile, output)); + } + return output; + } + + public static CompiledQueryProfile compile(QueryProfile in, CompiledQueryProfileRegistry registry) { + DimensionalMap.Builder<CompoundName, Object> values = new DimensionalMap.Builder<>(); + DimensionalMap.Builder<CompoundName, QueryProfileType> types = new DimensionalMap.Builder<>(); + DimensionalMap.Builder<CompoundName, Object> references = new DimensionalMap.Builder<>(); + DimensionalMap.Builder<CompoundName, Object> unoverridables = new DimensionalMap.Builder<>(); + + // Resolve values for each existing variant and combine into a single data structure + Set<DimensionBindingForPath> variants = new HashSet<>(); + collectVariants(CompoundName.empty, in, DimensionBinding.nullBinding, variants); + variants.add(new DimensionBindingForPath(DimensionBinding.nullBinding, CompoundName.empty)); // if this contains no variants + if (log.isLoggable(Level.FINE)) + log.fine("Compiling " + in.toString() + " having " + variants.size() + " variants"); + int i = 0; + for (DimensionBindingForPath variant : variants) { + if (log.isLoggable(Level.FINER)) + log.finer(" Compiling variant " + i++ + ": " + variant); + for (Map.Entry<String, Object> entry : in.listValues(variant.path(), variant.binding().getContext(), null).entrySet()) + values.put(variant.path().append(entry.getKey()), variant.binding(), entry.getValue()); + for (Map.Entry<CompoundName, QueryProfileType> entry : in.listTypes(variant.path(), variant.binding().getContext()).entrySet()) + types.put(variant.path().append(entry.getKey()), variant.binding(), entry.getValue()); + for (CompoundName reference : in.listReferences(variant.path(), variant.binding().getContext())) + references.put(variant.path().append(reference), variant.binding(), Boolean.TRUE); // Used as a set; value is ignored + for (CompoundName name : in.listUnoverridable(variant.path(), variant.binding().getContext())) + unoverridables.put(variant.path().append(name), variant.binding(), Boolean.TRUE); // Used as a set; value is ignored + } + + return new CompiledQueryProfile(in.getId(), in.getType(), + values.build(), types.build(), references.build(), unoverridables.build(), + registry); + } + + /** + * Returns all the unique combinations of dimension values which have values set reachable from this profile. + * + * @param profile the profile we are collecting the variants of + * @param currentVariant the variant we must have to arrive at this point in the query profile graph + * @param allVariants the set of all variants accumulated so far + */ + private static void collectVariants(CompoundName path, QueryProfile profile, DimensionBinding currentVariant, Set<DimensionBindingForPath> allVariants) { + for (QueryProfile inheritedProfile : profile.inherited()) + collectVariants(path, inheritedProfile, currentVariant, allVariants); + + collectVariantsFromValues(path, profile.getContent(), currentVariant, allVariants); + + collectVariantsInThis(path, profile, currentVariant, allVariants); + if (profile instanceof BackedOverridableQueryProfile) + collectVariantsInThis(path, ((BackedOverridableQueryProfile) profile).getBacking(), currentVariant, allVariants); + } + + private static void collectVariantsInThis(CompoundName path, QueryProfile profile, DimensionBinding currentVariant, Set<DimensionBindingForPath> allVariants) { + QueryProfileVariants profileVariants = profile.getVariants(); + if (profileVariants != null) { + for (QueryProfileVariant variant : profile.getVariants().getVariants()) { + DimensionBinding combinedVariant = + DimensionBinding.createFrom(profile.getDimensions(), variant.getDimensionValues()).combineWith(currentVariant); + if (combinedVariant.isInvalid()) continue; // values at this point in the graph are unreachable + collectVariantsFromValues(path, variant.values(), combinedVariant, allVariants); + for (QueryProfile variantInheritedProfile : variant.inherited()) + collectVariants(path, variantInheritedProfile, combinedVariant, allVariants); + } + } + } + + private static void collectVariantsFromValues(CompoundName path, Map<String, Object> values, DimensionBinding currentVariant, Set<DimensionBindingForPath> allVariants) { + if ( ! values.isEmpty()) + allVariants.add(new DimensionBindingForPath(currentVariant, path)); // there are actual values for this variant + + for (Map.Entry<String, Object> entry : values.entrySet()) { + if (entry.getValue() instanceof QueryProfile) + collectVariants(path.append(entry.getKey()), (QueryProfile)entry.getValue(), currentVariant, allVariants); + } + } + + private static class DimensionBindingForPath { + + private final DimensionBinding binding; + private final CompoundName path; + + public DimensionBindingForPath(DimensionBinding binding, CompoundName path) { + this.binding = binding; + this.path = path; + } + + public DimensionBinding binding() { return binding; } + public CompoundName path() { return path; } + + @Override + public boolean equals(Object o) { + if ( o == this ) return true; + if ( ! (o instanceof DimensionBindingForPath)) return false; + DimensionBindingForPath other = (DimensionBindingForPath)o; + return other.binding.equals(this.binding) && other.path.equals(this.path); + } + + @Override + public int hashCode() { + return binding.hashCode() + 17*path.hashCode(); + } + + @Override + public String toString() { + return binding + " for path " + path; + } + + } + +} diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileProperties.java b/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileProperties.java new file mode 100644 index 00000000000..2432cb2ab33 --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileProperties.java @@ -0,0 +1,258 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.query.profile; + +import com.yahoo.collections.Pair; +import com.yahoo.processing.request.CompoundName; +import com.yahoo.processing.request.properties.PropertyMap; +import com.yahoo.protect.Validator; +import com.yahoo.search.query.Properties; +import com.yahoo.search.query.profile.compiled.CompiledQueryProfile; +import com.yahoo.search.query.profile.compiled.DimensionalValue; +import com.yahoo.search.query.profile.types.FieldDescription; +import com.yahoo.search.query.profile.types.QueryProfileType; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * Properties backed by a query profile. + * This has the scope of one query and is not multithread safe. + * + * @author bratseth + */ +public class QueryProfileProperties extends Properties { + + private final CompiledQueryProfile profile; + + // Note: The priority order is: values has precedence over references + + /** Values which has been overridden at runtime, or null if none */ + private Map<CompoundName, Object> values = null; + /** Query profile references which has been overridden at runtime, or null if none. Earlier values has precedence */ + private List<Pair<CompoundName, CompiledQueryProfile>> references = null; + + /** Creates an instance from a profile, throws an exception if the given profile is null */ + public QueryProfileProperties(CompiledQueryProfile profile) { + Validator.ensureNotNull("The profile wrapped by this cannot be null", profile); + this.profile = profile; + } + + /** Returns the query profile backing this, or null if none */ + public CompiledQueryProfile getQueryProfile() { return profile; } + + /** Gets a value from the query profile, or from the nested profile if the value is null */ + @Override + public Object get(CompoundName name, Map<String,String> context, + com.yahoo.processing.request.Properties substitution) { + name = unalias(name, context); + Object value = null; + if (values != null) + value = values.get(name); + if (value == null) { + Pair<CompoundName, CompiledQueryProfile> reference = findReference(name); + if (reference != null) + return reference.getSecond().get(name.rest(reference.getFirst().size()), context, substitution); // yes; even if null + } + + if (value == null) + value = profile.get(name, context, substitution); + if (value == null) + value = super.get(name, context, substitution); + return value; + } + + /** + * Sets a value in this query profile + * + * @throws IllegalArgumentException if this property cannot be set in the wrapped query profile + */ + @Override + public void set(CompoundName name, Object value, Map<String,String> context) { + // TODO: Refactor + try { + name = unalias(name, context); + + if (context == null) + context = Collections.emptyMap(); + + if ( ! profile.isOverridable(name, context)) return; + + // Check runtime references + Pair<CompoundName, CompiledQueryProfile> runtimeReference = findReference(name); + if (runtimeReference != null && ! runtimeReference.getSecond().isOverridable(name.rest(runtimeReference.getFirst().size()), context)) + return; + + // Check types + if ( ! profile.getTypes().isEmpty()) { + for (int i = 0; i<name.size(); i++) { + QueryProfileType type = profile.getType(name.first(i), context); + if (type == null) continue; + String localName = name.get(i); + FieldDescription fieldDescription = type.getField(localName); + if (fieldDescription == null && type.isStrict()) + throw new IllegalArgumentException("'" + localName + "' is not declared in " + type + ", and the type is strict"); + + // TODO: In addition to strictness, check legality along the way + + if (i == name.size()-1 && fieldDescription != null) { // at the end of the path, check the assignment type + value = fieldDescription.getType().convertFrom(value, profile.getRegistry()); + if (value == null) + throw new IllegalArgumentException("'" + value + "' is not a " + fieldDescription.getType().toInstanceDescription()); + } + } + } + + if (value instanceof String && value.toString().startsWith("ref:")) { + if (profile.getRegistry() == null) + throw new IllegalArgumentException("Runtime query profile references does not work when the " + + "QueryProfileProperties are constructed without a registry"); + String queryProfileId = value.toString().substring(4); + value = profile.getRegistry().findQueryProfile(queryProfileId); + if (value == null) + throw new IllegalArgumentException("Query profile '" + queryProfileId + "' is not found"); + } + + if (value instanceof CompiledQueryProfile) { // this will be due to one of the two clauses above + if (references == null) + references = new ArrayList<>(); + references.add(0, new Pair<>(name, (CompiledQueryProfile)value)); // references set later has precedence - put first + } + else { + if (values == null) + values = new HashMap<>(); + values.put(name, value); + } + } + catch (IllegalArgumentException e) { + throw new IllegalArgumentException("Could not set '" + name + "' to '" + value + "': " + e.getMessage()); // TODO: Nest instead + } + } + + @Override + public Map<String, Object> listProperties(CompoundName path, Map<String,String> context, + com.yahoo.processing.request.Properties substitution) { + path = unalias(path, context); + if (context == null) context = Collections.emptyMap(); + + Map<String, Object> properties = profile.listValues(path, context, substitution); + + properties.putAll(super.listProperties(path, context, substitution)); + + if (references != null) { + for (Pair<CompoundName, CompiledQueryProfile> refEntry : references) { + if ( ! refEntry.getFirst().hasPrefix(path.first(Math.min(refEntry.getFirst().size(), path.size())))) continue; + + CompoundName pathInReference; + CompoundName prefixToReferenceKeys; + if (refEntry.getFirst().size() > path.size()) { + pathInReference = CompoundName.empty; + prefixToReferenceKeys = refEntry.getFirst().rest(path.size()); + } + else { + pathInReference = path.rest(refEntry.getFirst().size()); + prefixToReferenceKeys = CompoundName.empty; + } + for (Map.Entry<String, Object> valueEntry : refEntry.getSecond().listValues(pathInReference, context, substitution).entrySet()) { + properties.put(prefixToReferenceKeys.append(new CompoundName(valueEntry.getKey())).toString(), valueEntry.getValue()); + } + } + + } + + if (values != null) { + for (Map.Entry<CompoundName, Object> entry : values.entrySet()) { + if (entry.getKey().hasPrefix(path)) + properties.put(entry.getKey().rest(path.size()).toString(), entry.getValue()); + } + } + + return properties; + } + + public boolean isComplete(StringBuilder firstMissingName, Map<String,String> context) { + // Are all types reachable from this complete? + if ( ! reachableTypesAreComplete(CompoundName.empty, profile, firstMissingName, context)) + return false; + + // Are all runtime references in this complete? + if (references == null) return true; + for (Pair<CompoundName, CompiledQueryProfile> reference : references) { + if ( ! reachableTypesAreComplete(reference.getFirst(), reference.getSecond(), firstMissingName, context)) + return false; + } + + return true; + } + + private boolean reachableTypesAreComplete(CompoundName prefix, CompiledQueryProfile profile, StringBuilder firstMissingName, Map<String,String> context) { + for (Map.Entry<CompoundName, DimensionalValue<QueryProfileType>> typeEntry : profile.getTypes().entrySet()) { + QueryProfileType type = typeEntry.getValue().get(context); + if (type == null) continue; + if ( ! typeIsComplete(prefix.append(typeEntry.getKey()), type, firstMissingName, context)) + return false; + } + return true; + } + + private boolean typeIsComplete(CompoundName prefix, QueryProfileType type, StringBuilder firstMissingName, Map<String,String> context) { + if (type == null) return true; + for (FieldDescription field : type.fields().values()) { + if ( ! field.isMandatory()) continue; + + CompoundName fieldName = prefix.append(field.getName()); + if ( get(fieldName, null) != null) continue; + if ( hasReference(fieldName)) continue; + + if (profile.getReferences().get(fieldName, context) != null) continue; + + if (firstMissingName != null) + firstMissingName.append(fieldName); + return false; + } + return true; + } + + private boolean hasReference(CompoundName name) { + if (references == null) return false; + for (Pair<CompoundName, CompiledQueryProfile> reference : references) + if (reference.getFirst().equals(name)) + return true; + return false; + } + + private Pair<CompoundName, CompiledQueryProfile> findReference(CompoundName name) { + if (references == null) return null; + for (Pair<CompoundName, CompiledQueryProfile> entry : references) { + if (name.hasPrefix(entry.getFirst())) return entry; + } + return null; + } + + CompoundName unalias(CompoundName name, Map<String,String> context) { + if (profile.getTypes().isEmpty()) return name; + + CompoundName unaliasedName = name; + for (int i = 0; i<name.size(); i++) { + QueryProfileType type = profile.getType(name.first(i), context); + if (type == null) continue; + if (type.aliases() == null) continue; // TODO: Make never null + if (type.aliases().isEmpty()) continue; + String localName = name.get(i); + String unaliasedLocalName = type.unalias(localName); + unaliasedName = unaliasedName.set(i, unaliasedLocalName); + } + return unaliasedName; + } + + @Override + public QueryProfileProperties clone() { + QueryProfileProperties clone = (QueryProfileProperties)super.clone(); + if (this.values != null) + clone.values = PropertyMap.cloneMap(this.values); + return clone; + } + +} diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileRegistry.java b/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileRegistry.java new file mode 100644 index 00000000000..a4bca752d18 --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileRegistry.java @@ -0,0 +1,89 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.query.profile; + +import com.yahoo.component.ComponentSpecification; +import com.yahoo.component.provider.ComponentRegistry; +import com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry; +import com.yahoo.search.query.profile.types.QueryProfileType; +import com.yahoo.search.query.profile.types.QueryProfileTypeRegistry; + +/** + * A set of query profiles. This also holds the query profile types as a dependent registry + * + * @author bratseth + */ +public class QueryProfileRegistry extends ComponentRegistry<QueryProfile> { + + private QueryProfileTypeRegistry queryProfileTypeRegistry = new QueryProfileTypeRegistry(); + + /** The current default instance of this registry */ + private static QueryProfileRegistry instance = new QueryProfileRegistry(); + + /** Register this type by its id */ + public void register(QueryProfile profile) { + super.register(profile.getId(), profile); + } + + /** Returns a query profile type by name, or null if not found */ + public QueryProfileType getType(String type) { + return queryProfileTypeRegistry.getComponent(type); + } + + /** Returns the type registry attached to this */ + public QueryProfileTypeRegistry getTypeRegistry() { return queryProfileTypeRegistry; } + + /** + * <p>Returns a query profile for the given request string, or null if a suitable one is not found.</p> + * + * The request string must be a valid {@link com.yahoo.component.ComponentId} or null. + * + * <p> + * If the string is null, the profile named "default" is returned, or null if that does not exists. + * + * <p> + * The version part (if any) is matched used the usual component version patching rules. + * If the name part matches a query profile name perfectly, that profile is returned. + * If not, and the name is a slash-separated path, the profile with the longest matching left sub-path + * which has a type which allows path mahting is used. If there is no such profile, null is returned. + */ + public QueryProfile findQueryProfile(String idString) { + if (idString==null) return getComponent("default"); + ComponentSpecification id=new ComponentSpecification(idString); + QueryProfile profile=getComponent(id); + if (profile!=null) return profile; + + return findPathParentQueryProfile(new ComponentSpecification(idString)); + } + + private QueryProfile findPathParentQueryProfile(ComponentSpecification id) { + // Try the name with "/" appended - should have the same semantics with path matching + QueryProfile slashedProfile=getComponent(new ComponentSpecification(id.getName() + "/",id.getVersionSpecification())); + if (slashedProfile!=null && slashedProfile.getType()!=null && slashedProfile.getType().getMatchAsPath()) + return slashedProfile; + + // Extract the parent (if any) + int slashIndex=id.getName().lastIndexOf("/"); + if (slashIndex<1) return null; + String parentName=id.getName().substring(0,slashIndex); + if (parentName.equals("")) return null; + + ComponentSpecification parentId=new ComponentSpecification(parentName,id.getVersionSpecification()); + + QueryProfile pathParentProfile=getComponent(parentId); + + if (pathParentProfile!=null && pathParentProfile.getType()!=null && pathParentProfile.getType().getMatchAsPath()) + return pathParentProfile; + return findPathParentQueryProfile(parentId); + } + + /** Freezes this, and all owned query profiles and query profile types */ + public @Override void freeze() { + if (isFrozen()) return; + queryProfileTypeRegistry.freeze(); + for (QueryProfile queryProfile : allComponents()) + queryProfile.freeze(); + } + + public CompiledQueryProfileRegistry compile() { return QueryProfileCompiler.compile(this); } + +} diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileVariant.java b/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileVariant.java new file mode 100644 index 00000000000..42ea4a96d8f --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileVariant.java @@ -0,0 +1,157 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.query.profile; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.yahoo.search.query.profile.types.QueryProfileType; + +import java.util.*; + +/** + * A variant of a query profile + * + * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a> +*/ +public class QueryProfileVariant implements Cloneable, Comparable<QueryProfileVariant> { + + private List<QueryProfile> inherited=null; + + private DimensionValues dimensionValues; + + private Map<String,Object> values; + + private boolean frozen=false; + + private QueryProfile owner; + + public QueryProfileVariant(DimensionValues dimensionValues, QueryProfile owner) { + this.dimensionValues=dimensionValues; + this.owner = owner; + } + + public DimensionValues getDimensionValues() { return dimensionValues; } + + /** + * Returns the live reference to the values of this. This may be modified + * if this is not frozen. + */ + public Map<String,Object> values() { + if (values==null) { + if (frozen) + return Collections.emptyMap(); + else + values=new HashMap<>(); + } + return values; + } + + /** + * Returns the live reference to the inherited profiles of this. This may be modified + * if this is not frozen. + */ + public List<QueryProfile> inherited() { + if (inherited==null) { + if (frozen) + return Collections.emptyList(); + else + inherited=new ArrayList<>(); + } + return inherited; + } + + public void set(String key, Object newValue) { + if (values==null) + values=new HashMap<>(); + + Object oldValue = values.get(key); + + if (oldValue == null) { + values.put(key, newValue); + } else { + Object combinedOrNull = QueryProfile.combineValues(newValue, oldValue); + if (combinedOrNull != null) { + values.put(key, combinedOrNull); + } + } + } + + public void inherit(QueryProfile profile) { + if (inherited==null) + inherited=new ArrayList<>(1); + inherited.add(profile); + } + + /** + * Implements the sort order of this which is based on specificity + * where dimensions to the left are more significant. + * <p> + * <b>Note:</b> This ordering is not consistent with equals - it returns 0 when the same dimensions + * are <i>set</i>, regardless of what they are set <i>to</i>. + */ + public @Override int compareTo(QueryProfileVariant other) { + return this.dimensionValues.compareTo(other.dimensionValues); + } + + public boolean matches(DimensionValues givenDimensionValues) { + return this.dimensionValues.matches(givenDimensionValues); + } + + /** Accepts a visitor to the values of this */ + public void accept(boolean allowContent,QueryProfileType type,QueryProfileVisitor visitor, DimensionBinding dimensionBinding) { + // Visit this + if (allowContent) { + String key=visitor.getLocalKey(); + if (key!=null) { + if (type!=null) + type.unalias(key); + + visitor.acceptValue(key, values().get(key), dimensionBinding, owner); + if (visitor.isDone()) return; + } + else { + for (Map.Entry<String,Object> entry : values().entrySet()) { + visitor.acceptValue(entry.getKey(), entry.getValue(), dimensionBinding, owner); + if (visitor.isDone()) return; + } + } + } + + // Visit inherited + for (QueryProfile profile : inherited()) { + if (visitor.visitInherited()) { + profile.accept(allowContent,visitor,dimensionBinding.createFor(profile.getDimensions()), owner); + } + if (visitor.isDone()) return; + } + } + + public void freeze() { + if (frozen) return; + if (inherited != null) + inherited = ImmutableList.copyOf(inherited); + if (values != null) + values = ImmutableMap.copyOf(values); + frozen=true; + } + + public QueryProfileVariant clone() { + if (frozen) return this; + try { + QueryProfileVariant clone=(QueryProfileVariant)super.clone(); + if (this.inherited!=null) + clone.inherited=new ArrayList<>(this.inherited); // TODO: Deep clone is more correct, but probably does not matter in practice + + clone.values=CopyOnWriteContent.deepClone(this.values); + + return clone; + } + catch (CloneNotSupportedException e) { + throw new RuntimeException(e); + } + } + + public @Override String toString() { + return "query profile variant for " + dimensionValues; + } + +} diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileVariants.java b/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileVariants.java new file mode 100644 index 00000000000..fde851bdc75 --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileVariants.java @@ -0,0 +1,486 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.query.profile; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.yahoo.component.provider.Freezable; +import com.yahoo.search.query.profile.types.QueryProfileType; + +import java.util.*; + +/** + * This class represent a set of query profiles virtually - rather + * than storing and instantiating each profile this structure represents explicitly only + * the values set in the various virtual profiles. The set of virtual profiles are defined by a set of + * <i>dimensions</i>. Values may be set for any point in this multi-dimensional space, and may also be set for + * any regular hyper-region by setting values for any point in certain of these dimensions. + * The set of virtual profiles defined by this consists of all the combinations of dimension points for + * which one or more values is set in this, as well as any possible less specified regions. + * <p> + * A set of virtual profiles are always owned by a single profile, which is also their parent + * in the inheritance hierarchy. + * + * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a> + */ +public class QueryProfileVariants implements Freezable, Cloneable { + + private boolean frozen=false; + + /** Properties indexed by name, to support fast lookup of single values */ + private Map<String,FieldValues> fieldValuesByName=new HashMap<>(); + + /** The inherited profiles for various dimensions settings - a set of fieldvalues of List<QueryProfile> */ + private FieldValues inheritedProfiles=new FieldValues(); + + /** + * Field and inherited profiles sorted by specificity used for all-value visiting. + * This is the same as how the source data looks (apart from the sorting). + */ + private List<QueryProfileVariant> variants=new ArrayList<>(); + + /** + * The names of the dimensions (which are possible properties in the context given on lookup) of this. + * Order matters - more specific values to the left in this list are more significant than more specific values + * to the right + */ + private List<String> dimensions; + + /** The query profile this variants of */ + private QueryProfile owner; + + /** + * Creates a set of virtual query profiles which may return varying values over the set of dimensions given. + * Each dimension is a name for which a key-value may be supplied in the context properties + * on lookup time to influence the value returned. + */ + public QueryProfileVariants(String[] dimensions, QueryProfile owner) { + this(Arrays.asList(dimensions), owner); + } + + /** + * Creates a set of virtual query profiles which may return varying values over the set of dimensions given. + * Each dimension is a name for which a key-value may be supplied in the context properties + * on lookup time to influence the value returned. + * + * @param dimensions the dimension names this may vary over. The list gets owned by this, so it must not be further + * modified from outside). This will not modify the list. + */ + public QueryProfileVariants(List<String> dimensions, QueryProfile owner) { + // Note: This is not made unmodifiable (here or in freeze) because we depend on map identity comparisons of this + // list (in dimensionBinding) for performance reasons. + this.dimensions = dimensions; + this.owner = owner; + } + + /** Irreversibly prevents any further modifications to this */ + public void freeze() { + if (frozen) return; + for (FieldValues fieldValues : fieldValuesByName.values()) + fieldValues.freeze(); + fieldValuesByName = ImmutableMap.copyOf(fieldValuesByName); + inheritedProfiles.freeze(); + + Collections.sort(variants); + for (QueryProfileVariant variant : variants) + variant.freeze(); + variants = ImmutableList.copyOf(variants); + + frozen=true; + } + + @Override + public boolean isFrozen() { + return frozen; + } + + /** Visits the most specific match to the dimension binding of each variable (or the one named by the visitor) */ + void accept(boolean allowContent,QueryProfileType type,QueryProfileVisitor visitor,DimensionBinding dimensionBinding) { + String contentName=null; + if (allowContent) + contentName=visitor.getLocalKey(); + + if (contentName!=null) { + if (type!=null) + contentName=type.unalias(contentName); + acceptSingleValue(contentName,allowContent,visitor,dimensionBinding); // Special cased for performance + } + else { + acceptAllValues(allowContent,visitor,type,dimensionBinding); + } + } + + // PERF: 90% + void acceptSingleValue(String name,boolean allowContent,QueryProfileVisitor visitor,DimensionBinding dimensionBinding) { + FieldValues fieldValues=fieldValuesByName.get(name); + if (fieldValues==null || !allowContent) + fieldValues=new FieldValues(); + + fieldValues.sort(); + inheritedProfiles.sort(); + + int inheritedIndex=0; + int fieldIndex=0; + // Go through both the fields and the inherited profiles at the same time and try the single must specific pick + // from either of the lists at each step + while(fieldIndex<fieldValues.size() || inheritedIndex<inheritedProfiles.size()) { // PERF: 8% - fieldValues.size() + // Get the next most specific from field and inherited + FieldValue fieldValue=fieldValues.getIfExists(fieldIndex); // PERF: 11% - getIfExists + FieldValue inheritedProfileValue=inheritedProfiles.getIfExists(inheritedIndex); // PERF: 11% - getIfExists + + // Try the most specific first, then the other + if (inheritedProfileValue==null || (fieldValue!=null && fieldValue.compareTo(inheritedProfileValue)<=0)) { // Field is most specific, or both are equally specific + if (fieldValue.matches(dimensionBinding.getValues())) { // PERF: 42% - matches, together with the other matches + visitor.acceptValue(name, fieldValue.getValue(), dimensionBinding, owner); + } + if (visitor.isDone()) return; + fieldIndex++; + } + else if (inheritedProfileValue!=null) { // Inherited is most specific at this point + if (inheritedProfileValue.matches(dimensionBinding.getValues())) { // PERF: 42% - matches, together with the other matches + @SuppressWarnings("unchecked") + List<QueryProfile> inheritedProfileList=(List<QueryProfile>)inheritedProfileValue.getValue(); + for (QueryProfile inheritedProfile : inheritedProfileList) { + if (visitor.visitInherited()) { + inheritedProfile.accept(allowContent,visitor,dimensionBinding.createFor(inheritedProfile.getDimensions()), owner); + } + if (visitor.isDone()) return; + } + } + inheritedIndex++; + } + if (visitor.isDone()) return; + } + } + + void acceptAllValues(boolean allowContent,QueryProfileVisitor visitor, QueryProfileType type,DimensionBinding dimensionBinding) { + if (!frozen) + Collections.sort(variants); + for (QueryProfileVariant variant : variants) { + if (variant.matches(dimensionBinding.getValues())) + variant.accept(allowContent,type,visitor,dimensionBinding); + if (visitor.isDone()) return; + } + } + + /** + * Returns the most specific matching value of a name for a given set of <b>canonical</b> dimension values. + * + * @param name the name to return the best matching value of + * @param dimensionBinding the dimension bindings to use in this + */ + public Object get(String name, QueryProfileType type, boolean allowQueryProfileResult, DimensionBinding dimensionBinding) { + SingleValueQueryProfileVisitor visitor=new SingleValueQueryProfileVisitor(Collections.singletonList(name),allowQueryProfileResult); + visitor.enter(""); + accept(true,type,visitor,dimensionBinding); + visitor.leave(""); + return visitor.getResult(); + } + + /** Inherits a particular profile in a variant of this */ + public void inherit(QueryProfile profile,DimensionValues dimensionValues) { + ensureNotFrozen(); + + // Update variant + getVariant(dimensionValues,true).inherit(profile); + + // Update per-variable optimized structure + @SuppressWarnings("unchecked") + List<QueryProfile> inheritedAtDimensionValues=(List<QueryProfile>)inheritedProfiles.getExact(dimensionValues); + if (inheritedAtDimensionValues==null) { + inheritedAtDimensionValues=new ArrayList<>(); + inheritedProfiles.put(dimensionValues,inheritedAtDimensionValues); + } + inheritedAtDimensionValues.add(profile); + } + + /** + * Sets a value to this + * + * @param fieldName the name of the field to set. This cannot be a compound (dotted) name + * @param binding the dimension values for which this value applies. + * The dimensions must be canonicalized, and ownership is transferred to this. + * @param value the value to set + */ + /** + * Sets a value to this + * + * @param fieldName the name of the field to set. This cannot be a compound (dotted) name + * @param dimensionValues the dimension values for which this value applies + * @param value the value to set + */ + public void set(String fieldName,DimensionValues dimensionValues,Object value) { + ensureNotFrozen(); + + // Update variant + getVariant(dimensionValues,true).set(fieldName,value); + + // Update per-variable optimized structure + FieldValues fieldValues=fieldValuesByName.get(fieldName); + if (fieldValues==null) { + fieldValues=new FieldValues(); + fieldValuesByName.put(fieldName,fieldValues); + } + + Object combinedValue=QueryProfile.combineValues(value,fieldValues.getExact(dimensionValues)); + if (combinedValue!=null) + fieldValues.put(dimensionValues,combinedValue); + } + + /** + * Returns the dimensions over which the virtual profiles in this may return different values. + * Each dimension is a name for which a key-value may be supplied in the context properties + * on lookup time to influence the value returned. + * The dimensions may not be modified - the returned list is always read only. + */ + // Note: A performance optimization in DimensionBinding depends on the identity of the list returned from this + public List<String> getDimensions() { return dimensions; } + + /** Returns the map of field values of this indexed by field name. */ + public Map<String,FieldValues> getFieldValues() { return fieldValuesByName; } + + /** Returns the profiles inherited from various variants of this */ + public FieldValues getInherited() { return inheritedProfiles; } + + /** + * Returns all the variants of this, sorted by specificity. This is content as declared. + * The returned list is always unmodifiable. + */ + public List<QueryProfileVariant> getVariants() { + if (frozen) return variants; // Already unmodifiable + return Collections.unmodifiableList(variants); + } + + public QueryProfileVariants clone() { + try { + if (frozen) return this; + QueryProfileVariants clone=(QueryProfileVariants)super.clone(); + clone.inheritedProfiles=inheritedProfiles.clone(); + + clone.variants=new ArrayList<>(); + for (QueryProfileVariant variant : variants) + clone.variants.add(variant.clone()); + + clone.fieldValuesByName=new HashMap<>(); + for (Map.Entry<String,FieldValues> entry : fieldValuesByName.entrySet()) + clone.fieldValuesByName.put(entry.getKey(),entry.getValue().clone(entry.getKey(),clone.variants)); + + return clone; + } + catch (CloneNotSupportedException e) { + throw new RuntimeException(e); + } + } + + /** Throws an IllegalStateException if this is frozen */ + protected void ensureNotFrozen() { + if (frozen) + throw new IllegalStateException(this + " is frozen and cannot be modified"); + } + + /** + * Returns the query profile variant having exactly the given dimensions, and creates it if create is set and + * it is missing + * + * @param dimensionValues the dimension values + * @param create whether or not to create the variant if missing + * @return the profile variant, or null if not found and create is false + */ + public QueryProfileVariant getVariant(DimensionValues dimensionValues,boolean create) { + for (QueryProfileVariant profileVariant : variants) + if (profileVariant.getDimensionValues().equals(dimensionValues)) + return profileVariant; + + // Not found + if (!create) return null; + QueryProfileVariant variant=new QueryProfileVariant(dimensionValues, owner); + variants.add(variant); + return variant; + } + + public static class FieldValues implements Freezable, Cloneable { + + private List<FieldValue> resolutionList=null; + + private boolean frozen=false; + + @Override + public void freeze() { + if (frozen) return; + sort(); + if (resolutionList != null) + resolutionList = ImmutableList.copyOf(resolutionList); + frozen = true; + } + + @Override + public boolean isFrozen() { + return frozen; + } + + public void put(DimensionValues dimensionValues,Object value) { + ensureNotFrozen(); + if (resolutionList==null) resolutionList=new ArrayList<>(); + FieldValue fieldValue=getExactFieldValue(dimensionValues); + if (fieldValue!=null) // Replace + fieldValue.setValue(value); + else + resolutionList.add(new FieldValue(dimensionValues,value)); + } + + /** Returns the value having exactly the given dimensions, or null if none */ + public Object getExact(DimensionValues dimensionValues) { + FieldValue value=getExactFieldValue(dimensionValues); + if (value==null) return null; + return value.getValue(); + } + + /** Returns the field value having exactly the given dimensions, or null if none */ + private FieldValue getExactFieldValue(DimensionValues dimensionValues) { + for (FieldValue fieldValue : asList()) + if (fieldValue.getDimensionValues().equals(dimensionValues)) + return fieldValue; + return null; + } + + /** Returns the field values (values for various dimensions) for this field as a read-only list (never null) */ + public List<FieldValue> asList() { + if (resolutionList==null) return Collections.emptyList(); + return resolutionList; + } + + public FieldValue getIfExists(int index) { + if (index>=size()) return null; + return resolutionList.get(index); + } + + public void sort() { + if (frozen) return ; // sorted already + if (resolutionList!=null) + Collections.sort(resolutionList); + } + + /** Same as asList().size() */ + public int size() { + if (resolutionList==null) return 0; + return resolutionList.size(); + } + + /** Throws an IllegalStateException if this is frozen */ + protected void ensureNotFrozen() { + if (frozen) + throw new IllegalStateException(this + " is frozen and cannot be modified"); + } + + /** Clone by filling in values from the given variants */ + public FieldValues clone(String fieldName,List<QueryProfileVariant> clonedVariants) { + try { + if (frozen) return this; + FieldValues clone=(FieldValues)super.clone(); + + if (resolutionList!=null) { + clone.resolutionList=new ArrayList<>(resolutionList.size()); + for (FieldValue value : resolutionList) + clone.resolutionList.add(value.clone(fieldName,clonedVariants)); + } + + return clone; + } + catch (CloneNotSupportedException e) { + throw new RuntimeException(e); + } + } + + public @Override FieldValues clone() { + try { + if (frozen) return this; + FieldValues clone=(FieldValues)super.clone(); + + if (resolutionList!=null) { + clone.resolutionList=new ArrayList<>(resolutionList.size()); + for (FieldValue value : resolutionList) + clone.resolutionList.add(value.clone()); + } + + return clone; + } + catch (CloneNotSupportedException e) { + throw new RuntimeException(e); + } + } + + } + + public static class FieldValue implements Comparable<FieldValue>, Cloneable { + + private DimensionValues dimensionValues; + private Object value; + + public FieldValue(DimensionValues dimensionValues,Object value) { + this.dimensionValues=dimensionValues; + this.value=value; + } + + /** + * Returns the dimension values for which this value should be used. + * The dimension array is always of the exact size of the dimensions specified by the owning QueryProfileVariants, + * and the values appear in the order defined. "Wildcard" dimensions are represented by a null. + */ + public DimensionValues getDimensionValues() { return dimensionValues; } + + /** Returns the value to use for this set of dimension values */ + public Object getValue() { return value; } + + /** Sets the value to use for this set of dimension values */ + public void setValue(Object value) { this.value=value; } + + public boolean matches(DimensionValues givenDimensionValues) { + return dimensionValues.matches(givenDimensionValues); + } + + /** + * Implements the sort order of this which is based on specificity + * where dimensions to the left are more significant. + * <p> + * <b>Note:</b> This ordering is not consistent with equals - it returns 0 when the same dimensions + * are <i>set</i>, regardless of what they are set <i>to</i>. + */ + public @Override int compareTo(FieldValue other) { + return this.dimensionValues.compareTo(other.dimensionValues); + } + + /** Clone by filling in the value from the given variants */ + public FieldValue clone(String fieldName,List<QueryProfileVariant> clonedVariants) { + try { + FieldValue clone=(FieldValue)super.clone(); + if (this.value instanceof QueryProfile) + clone.value=lookupInVariants(fieldName,dimensionValues,clonedVariants); + // Otherwise the value is immutable, so keep it as-is + return clone; + } + catch (CloneNotSupportedException e) { + throw new RuntimeException(e); + } + } + + public FieldValue clone() { + try { + FieldValue clone=(FieldValue)super.clone(); + clone.value=QueryProfile.cloneIfNecessary(this.value); + return clone; + } + catch (CloneNotSupportedException e) { + throw new RuntimeException(e); + } + } + + private Object lookupInVariants(String fieldName,DimensionValues dimensionValues,List<QueryProfileVariant> variants) { + for (QueryProfileVariant variant : variants) { + if ( ! variant.getDimensionValues().equals(dimensionValues)) continue; + return variant.values().get(fieldName); + } + return null; + } + + } + +} diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileVisitor.java b/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileVisitor.java new file mode 100644 index 00000000000..8cb6bf34021 --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileVisitor.java @@ -0,0 +1,87 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.query.profile; + +/** + * Instances of this is used to visit nodes in a graph of query profiles + * + * <code> + * Visitor are called in the following sequence on each query profile: + * enter=enter(referenceName); + * onQueryProfile(this) + * if (enter) { + * getLocalKey() + * ...calls on nested content found in variants, this and inherited, in that order + * leave(referenceName) + * } + * + * The first enter call will be on the root node, which has an empt reference name. + * </code> + * + * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a> + */ +abstract class QueryProfileVisitor { + + /** + * Called when a new <b>nested</b> profile in the graph is entered. + * This default implementation does nothing but returning true. + * If the node is entered (if true is returned from this), a corresponding {@link #leave(String)} call will happen + * later. + * + * @param name the name this profile is nested as, or the empty string if we are entering the root profile + * @return whether we should visit the content of this node or not + */ + public boolean enter(String name) { return true; } + + /** + * Called when the last {@link #enter(String) entered} nested profile is left. + * That is: One leave call is made for each enter call which returns true, + * but due to nesting those calls are not necessarily alternating. + * This default implementation does nothing. + */ + public void leave(String name) { } + + /** + * Called when a value (not a query profile) is encountered. + * + * @param localName the local name of this value (the full name, if needed, must be reconstructed + * by the information given by the history of {@link #enter(String)} and {@link #leave(String)} calls + * @param value the value + * @param binding the binding this holds for + * @param owner the query profile having this value, or null only when profile is the root profile + */ + public abstract void onValue(String localName, Object value, DimensionBinding binding, QueryProfile owner); + + /** + * Called when a query profile is encountered. + * + * @param profile the query profile reference encountered + * @param binding the binding this holds for + * @param owner the profile making this reference, or null only when profile is the root profile + */ + public abstract void onQueryProfile(QueryProfile profile, DimensionBinding binding, QueryProfile owner); + + /** Returns whether this visitor is done visiting what it needed to visit at this point */ + public abstract boolean isDone(); + + /** Returns whether we should, at this point, visit inherited profiles. This default implementation returns true */ + public boolean visitInherited() { return true; } + + /** + * Returns the current local key which should be visited in the last {@link #enter(String) entered} sub-profile + * (or in the top level profile if none is entered), or null to visit all content + */ + public abstract String getLocalKey(); + + /** Calls onValue or onQueryProfile on this and visits the content if it's a profile */ + final void acceptValue(String key, Object value, DimensionBinding dimensionBinding, QueryProfile owner) { + if (value==null) return; + if (value instanceof QueryProfile) { + QueryProfile queryProfileValue=(QueryProfile)value; + queryProfileValue.acceptAndEnter(key, this, dimensionBinding.createFor(queryProfileValue.getDimensions()), owner); + } + else { + onValue(key, value, dimensionBinding, owner); + } + } + +} diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/SingleValueQueryProfileVisitor.java b/container-search/src/main/java/com/yahoo/search/query/profile/SingleValueQueryProfileVisitor.java new file mode 100644 index 00000000000..6d5d1b0686a --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/query/profile/SingleValueQueryProfileVisitor.java @@ -0,0 +1,76 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.query.profile; + +import java.util.List; + +/** + * Visitor which stores the first non-query-profile value encountered, + * or the first query profile encountered at a stop where we do not have any name components left which can be used to + * visit further subprofiles. Hence this may be used both to get the highest prioritized primitive + * value, or query profile, whichever is encountered first which matches the name. + * <p> + * + * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a> + */ +final class SingleValueQueryProfileVisitor extends QueryProfileVisitor { + + /** The value found, or null if none */ + private Object value=null; + + private final List<String> name; + + private int nameIndex=-1; + + private final boolean allowQueryProfileResult; + + private boolean enteringContent=true; + + public SingleValueQueryProfileVisitor(List<String> name,boolean allowQueryProfileResult) { + this.name=name; + this.allowQueryProfileResult=allowQueryProfileResult; + } + + public @Override String getLocalKey() { + return name.get(nameIndex); + } + + public @Override boolean enter(String name) { + if (nameIndex+1<this.name.size()) { + nameIndex++; + enteringContent=true; + } + else { + enteringContent=false; + } + return enteringContent; + } + + public @Override void leave(String name) { + nameIndex--; + } + + public @Override void onValue(String key,Object value, DimensionBinding binding, QueryProfile owner) { + if (nameIndex==name.size()-1) + this.value=value; + } + + public @Override void onQueryProfile(QueryProfile profile,DimensionBinding binding, QueryProfile owner) { + if (enteringContent) return; // still waiting for content + if (allowQueryProfileResult) + this.value = profile; + else + this.value = profile.getValue(); + } + + public @Override boolean isDone() { + return value!=null; + } + + /** Returns the value found during visiting, or null if none */ + public Object getResult() { return value; } + + public @Override String toString() { + return "a single value visitor (hash " + hashCode() + ") with current value " + value; + } + +} diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/SubstituteString.java b/container-search/src/main/java/com/yahoo/search/query/profile/SubstituteString.java new file mode 100644 index 00000000000..59401592378 --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/query/profile/SubstituteString.java @@ -0,0 +1,127 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.query.profile; + +import com.yahoo.processing.request.Properties; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +/** + * A string which contains one or more elements of the form %{name}, + * where these occurrences are to be replaced by a query profile lookup on name. + * <p> + * This objects does the analysis on creation and provides a (reasonably) fast method of + * performing the actual substitution (at lookup time). + * <p> + * This is a value object. Lookups in this are thread safe. + * + * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a> + */ +public class SubstituteString { + + private final List<Component> components; + private final String stringValue; + + /** + * Returns a new SubstituteString if the given string contains substitutions, null otherwise. + */ + public static SubstituteString create(String value) { + int lastEnd=0; + int start=value.indexOf("%{"); + if (start<0) return null; // Shortcut + List<Component> components=new ArrayList<>(); + while (start>=0) { + int end=value.indexOf("}",start+2); + if (end<0) + throw new IllegalArgumentException("Unterminated value substitution '" + value.substring(start) + "'"); + String propertyName=value.substring(start+2,end); + if (propertyName.indexOf("%{")>=0) + throw new IllegalArgumentException("Unterminated value substitution '" + value.substring(start) + "'"); + components.add(new StringComponent(value.substring(lastEnd,start))); + components.add(new PropertyComponent(propertyName)); + lastEnd=end+1; + start=value.indexOf("%{",lastEnd); + } + components.add(new StringComponent(value.substring(lastEnd,value.length()))); + return new SubstituteString(components, value); + } + + private SubstituteString(List<Component> components, String stringValue) { + this.components = components; + this.stringValue = stringValue; + } + + /** + * Perform the substitution in this, by looking up in the given query profile, + * and returns the resulting string + */ + public String substitute(Map<String,String> context,Properties substitution) { + StringBuilder b=new StringBuilder(); + for (Component component : components) + b.append(component.getValue(context,substitution)); + return b.toString(); + } + + @Override + public int hashCode() { + return stringValue.hashCode(); + } + + @Override + public boolean equals(Object other) { + if (other == this) return true; + if ( ! (other instanceof SubstituteString)) return false; + return this.stringValue.equals(((SubstituteString)other).stringValue); + } + + /** Returns this string in original (unsubstituted) form */ + public @Override String toString() { + return stringValue; + } + + private abstract static class Component { + + protected abstract String getValue(Map<String,String> context,Properties substitution); + + } + + private final static class StringComponent extends Component { + + private final String value; + + public StringComponent(String value) { + this.value=value; + } + + public @Override String getValue(Map<String,String> context,Properties substitution) { + return value; + } + + public @Override String toString() { + return value; + } + + } + + private final static class PropertyComponent extends Component { + + private final String propertyName; + + public PropertyComponent(String propertyName) { + this.propertyName=propertyName; + } + + public @Override String getValue(Map<String,String> context,Properties substitution) { + Object value=substitution.get(propertyName,context,substitution); + if (value==null) return ""; + return String.valueOf(value); + } + + public @Override String toString() { + return "%{" + propertyName + "}"; + } + + } + +} diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/compiled/Binding.java b/container-search/src/main/java/com/yahoo/search/query/profile/compiled/Binding.java new file mode 100644 index 00000000000..a440365ceba --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/query/profile/compiled/Binding.java @@ -0,0 +1,128 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.query.profile.compiled; + +import com.yahoo.search.query.profile.DimensionBinding; + +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +/** + * An immutable binding of a set of dimensions to values. + * This binding is minimal in that it only includes dimensions which actually have values. + * + * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a> + */ +public class Binding implements Comparable<Binding> { + + private static final int maxDimensions = 31; + + /** + * A higher number means this is more general. This accounts for both the number and position of the bindings + * in the dimensional space, such that bindings in earlier dimensions are matched before bindings in + * later dimensions + */ + private final int generality; + + /** The dimensions of this. Unenforced invariant: Content never changes. */ + private final String[] dimensions; + + /** The values of those dimensions. Unenforced invariant: Content never changes. */ + private final String[] dimensionValues; + + private final int hashCode; + + @SuppressWarnings("unchecked") + public static final Binding nullBinding= new Binding(Integer.MAX_VALUE, Collections.<String,String>emptyMap()); + + public static Binding createFrom(DimensionBinding dimensionBinding) { + if (dimensionBinding.getDimensions().size() > maxDimensions) + throw new IllegalArgumentException("More than 31 dimensions is not supported"); + + int generality = 0; + Map<String, String> context = new HashMap<>(); + if (dimensionBinding.getDimensions() == null || dimensionBinding.getDimensions().isEmpty()) { // TODO: Just have this return the nullBinding + generality = Integer.MAX_VALUE; + } + else { + for (int i = 0; i <= maxDimensions; i++) { + String value = i < dimensionBinding.getDimensions().size() ? dimensionBinding.getValues().get(i) : null; + if (value == null) + generality += Math.pow(2, maxDimensions - i-1); + else + context.put(dimensionBinding.getDimensions().get(i), value); + } + } + return new Binding(generality, context); + } + + private Binding(int generality, Map<String, String> binding) { + this.generality = generality; + + // Map -> arrays to limit memory consumption and speed up evaluation + dimensions = new String[binding.size()]; + dimensionValues = new String[binding.size()]; + + int i = 0; + int bindingHash = 0; + for (Map.Entry<String,String> entry : binding.entrySet()) { + dimensions[i] = entry.getKey(); + dimensionValues[i] = entry.getValue(); + bindingHash += i * entry.getKey().hashCode() + 11 * i * entry.getValue().hashCode(); + i++; + } + this.hashCode = bindingHash; + } + + /** Returns true only if this binding is null (contains no values for its dimensions (if any) */ + public boolean isNull() { return dimensions.length == 0; } + + @Override + public String toString() { + StringBuilder b = new StringBuilder("Binding["); + for (int i = 0; i < dimensions.length; i++) + b.append(dimensions[i]).append("=").append(dimensionValues[i]).append(","); + if (dimensions.length > 0) + b.setLength(b.length()-1); + b.append("] (generality " + generality + ")"); + return b.toString(); + } + + /** Returns whether the given binding has exactly the same values as this */ + @Override + public boolean equals(Object o) { + if (o == this) return true; + if (! (o instanceof Binding)) return false; + Binding other = (Binding)o; + return Arrays.equals(this.dimensions, other.dimensions) + && Arrays.equals(this.dimensionValues, other.dimensionValues); + } + + @Override + public int hashCode() { return hashCode; } + + /** + * Returns true if all the dimension values in this have the same values + * in the given context. + */ + public boolean matches(Map<String,String> context) { + for (int i = 0; i < dimensions.length; i++) { + if ( ! dimensionValues[i].equals(context.get(dimensions[i]))) return false; + } + return true; + } + + /** + * Implements a partial ordering where more specific bindings come before less specific ones, + * taking both the number of bindings and their positions into account (earlier dimensions + * take precedence over later ones. + * <p> + * The order is not well defined for bindings in different dimensional spaces. + */ + @Override + public int compareTo(Binding other) { + return Integer.compare(this.generality, other.generality); + } + +} diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/compiled/CompiledQueryProfile.java b/container-search/src/main/java/com/yahoo/search/query/profile/compiled/CompiledQueryProfile.java new file mode 100644 index 00000000000..a4056ee55a2 --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/query/profile/compiled/CompiledQueryProfile.java @@ -0,0 +1,183 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.query.profile.compiled; + +import com.yahoo.component.AbstractComponent; +import com.yahoo.component.ComponentId; +import com.yahoo.processing.request.CompoundName; +import com.yahoo.processing.request.Properties; +import com.yahoo.search.query.profile.QueryProfileProperties; +import com.yahoo.search.query.profile.SubstituteString; +import com.yahoo.search.query.profile.types.QueryProfileType; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * A query profile in a state where it is optimized for fast lookups. + * + * @author bratseth + */ +public class CompiledQueryProfile extends AbstractComponent implements Cloneable { + + private static final Pattern namePattern=Pattern.compile("[$a-zA-Z_/][-$a-zA-Z0-9_/()]*"); + + private final CompiledQueryProfileRegistry registry; + + /** The type of this, or null if none */ + private final QueryProfileType type; + + /** The values of this */ + private final DimensionalMap<CompoundName, Object> entries; + + /** Keys which have a type in this */ + private final DimensionalMap<CompoundName, QueryProfileType> types; + + /** Keys which are (typed or untyped) references to other query profiles in this. Used as a set. */ + private final DimensionalMap<CompoundName, Object> references; + + /** Values which are not overridable in this. Used as a set. */ + private final DimensionalMap<CompoundName, Object> unoverridables; + + /** + * Creates a new query profile from an id. + */ + public CompiledQueryProfile(ComponentId id, QueryProfileType type, + DimensionalMap<CompoundName, Object> entries, + DimensionalMap<CompoundName, QueryProfileType> types, + DimensionalMap<CompoundName, Object> references, + DimensionalMap<CompoundName, Object> unoverridables, + CompiledQueryProfileRegistry registry) { + super(id); + this.registry = registry; + if (type != null) + type.freeze(); + this.type = type; + this.entries = entries; + this.types = types; + this.references = references; + this.unoverridables = unoverridables; + if ( ! id.isAnonymous()) + validateName(id.getName()); + } + + // ----------------- Public API ------------------------------------------------------------------------------- + + /** Returns the registry this belongs to, or null if none (in which case runtime profile reference assignment won't work) */ + public CompiledQueryProfileRegistry getRegistry() { return registry; } + + /** Returns the type of this or null if it has no type */ + // TODO: Move into below + public QueryProfileType getType() { return type; } + + /** + * Returns whether or not the given field name can be overridden at runtime. + * Attempts to override values which cannot be overridden will not fail but be ignored. + * Default: true. + * + * @param name the name of the field to check + * @param context the context in which to check, or null if none + */ + public final boolean isOverridable(CompoundName name, Map<String, String> context) { + return unoverridables.get(name, context) == null; + } + + /** Returns the type of a given prefix reachable from this profile, or null if none */ + public final QueryProfileType getType(CompoundName name, Map<String, String> context) { + return types.get(name, context); + } + + /** Returns the types reachable from this, or an empty map (never null) if none */ + public DimensionalMap<CompoundName, QueryProfileType> getTypes() { return types; } + + /** Returns the references reachable from this, or an empty map (never null) if none */ + public DimensionalMap<CompoundName, Object> getReferences() { return references; } + + /** + * Return all objects that start with the given prefix path using no context. Use "" to list all. + * <p> + * For example, if {a.d => "a.d-value" ,a.e => "a.e-value", b.d => "b.d-value", then calling listValues("a") + * will return {"d" => "a.d-value","e" => "a.e-value"} + */ + public final Map<String, Object> listValues(final CompoundName prefix) { return listValues(prefix, Collections.<String,String>emptyMap()); } + public final Map<String, Object> listValues(final String prefix) { return listValues(new CompoundName(prefix)); } + /** + * Return all objects that start with the given prefix path. Use "" to list all. + * <p> + * For example, if {a.d => "a.d-value" ,a.e => "a.e-value", b.d => "b.d-value", then calling listValues("a") + * will return {"d" => "a.d-value","e" => "a.e-value"} + */ + public final Map<String, Object> listValues(final String prefix,Map<String,String> context) { + return listValues(new CompoundName(prefix), context); + } + /** + * Return all objects that start with the given prefix path. Use "" to list all. + * <p> + * For example, if {a.d => "a.d-value" ,a.e => "a.e-value", b.d => "b.d-value", then calling listValues("a") + * will return {"d" => "a.d-value","e" => "a.e-value"} + */ + public final Map<String, Object> listValues(final CompoundName prefix,Map<String,String> context) { + return listValues(prefix, context, null); + } + /** + * Adds all objects that start with the given path prefix to the given value map. Use "" to list all. + * <p> + * For example, if {a.d => "a.d-value" ,a.e => "a.e-value", b.d => "b.d-value", then calling listValues("a") + * will return {"d" => "a.d-value","e" => "a.e-value"} + */ + public Map<String, Object> listValues(CompoundName prefix, Map<String,String> context, Properties substitution) { + Map<String, Object> values = new HashMap<>(); + for (Map.Entry<CompoundName, DimensionalValue<Object>> entry : entries.entrySet()) { + if ( entry.getKey().size() <= prefix.size()) continue; + if ( ! entry.getKey().hasPrefix(prefix)) continue; + + Object value = entry.getValue().get(context); + if (value == null) continue; + + value = substitute(value, context, substitution); + CompoundName suffixName = entry.getKey().rest(prefix.size()); + values.put(suffixName.toString(), value); + } + return values; + } + + public final Object get(String name) { + return get(name, Collections.<String,String>emptyMap()); + } + public final Object get(String name, Map<String,String> context) { + return get(name, context, new QueryProfileProperties(this)); + } + public final Object get(String name, Map<String,String> context, Properties substitution) { + return get(new CompoundName(name), context, substitution); + } + public final Object get(CompoundName name, Map<String, String> context, Properties substitution) { + return substitute(entries.get(name, context), context, substitution); + } + + private Object substitute(Object value, Map<String,String> context, Properties substitution) { + if (value == null) return value; + if (substitution == null) return value; + if (value.getClass() != SubstituteString.class) return value; + return ((SubstituteString)value).substitute(context, substitution); + } + + /** Throws IllegalArgumentException if the given string is not a valid query profile name */ + private static void validateName(String name) { + Matcher nameMatcher=namePattern.matcher(name); + if ( ! nameMatcher.matches()) + throw new IllegalArgumentException("Illegal name '" + name + "'"); + } + + @Override + public CompiledQueryProfile clone() { + return this; // immutable + } + + @Override + public String toString() { + return "query profile '" + getId() + "'" + (type!=null ? " of type '" + type.getId() + "'" : ""); + } + +} diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/compiled/CompiledQueryProfileRegistry.java b/container-search/src/main/java/com/yahoo/search/query/profile/compiled/CompiledQueryProfileRegistry.java new file mode 100644 index 00000000000..91a81888267 --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/query/profile/compiled/CompiledQueryProfileRegistry.java @@ -0,0 +1,76 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.query.profile.compiled; + +import com.yahoo.component.ComponentSpecification; +import com.yahoo.component.provider.ComponentRegistry; +import com.yahoo.search.query.profile.types.QueryProfileType; +import com.yahoo.search.query.profile.types.QueryProfileTypeRegistry; + +/** + * A set of compiled query profiles. + * + * @author bratseth + */ +public class CompiledQueryProfileRegistry extends ComponentRegistry<CompiledQueryProfile> { + + private final QueryProfileTypeRegistry typeRegistry; + + /** Creates a compiled query profile registry with no types */ + public CompiledQueryProfileRegistry() { + this(QueryProfileTypeRegistry.emptyFrozen()); + } + + public CompiledQueryProfileRegistry(QueryProfileTypeRegistry typeRegistry) { + this.typeRegistry = typeRegistry; + } + + /** Registers a type by its id */ + public void register(CompiledQueryProfile profile) { + super.register(profile.getId(), profile); + } + + public QueryProfileTypeRegistry getTypeRegistry() { return typeRegistry; } + + /** + * <p>Returns a query profile for the given request string, or null if a suitable one is not found.</p> + * + * The request string must be a valid {@link com.yahoo.component.ComponentId} or null.<br> + * If the string is null, the profile named "default" is returned, or null if that does not exists. + * + * <p> + * The version part (if any) is matched used the usual component version patching rules. + * If the name part matches a query profile name perfectly, that profile is returned. + * If not, and the name is a slash-separated path, the profile with the longest matching left sub-path + * which has a type which allows path matching is used. If there is no such profile, null is returned. + */ + public CompiledQueryProfile findQueryProfile(String idString) { + if (idString==null || idString.isEmpty()) return getComponent("default"); + ComponentSpecification id=new ComponentSpecification(idString); + CompiledQueryProfile profile=getComponent(id); + if (profile!=null) return profile; + + return findPathParentQueryProfile(new ComponentSpecification(idString)); + } + + private CompiledQueryProfile findPathParentQueryProfile(ComponentSpecification id) { + // Try the name with "/" appended - should have the same semantics with path matching + CompiledQueryProfile slashedProfile=getComponent(new ComponentSpecification(id.getName() + "/",id.getVersionSpecification())); + if (slashedProfile!=null && slashedProfile.getType()!=null && slashedProfile.getType().getMatchAsPath()) + return slashedProfile; + + // Extract the parent (if any) + int slashIndex=id.getName().lastIndexOf("/"); + if (slashIndex<1) return null; + String parentName=id.getName().substring(0,slashIndex); + if (parentName.equals("")) return null; + + ComponentSpecification parentId=new ComponentSpecification(parentName,id.getVersionSpecification()); + + CompiledQueryProfile pathParentProfile=getComponent(parentId); + + if (pathParentProfile!=null && pathParentProfile.getType()!=null && pathParentProfile.getType().getMatchAsPath()) + return pathParentProfile; + return findPathParentQueryProfile(parentId); + } + +} diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/compiled/DimensionalMap.java b/container-search/src/main/java/com/yahoo/search/query/profile/compiled/DimensionalMap.java new file mode 100644 index 00000000000..b82939fa4ac --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/query/profile/compiled/DimensionalMap.java @@ -0,0 +1,68 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.query.profile.compiled; + +import com.google.common.collect.ImmutableMap; +import com.yahoo.search.query.profile.DimensionBinding; + +import java.util.HashMap; +import java.util.Map; +import java.util.Set; + +/** + * A map which may return different values depending on the values given in a context + * supplied with the key on all operations. + * <p> + * Dimensional maps are immutable and created through a DimensionalMap.Builder + * + * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a> + */ +public class DimensionalMap<KEY, VALUE> { + + private final Map<KEY, DimensionalValue<VALUE>> values; + + private DimensionalMap(Map<KEY, DimensionalValue<VALUE>> values) { + this.values = ImmutableMap.copyOf(values); + } + + /** Returns the value for this key matching a context, or null if none */ + public VALUE get(KEY key, Map<String, String> context) { + DimensionalValue<VALUE> variants = values.get(key); + if (variants == null) return null; + return variants.get(context); + } + + /** Returns the set of dimensional entries across all contexts. */ + public Set<Map.Entry<KEY, DimensionalValue<VALUE>>> entrySet() { + return values.entrySet(); + } + + /** Returns true if this is empty for all contexts. */ + public boolean isEmpty() { + return values.isEmpty(); + } + + public static class Builder<KEY, VALUE> { + + private Map<KEY, DimensionalValue.Builder<VALUE>> entries = new HashMap<>(); + + // TODO: DimensionBinding -> Binding? + public void put(KEY key, DimensionBinding binding, VALUE value) { + DimensionalValue.Builder<VALUE> entry = entries.get(key); + if (entry == null) { + entry = new DimensionalValue.Builder<>(); + entries.put(key, entry); + } + entry.add(value, binding); + } + + public DimensionalMap<KEY, VALUE> build() { + Map<KEY, DimensionalValue<VALUE>> map = new HashMap<>(); + for (Map.Entry<KEY, DimensionalValue.Builder<VALUE>> entry : entries.entrySet()) { + map.put(entry.getKey(), entry.getValue().build()); + } + return new DimensionalMap<>(map); + } + + } + +} diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/compiled/DimensionalValue.java b/container-search/src/main/java/com/yahoo/search/query/profile/compiled/DimensionalValue.java new file mode 100644 index 00000000000..0112928ada6 --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/query/profile/compiled/DimensionalValue.java @@ -0,0 +1,159 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.query.profile.compiled; + +import com.yahoo.search.query.profile.DimensionBinding; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * Contains the values a given key in a DimensionalMap may take for different dimensional contexts. + * + * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a> + */ +public class DimensionalValue<VALUE> { + + private final List<Value<VALUE>> values; + + /** Create a set of variants which is a single value regardless of dimensions */ + public DimensionalValue(Value<VALUE> value) { + this.values = Collections.singletonList(value); + } + + public DimensionalValue(List<Value<VALUE>> valueVariants) { + if (valueVariants.size() == 1) { // special cased for efficiency + this.values = Collections.singletonList(valueVariants.get(0)); + } + else { + this.values = new ArrayList<>(valueVariants); + Collections.sort(this.values); + } + } + + /** Returns the value matching this context, or null if none */ + public VALUE get(Map<String, String> context) { + if (context == null) + context = Collections.emptyMap(); + for (Value<VALUE> value : values) { + if (value.matches(context)) + return value.value(); + } + return null; + } + + public boolean isEmpty() { return values.isEmpty(); } + + @Override + public String toString() { + return values.toString(); + } + + public static class Builder<VALUE> { + + /** The minimal set of variants needed to capture all values at this key */ + private Map<VALUE, Value.Builder<VALUE>> buildableVariants = new HashMap<>(); + + public void add(VALUE value, DimensionBinding variantBinding) { + // Note: We know we can index by the value because its possible types are constrained + // to what query profiles allow: String, primitives and query profiles + Value.Builder variant = buildableVariants.get(value); + if (variant == null) { + variant = new Value.Builder<>(value); + buildableVariants.put(value, variant); + } + variant.addVariant(variantBinding); + } + + public DimensionalValue<VALUE> build() { + List<Value> variants = new ArrayList<>(); + for (Value.Builder buildableVariant : buildableVariants.values()) { + variants.addAll(buildableVariant.build()); + } + return new DimensionalValue(variants); + } + + } + + /** A value for a particular binding */ + private static class Value<VALUE> implements Comparable<Value> { + + private VALUE value = null; + + /** The minimal binding this holds for */ + private Binding binding = null; + + public Value(VALUE value, Binding binding) { + this.value = value; + this.binding = binding; + } + + /** Returns the value at this entry or null if none */ + public VALUE value() { return value; } + + /** Returns the binding that must match for this to be a valid entry, or Binding.nullBinding if none */ + public Binding binding() { + if (binding == null) return Binding.nullBinding; + return binding; + } + + public boolean matches(Map<String, String> context) { + return binding.matches(context); + } + + @Override + public int compareTo(Value other) { + return this.binding.compareTo(other.binding); + } + + @Override + public String toString() { + return " value '" + value + "' for " + binding; + } + + /** + * A single value with the minimal set of dimension combinations it holds for. + */ + private static class Builder<VALUE> { + + private final VALUE value; + + /** + * The set of bindings this value is for. + * Some of these are more general versions of others. + * We need to keep both to allow interleaving a different value with medium generality. + */ + private Set<DimensionBinding> variants = new HashSet<>(); + + public Builder(VALUE value) { + this.value = value; + } + + /** Add a binding this holds for */ + public void addVariant(DimensionBinding binding) { + variants.add(binding); + } + + /** Build a separate value object for each dimension combination which has this value */ + public List<Value<VALUE>> build() { + // Shortcut for efficiency of the normal case + if (variants.size()==1) + return Collections.singletonList(new Value<>(value, Binding.createFrom(variants.iterator().next()))); + + List<Value<VALUE>> values = new ArrayList<>(variants.size()); + for (DimensionBinding variant : variants) + values.add(new Value<>(value, Binding.createFrom(variant))); + return values; + } + + public Object value() { + return value; + } + + } + } +} diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/config/QueryProfileConfigurer.java b/container-search/src/main/java/com/yahoo/search/query/profile/config/QueryProfileConfigurer.java new file mode 100644 index 00000000000..5770665e3a1 --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/query/profile/config/QueryProfileConfigurer.java @@ -0,0 +1,227 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.query.profile.config; + +import com.yahoo.component.ComponentId; +import com.yahoo.component.ComponentSpecification; +import com.yahoo.config.subscription.ConfigSubscriber; +import com.yahoo.search.query.profile.DimensionValues; +import com.yahoo.search.query.profile.QueryProfile; +import com.yahoo.search.query.profile.QueryProfileRegistry; +import com.yahoo.search.query.profile.types.FieldDescription; +import com.yahoo.search.query.profile.types.FieldType; +import com.yahoo.search.query.profile.types.QueryProfileType; +import com.yahoo.search.query.profile.types.QueryProfileTypeRegistry; +import com.yahoo.text.BooleanParser; + +import java.util.HashSet; +import java.util.Set; + +/** + * @author bratseth + */ +public class QueryProfileConfigurer implements ConfigSubscriber.SingleSubscriber<QueryProfilesConfig> { + + private final ConfigSubscriber subscriber = new ConfigSubscriber(); + + private volatile QueryProfileRegistry currentRegistry; + + public QueryProfileConfigurer(String configId) { + subscriber.subscribe(this, QueryProfilesConfig.class, configId); + } + + /** Returns the registry created by the last occurring call to configure */ + public QueryProfileRegistry getCurrentRegistry() { return currentRegistry; } + + private void setCurrentRegistry(QueryProfileRegistry registry) { + this.currentRegistry=registry; + } + + public void configure(QueryProfilesConfig config) { + QueryProfileRegistry registry = createFromConfig(config); + setCurrentRegistry(registry); + } + + public static QueryProfileRegistry createFromConfig(QueryProfilesConfig config) { + QueryProfileRegistry registry=new QueryProfileRegistry(); + + // Pass 1: Create all profiles and profile types + for (QueryProfilesConfig.Queryprofiletype profileTypeConfig : config.queryprofiletype()) { + createProfileType(profileTypeConfig,registry.getTypeRegistry()); + } + for (QueryProfilesConfig.Queryprofile profileConfig : config.queryprofile()) { + createProfile(profileConfig,registry); + } + + // Pass 2: Resolve references and add content + for (QueryProfilesConfig.Queryprofiletype profileTypeConfig : config.queryprofiletype()) { + fillProfileType(profileTypeConfig,registry.getTypeRegistry()); + } + + // To ensure topological sorting, using DPS. This will _NOT_ detect cycles (but it will not fail if they + // exist either) + Set<ComponentId> filled = new HashSet<>(); + for (QueryProfilesConfig.Queryprofile profileConfig : config.queryprofile()) { + fillProfile(profileConfig, config, registry, filled); + } + + registry.freeze(); + return registry; + } + + /** Stop subscribing from this configurer */ + public void shutdown() { + subscriber.close(); + } + + private static void createProfile(QueryProfilesConfig.Queryprofile config,QueryProfileRegistry registry) { + QueryProfile profile=new QueryProfile(config.id()); + try { + String typeId=config.type(); + if (typeId!=null && !typeId.isEmpty()) + profile.setType(registry.getType(typeId)); + + if (config.dimensions().size()>0) { + String[] dimensions=new String[config.dimensions().size()]; + for (int i=0; i<config.dimensions().size(); i++) + dimensions[i]=config.dimensions().get(i); + profile.setDimensions(dimensions); + } + + registry.register(profile); + } + catch (IllegalArgumentException e) { + throw new IllegalArgumentException("Invalid " + profile,e); + } + } + + private static void createProfileType(QueryProfilesConfig.Queryprofiletype config, QueryProfileTypeRegistry registry) { + QueryProfileType type=new QueryProfileType(config.id()); + type.setStrict(config.strict()); + type.setMatchAsPath(config.matchaspath()); + registry.register(type); + } + + private static void fillProfile(QueryProfilesConfig.Queryprofile config, + QueryProfilesConfig queryProfilesConfig, + QueryProfileRegistry registry, + Set<ComponentId> filled) { + QueryProfile profile=registry.getComponent(new ComponentSpecification(config.id()).toId()); + if (filled.contains(profile.getId())) return; + filled.add(profile.getId()); + try { + for (String inheritedId : config.inherit()) { + QueryProfile inherited=registry.getComponent(inheritedId); + if (inherited==null) + throw new IllegalArgumentException("Inherited query profile '" + inheritedId + "' in " + profile + " was not found"); + fillProfile(inherited, queryProfilesConfig, registry, filled); + profile.addInherited(inherited); + } + + for (QueryProfilesConfig.Queryprofile.Reference referenceConfig : config.reference()) { + QueryProfile referenced=registry.getComponent(referenceConfig.value()); + if (referenced==null) + throw new IllegalArgumentException("Query profile '" + referenceConfig.value() + "' referenced as '" + + referenceConfig.name() + "' in " + profile + " was not found"); + profile.set(referenceConfig.name(),referenced, registry); + if (referenceConfig.overridable()!=null && !referenceConfig.overridable().isEmpty()) + profile.setOverridable(referenceConfig.name(),BooleanParser.parseBoolean(referenceConfig.overridable()),null); + } + + for (QueryProfilesConfig.Queryprofile.Property propertyConfig : config.property()) { + profile.set(propertyConfig.name(),propertyConfig.value(), registry); + if (propertyConfig.overridable()!=null && !propertyConfig.overridable().isEmpty()) + profile.setOverridable(propertyConfig.name(),BooleanParser.parseBoolean(propertyConfig.overridable()),null); + } + + for (QueryProfilesConfig.Queryprofile.Queryprofilevariant variantConfig : config.queryprofilevariant()) { + String[] forDimensionValueArray=new String[variantConfig.fordimensionvalues().size()]; + for (int i=0; i<variantConfig.fordimensionvalues().size(); i++) { + forDimensionValueArray[i]=variantConfig.fordimensionvalues().get(i).trim(); + if ("*".equals(forDimensionValueArray[i])) + forDimensionValueArray[i]=null; + } + DimensionValues forDimensionValues=DimensionValues.createFrom(forDimensionValueArray); + + for (String inheritedId : variantConfig.inherit()) { + QueryProfile inherited=registry.getComponent(inheritedId); + if (inherited==null) + throw new IllegalArgumentException("Inherited query profile '" + inheritedId + "' in " + profile + + " for '" + forDimensionValues + "' was not found"); + fillProfile(inherited, queryProfilesConfig, registry, filled); + profile.addInherited(inherited, forDimensionValues); + } + + for (QueryProfilesConfig.Queryprofile.Queryprofilevariant.Reference referenceConfig : variantConfig.reference()) { + QueryProfile referenced=registry.getComponent(referenceConfig.value()); + if (referenced==null) + throw new IllegalArgumentException("Query profile '" + referenceConfig.value() + "' referenced as '" + + referenceConfig.name() + "' in " + profile + " for '" + forDimensionValues + "' was not found"); + profile.set(referenceConfig.name(), referenced, forDimensionValues, registry); + } + + for (QueryProfilesConfig.Queryprofile.Queryprofilevariant.Property propertyConfig : variantConfig.property()) { + profile.set(propertyConfig.name(), propertyConfig.value(), forDimensionValues, registry); + } + + } + + } + catch (IllegalArgumentException e) { + throw new IllegalArgumentException("Invalid " + profile,e); + } + } + + /** Fill a given profile by locating its config */ + private static void fillProfile(QueryProfile inherited, + QueryProfilesConfig queryProfilesConfig, + QueryProfileRegistry registry, + Set<ComponentId> visited) { + for (QueryProfilesConfig.Queryprofile inheritedConfig : queryProfilesConfig.queryprofile()) { + if (inherited.getId().stringValue().equals(inheritedConfig.id())) { + fillProfile(inheritedConfig, queryProfilesConfig, registry, visited); + } + } + } + + private static void fillProfileType(QueryProfilesConfig.Queryprofiletype config,QueryProfileTypeRegistry registry) { + QueryProfileType type=registry.getComponent(new ComponentSpecification(config.id()).toId()); + try { + + for (String inheritedId : config.inherit()) { + QueryProfileType inherited=registry.getComponent(inheritedId); + if (inherited==null) + throw new IllegalArgumentException("Inherited query profile type '" + inheritedId + "' in " + type + " was not found"); + else + type.inherited().add(inherited); + + } + + for (QueryProfilesConfig.Queryprofiletype.Field fieldConfig : config.field()) + instantiateFieldDescription(fieldConfig,type,registry); + } + catch (IllegalArgumentException e) { + throw new IllegalArgumentException("Invalid " + type,e); + } + } + + private static void instantiateFieldDescription(QueryProfilesConfig.Queryprofiletype.Field fieldConfig, + QueryProfileType type, + QueryProfileTypeRegistry registry) { + try { + FieldType fieldType=FieldType.fromString(fieldConfig.type(),registry); + FieldDescription field=new FieldDescription( + fieldConfig.name(), + fieldType, + fieldConfig.alias(), + fieldConfig.mandatory(), + fieldConfig.overridable() + ); + type.addField(field, registry); + } + catch (IllegalArgumentException e) { + throw new IllegalArgumentException("Invalid field '" + fieldConfig.name() + "' in " + type,e); + } + } + + +} diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/config/QueryProfileXMLReader.java b/container-search/src/main/java/com/yahoo/search/query/profile/config/QueryProfileXMLReader.java new file mode 100644 index 00000000000..97e3fb90dc9 --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/query/profile/config/QueryProfileXMLReader.java @@ -0,0 +1,366 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.query.profile.config; + +import com.yahoo.component.ComponentId; +import com.yahoo.component.ComponentSpecification; +import com.yahoo.io.reader.NamedReader; +import com.yahoo.search.query.profile.DimensionValues; +import com.yahoo.search.query.profile.QueryProfile; +import com.yahoo.search.query.profile.QueryProfileRegistry; +import com.yahoo.search.query.profile.types.FieldDescription; +import com.yahoo.search.query.profile.types.FieldType; +import com.yahoo.search.query.profile.types.QueryProfileType; +import com.yahoo.search.query.profile.types.QueryProfileTypeRegistry; +import com.yahoo.text.XML; +import org.w3c.dom.Element; + +import java.io.File; +import java.io.FileReader; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.logging.Logger; + +/** + * A class which imports query profiles and types from XML files + * + * @author bratseth + */ +public class QueryProfileXMLReader { + + private static Logger logger=Logger.getLogger(QueryProfileXMLReader.class.getName()); + + /** + * Reads all query profile xml files in a given directory, + * and all type xml files from the immediate subdirectory "types/" (if any) + * + * @throws RuntimeException if <code>directory</code> is not a readable directory, or if there is some error in the XML + */ + public QueryProfileRegistry read(String directory) { + List<NamedReader> queryProfileReaders=new ArrayList<>(); + List<NamedReader> queryProfileTypeReaders=new ArrayList<>(); + try { + File dir=new File(directory); + if ( !dir.isDirectory() ) throw new IllegalArgumentException("Could not read query profiles: '" + + directory + "' is not a valid directory."); + + for (File file : sortFiles(dir)) { + if ( ! file.getName().endsWith(".xml")) continue; + queryProfileReaders.add(new NamedReader(file.getName(),new FileReader(file))); + } + File typeDir=new File(dir,"types"); + if (typeDir.isDirectory()) { + for (File file : sortFiles(typeDir)) { + if ( ! file.getName().endsWith(".xml")) continue; + queryProfileTypeReaders.add(new NamedReader(file.getName(),new FileReader(file))); + } + } + + return read(queryProfileTypeReaders,queryProfileReaders); + } + catch (IOException e) { + throw new IllegalArgumentException("Could not read query profiles from '" + directory + "'",e); + } + finally { + closeAll(queryProfileReaders); + closeAll(queryProfileTypeReaders); + } + } + + private List<File> sortFiles(File dir) { + ArrayList<File> files = new ArrayList<>(); + files.addAll(Arrays.asList(dir.listFiles())); + Collections.sort(files); + return files; + } + + private void closeAll(List<NamedReader> readers) { + for (NamedReader reader : readers) { + try { reader.close(); } catch (IOException e) { } + } + } + + /** + * Read the XML file readers into a registry. This does not close the readers. + * This method is used directly from the admin system. + */ + public QueryProfileRegistry read(List<NamedReader> queryProfileTypeReaders,List<NamedReader> queryProfileReaders) { + QueryProfileRegistry registry=new QueryProfileRegistry(); + + // Phase 1 + List<Element> queryProfileTypeElements=createQueryProfileTypes(queryProfileTypeReaders,registry.getTypeRegistry()); + List<Element> queryProfileElements=createQueryProfiles(queryProfileReaders,registry); + + // Phase 2 + fillQueryProfileTypes(queryProfileTypeElements,registry.getTypeRegistry()); + fillQueryProfiles(queryProfileElements,registry); + return registry; + } + + public List<Element> createQueryProfileTypes(List<NamedReader> queryProfileTypeReaders, QueryProfileTypeRegistry registry) { + List<Element> queryProfileTypeElements=new ArrayList<>(queryProfileTypeReaders.size()); + for (NamedReader reader : queryProfileTypeReaders) { + Element root=XML.getDocument(reader).getDocumentElement(); + if ( ! root.getNodeName().equals("query-profile-type")) { + logger.info("Ignoring '" + reader.getName() + + "': Expected XML root element 'query-profile-type' but was '" + root.getNodeName() + "'"); + continue; + } + + String idString=root.getAttribute("id"); + if (idString==null || idString.equals("")) + throw new IllegalArgumentException("'" + reader.getName() + "' has no 'id' attribute in the root element"); + ComponentId id=new ComponentId(idString); + validateFileNameToId(reader.getName(),id,"query profile type"); + QueryProfileType type=new QueryProfileType(id); + type.setMatchAsPath(XML.getChild(root,"match") != null); + type.setStrict(XML.getChild(root,"strict") != null); + registry.register(type); + queryProfileTypeElements.add(root); + } + return queryProfileTypeElements; + } + + public List<Element> createQueryProfiles(List<NamedReader> queryProfileReaders, QueryProfileRegistry registry) { + List<Element> queryProfileElements=new ArrayList<>(queryProfileReaders.size()); + for (NamedReader reader : queryProfileReaders) { + Element root=XML.getDocument(reader).getDocumentElement(); + if ( ! root.getNodeName().equals("query-profile")) { + logger.info("Ignoring '" + reader.getName() + + "': Expected XML root element 'query-profile' but was '" + root.getNodeName() + "'"); + continue; + } + + String idString=root.getAttribute("id"); + if (idString==null || idString.equals("")) + throw new IllegalArgumentException("Query profile '" + reader.getName() + "' has no 'id' attribute in the root element"); + ComponentId id=new ComponentId(idString); + validateFileNameToId(reader.getName(),id,"query profile"); + + QueryProfile queryProfile=new QueryProfile(id); + String typeId=root.getAttribute("type"); + if (typeId!=null && ! typeId.equals("")) { + QueryProfileType type=registry.getType(typeId); + if (type==null) + throw new IllegalArgumentException("Query profile '" + reader.getName() + "': Type id '" + typeId + "' can not be resolved"); + queryProfile.setType(type); + } + + Element dimensions=XML.getChild(root,"dimensions"); + if (dimensions!=null) + queryProfile.setDimensions(toArray(XML.getValue(dimensions))); + + registry.register(queryProfile); + queryProfileElements.add(root); + } + return queryProfileElements; + } + + /** Throws an exception if the name is not corresponding to the id */ + private void validateFileNameToId(final String actualName,ComponentId id,String artifactName) { + String expectedCanonicalFileName=id.toFileName(); + String expectedAlternativeFileName=id.stringValue().replace(":","-").replace("/","_"); // legacy + String fileName=new File(actualName).getName(); + fileName=stripXmlEnding(fileName); + String canonicalFileName=ComponentId.fromFileName(fileName).toFileName(); + if ( ! canonicalFileName.equals(expectedCanonicalFileName) && ! canonicalFileName.equals(expectedAlternativeFileName)) + throw new IllegalArgumentException("The file name of " + artifactName + " '" + id + + "' must be '" + expectedCanonicalFileName + ".xml' but was '" + actualName + "'"); + } + + private String stripXmlEnding(String fileName) { + if (!fileName.endsWith(".xml")) + throw new IllegalArgumentException("'" + fileName + "' should have a .xml ending"); + else + return fileName.substring(0,fileName.length()-4); + } + + private String[] toArray(String csv) { + String[] array=csv.split(","); + for (int i=0; i<array.length; i++) + array[i]=array[i].trim(); + return array; + } + + public void fillQueryProfileTypes(List<Element> queryProfileTypeElements, QueryProfileTypeRegistry registry) { + for (Element element : queryProfileTypeElements) { + QueryProfileType type=registry.getComponent(new ComponentSpecification(element.getAttribute("id")).toId()); + try { + readInheritedTypes(element,type,registry); + readFieldDefinitions(element,type,registry); + } + catch (RuntimeException e) { + throw new IllegalArgumentException("Error reading " + type,e); + } + } + } + + private void readInheritedTypes(Element element,QueryProfileType type,QueryProfileTypeRegistry registry) { + String inheritedString=element.getAttribute("inherits"); + if (inheritedString==null || inheritedString.equals("")) return; + for (String inheritedId : inheritedString.split(" ")) { + inheritedId=inheritedId.trim(); + if (inheritedId.equals("")) continue; + QueryProfileType inheritedType=registry.getComponent(inheritedId); + if (inheritedType==null) throw new IllegalArgumentException("Could not resolve inherited query profile type '" + inheritedId); + type.inherited().add(inheritedType); + } + } + + private void readFieldDefinitions(Element element,QueryProfileType type,QueryProfileTypeRegistry registry) { + for (Element field : XML.getChildren(element,"field")) { + String name=field.getAttribute("name"); + if (name==null || name.equals("")) throw new IllegalArgumentException("A field has no 'name' attribute"); + try { + String fieldTypeName=field.getAttribute("type"); + if (fieldTypeName==null) throw new IllegalArgumentException("Field '" + field + "' has no 'type' attribute"); + FieldType fieldType=FieldType.fromString(fieldTypeName,registry); + type.addField(new FieldDescription(name,fieldType,field.getAttribute("alias"), + getBooleanAttribute("mandatory",false,field),getBooleanAttribute("overridable",true,field)), registry); + } + catch(RuntimeException e) { + throw new IllegalArgumentException("Invalid field '" + name + "'",e); + } + } + } + + public void fillQueryProfiles(List<Element> queryProfileElements, QueryProfileRegistry registry) { + for (Element element : queryProfileElements) { + // Lookup by exact id + QueryProfile profile=registry.getComponent(new ComponentSpecification(element.getAttribute("id")).toId()); + try { + readInherited(element,profile,registry,null,profile.toString()); + readFields(element,profile,registry,null,profile.toString()); + readVariants(element,profile,registry); + } + catch (RuntimeException e) { + throw new IllegalArgumentException("Error reading " + profile,e); + } + } + } + + private void readInherited(Element element,QueryProfile profile,QueryProfileRegistry registry,DimensionValues dimensionValues,String sourceDescription) { + String inheritedString=element.getAttribute("inherits"); + if (inheritedString==null || inheritedString.equals("")) return; + for (String inheritedId : inheritedString.split(" ")) { + inheritedId=inheritedId.trim(); + if (inheritedId.equals("")) continue; + QueryProfile inheritedProfile=registry.getComponent(inheritedId); + if (inheritedProfile==null) throw new IllegalArgumentException("Could not resolve inherited query profile '" + inheritedId + "' in " + sourceDescription); + profile.addInherited(inheritedProfile,dimensionValues); + } + } + + private void readFields(Element element,QueryProfile profile,QueryProfileRegistry registry,DimensionValues dimensionValues,String sourceDescription) { + List<KeyValue> references=new ArrayList<>(); + List<KeyValue> properties=new ArrayList<>(); + for (Element field : XML.getChildren(element,"field")) { + String name=field.getAttribute("name"); + if (name==null || name.equals("")) throw new IllegalArgumentException("A field in " + sourceDescription + " has no 'name' attribute"); + try { + Boolean overridable=getBooleanAttribute("overridable",null,field); + if (overridable!=null) + profile.setOverridable(name,overridable,null); + + Object fieldValue=readFieldValue(field,name,sourceDescription,registry); + if (fieldValue instanceof QueryProfile) + references.add(new KeyValue(name,fieldValue)); + else + properties.add(new KeyValue(name,fieldValue)); + } + catch (IllegalArgumentException e) { + throw new IllegalArgumentException("Invalid field '" + name + "' in " + sourceDescription,e); + } + } + // Must set references before properties + for (KeyValue keyValue : references) + profile.set(keyValue.getKey() ,keyValue.getValue(), dimensionValues, registry); + for (KeyValue keyValue : properties) + profile.set(keyValue.getKey(), keyValue.getValue(), dimensionValues, registry); + + } + + private Object readFieldValue(Element field,String name,String targetDescription,QueryProfileRegistry registry) { + Element ref=XML.getChild(field,"ref"); + if (ref!=null) { + String referencedName=XML.getValue(ref); + QueryProfile referenced=registry.getComponent(referencedName); + if (referenced==null) + throw new IllegalArgumentException("Could not find query profile '" + referencedName + "' referenced as '" + + name + "' in " + targetDescription); + return referenced; + } + else { + return XML.getValue(field); + } + } + + private void readVariants(Element element,QueryProfile profile,QueryProfileRegistry registry) { + for (Element queryProfileVariantElement : XML.getChildren(element,"query-profile")) { // A "virtual" query profile contained inside another + List<String> dimensions=profile.getDimensions(); + if (dimensions==null) + throw new IllegalArgumentException("Cannot create a query profile variant in " + profile + + ", as it has not declared any variable dimensions"); + String dimensionString=queryProfileVariantElement.getAttribute("for"); + String[] dimensionValueArray=makeStarsNull(toArray(dimensionString)); + if (dimensions.size()<dimensionValueArray.length) + throw new IllegalArgumentException("Cannot create a query profile variant for '" + dimensionString + + "' as only " + dimensions.size() + " dimensions has been defined"); + DimensionValues dimensionValues=DimensionValues.createFrom(dimensionValueArray); + + String description="variant '" + dimensionString + "' in " + profile.toString(); + readInherited(queryProfileVariantElement,profile,registry,dimensionValues,description); + readFields(queryProfileVariantElement,profile,registry,dimensionValues,description); + } + } + + private String[] makeStarsNull(String[] strings) { + for (int i=0; i<strings.length; i++) + if (strings[i].equals("*")) + strings[i]=null; + return strings; + } + + /** + * Returns true if the string is "true".<br> + * Returns false if the string is "false".<br> + * Returns <code>default</code> if the string is null or empty (this parameter may be null)<br> + * @throws IllegalArgumentException if the string has any other value + */ + private Boolean asBoolean(String s,Boolean defaultValue) { + if (s==null) return defaultValue; + if (s.isEmpty()) return defaultValue; + if ("true".equals(s)) return true; + if ("false".equals(s)) return false; + throw new IllegalArgumentException("Expected 'true' or 'false' but was'" + s + "'"); + } + + /** Returns the given attribute as a boolean, using the semantics of {@link #asBoolean} */ + private Boolean getBooleanAttribute(String attributeName,Boolean defaultValue,Element from) { + try { + return asBoolean(from.getAttribute(attributeName),defaultValue); + } + catch (IllegalArgumentException e) { + throw new IllegalArgumentException("Attribute '" + attributeName,e); + } + } + + private static class KeyValue { + + private String key; + private Object value; + + public KeyValue(String key,Object value) { + this.key=key; + this.value=value; + } + + public String getKey() { return key; } + + public Object getValue() { return value; } + + } + +} diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/config/package-info.java b/container-search/src/main/java/com/yahoo/search/query/profile/config/package-info.java new file mode 100644 index 00000000000..8ea4e887661 --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/query/profile/config/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.search.query.profile.config; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/package-info.java b/container-search/src/main/java/com/yahoo/search/query/profile/package-info.java new file mode 100644 index 00000000000..df3f4ac45ab --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/query/profile/package-info.java @@ -0,0 +1,12 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * Query Profiles provide nested sets of named (and optionally typed) key-values which can be referenced in a Query + * to proviode initial values of Query properties. Values in nested query profiles can be looked up from + * the query properties by dotting the names. Query profiles supports inheritance to allow variations + * for, e.g different buckets, client types, markets etc. */ +@ExportPackage +@PublicApi +package com.yahoo.search.query.profile; + +import com.yahoo.api.annotations.PublicApi; +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/types/FieldDescription.java b/container-search/src/main/java/com/yahoo/search/query/profile/types/FieldDescription.java new file mode 100644 index 00000000000..c522ec04023 --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/query/profile/types/FieldDescription.java @@ -0,0 +1,148 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.query.profile.types; + +import com.google.common.collect.ImmutableList; +import com.yahoo.processing.request.CompoundName; +import com.yahoo.search.query.profile.QueryProfile; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +/** + * A field description of a query profile type. Immutable. + * Field descriptions can be sorted by name. + * + * @author bratseth + */ +public class FieldDescription implements Comparable<FieldDescription> { + + private final CompoundName name; + private final FieldType type; + private final List<String> aliases; + + /** If true, this value must be provided either in the query profile or in the search request */ + private final boolean mandatory; + + /** If true, assignments to this value from outside will be ignored */ + private final boolean overridable; + + public FieldDescription(String name, FieldType type) { + this(name,type,false); + } + + public FieldDescription(String name, String type) { + this(name,FieldType.fromString(type,null)); + } + + public FieldDescription(String name, FieldType type, boolean mandatory) { + this(name, type, mandatory, true); + } + + public FieldDescription(String name, String type, String aliases) { + this(name,type,aliases,false,true); + } + + public FieldDescription(String name, FieldType type, String aliases) { + this(name, type, aliases, false, true); + } + + /** + * Creates a field description + * + * @param name the name of the field + * @param typeString the type of the field represented as a string - see {@link com.yahoo.search.query.profile.types.FieldType} + * @param aliases a space-separated list of alias names of this field name. Aliases are not following dotted + * (meaning they are global, not that they cannot contain dots) and are case insensitive. Null is permissible + * if there are no aliases + * @param mandatory whether it is mandatory to provide a value for this field. default: false + * @param overridable whether this can be overridden when first set in a profile. Default: true + */ + public FieldDescription(String name, String typeString, String aliases, boolean mandatory, boolean overridable) { + this(name,FieldType.fromString(typeString,null),aliases,mandatory,overridable); + } + + public FieldDescription(String name, FieldType type, boolean mandatory, boolean overridable) { + this(name, type, null, mandatory, overridable); + } + + public FieldDescription(String name, FieldType type, String aliases, boolean mandatory, boolean overridable) { + this(new CompoundName(name), type, aliases, mandatory, overridable); + } + + /** + * Creates a field description from a list where the aliases are represented as a comma-separated string + */ + public FieldDescription(CompoundName name, FieldType type, String aliases, boolean mandatory, boolean overridable) { + this(name, type, toList(aliases), mandatory, overridable); + } + + /** + * Creates a field description + * + * @param name the name of the field + * @param type the type of the field represented as a string - see {@link com.yahoo.search.query.profile.types.FieldType} + * @param aliases a list of aliases, never null. Aliases are not following dotted + * (meaning they are global, not that they cannot contain dots) and are case insensitive. + * @param mandatory whether it is mandatory to provide a value for this field. default: false + * @param overridable whether this can be overridden when first set in a profile. Default: true + */ + public FieldDescription(CompoundName name, FieldType type, List<String> aliases, boolean mandatory, boolean overridable) { + if (name.isEmpty()) + throw new IllegalArgumentException("Illegal name ''"); + for (String nameComponent : name.asList()) + QueryProfile.validateName(nameComponent); + this.name = name; + this.type = type; + + // Forbidden until we can figure out the right semantics + if (name.isCompound() && ! aliases.isEmpty()) throw new IllegalArgumentException("Aliases is not allowed with compound names"); + + this.aliases = ImmutableList.copyOf(aliases); + this.mandatory = mandatory; + this.overridable = overridable; + } + + private static List<String> toList(String string) { + if (string == null || string.isEmpty()) return ImmutableList.of(); + return ImmutableList.copyOf(Arrays.asList(string.split(" "))); + } + + /** Returns the full name of this as a string */ + public String getName() { return name.toString(); } + + /** Returns the full name of this as a compound name */ + public CompoundName getCompoundName() { return name; } + + public FieldType getType() { return type; } + + /** Returns a unmodifiable list of the aliases of this. An empty list (never null) if there are none. */ + public List<String> getAliases() { return aliases; } + + /** Returns whether this field must be provided in the query profile or the search definition. Default: false */ + public boolean isMandatory() { return mandatory; } + + /** Returns false if overrides to values for this field from the outside should be ignored. Default: true */ + public boolean isOverridable() { return overridable; } + + public int compareTo(FieldDescription other) { + return name.toString().compareTo(other.name.toString()); + } + + /** Returns a copy of this with the name set to the argument name */ + public FieldDescription withName(CompoundName name) { + return new FieldDescription(name, type, aliases, mandatory, overridable); + } + + /** Returns a copy of this with the type set to the argument type */ + public FieldDescription withType(FieldType type) { + return new FieldDescription(name, type, aliases, mandatory, overridable); + } + + @Override + public String toString() { + return "field '" + name + "' type " + type.stringValue() + "" + + (mandatory?" (mandatory)":"") + (!overridable?" (not overridable)":""); + } + +} diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/types/FieldType.java b/container-search/src/main/java/com/yahoo/search/query/profile/types/FieldType.java new file mode 100644 index 00000000000..abe3c4425ae --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/query/profile/types/FieldType.java @@ -0,0 +1,94 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.query.profile.types; + +import com.yahoo.search.query.profile.QueryProfile; +import com.yahoo.search.query.profile.QueryProfileRegistry; +import com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry; +import com.yahoo.search.yql.YqlQuery; +import com.yahoo.tensor.Tensor; + +import java.util.Optional; + +/** + * Superclass of query type field types. + * Field types are immutable. + * + * @author bratseth + */ +@SuppressWarnings("rawtypes") +public abstract class FieldType { + + public static final PrimitiveFieldType stringType = new PrimitiveFieldType(String.class); + public static final PrimitiveFieldType integerType = new PrimitiveFieldType(Integer.class); + public static final PrimitiveFieldType longType = new PrimitiveFieldType(Long.class); + public static final PrimitiveFieldType floatType = new PrimitiveFieldType(Float.class); + public static final PrimitiveFieldType doubleType = new PrimitiveFieldType(Double.class); + public static final PrimitiveFieldType booleanType = new PrimitiveFieldType(Boolean.class); + public static final TensorFieldType genericTensorType = new TensorFieldType(Optional.empty()); + public static final QueryFieldType queryType = new QueryFieldType(); + public static final QueryProfileFieldType genericQueryProfileType = new QueryProfileFieldType(); + + /** Returns the class of instance values of this field type */ + public abstract Class getValueClass(); + + /** Returns a string representation of this type which can be converted back to a type class by {@link #fromString} */ + public abstract String stringValue(); + + public abstract String toString(); + + /** Returns a string describing possible instances of this type, suitable for user error messages */ + public abstract String toInstanceDescription(); + + /** Converts the given type to an instance of this type, if possible. Returns null if not possible. */ + public abstract Object convertFrom(Object o, QueryProfileRegistry registry); + + /** Converts the given type to an instance of this type, if possible. Returns null if not possible. */ + public abstract Object convertFrom(Object o, CompiledQueryProfileRegistry registry); + + /** + * Returns the field type for a given string name. + * + * @param typeString a type string - a primitive name, "query-profile" or "query-profile:profile-name" + * @param registry the registry in which query profile references are resolved when the last form above is used, + * or null in which case that form cannot be used + * @throws IllegalArgumentException if the string does not resolve to a type + */ + public static FieldType fromString(String typeString, QueryProfileTypeRegistry registry) { + if ("string".equals(typeString)) + return stringType; + if ("integer".equals(typeString)) + return integerType; + if ("long".equals(typeString)) + return longType; + if ("float".equals(typeString)) + return floatType; + if ("double".equals(typeString)) + return doubleType; + if ("boolean".equals(typeString)) + return booleanType; + if ("query".equals(typeString)) + return queryType; + if (typeString.startsWith("tensor")) + return TensorFieldType.fromTypeString(typeString); + if ("query-profile".equals(typeString)) + return genericQueryProfileType; + if (typeString.startsWith("query-profile:")) + return QueryProfileFieldType.fromString(typeString.substring("query-profile:".length()),registry); + throw new IllegalArgumentException("Unknown type '" + typeString + "'"); + } + + /** Returns the field type from a value class, or null if there is no type for it */ + public static FieldType fromClass(Class clazz) { + if (clazz == String.class) return stringType; + if (clazz == Integer.class) return integerType; + if (clazz == Long.class) return longType; + if (clazz == Float.class) return floatType; + if (clazz == Double.class) return doubleType; + if (clazz == Boolean.class) return booleanType; + if (clazz == Tensor.class) return genericTensorType; + if (clazz == YqlQuery.class) return queryType; + if (clazz == QueryProfile.class) return genericQueryProfileType; + return null; + } + +} diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/types/PrimitiveFieldType.java b/container-search/src/main/java/com/yahoo/search/query/profile/types/PrimitiveFieldType.java new file mode 100644 index 00000000000..76b3f78ac2f --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/query/profile/types/PrimitiveFieldType.java @@ -0,0 +1,86 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.query.profile.types; + +import com.yahoo.search.query.profile.QueryProfileRegistry; +import com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry; + +import static com.yahoo.text.Lowercase.toLowerCase; + +/** + * Represents a query field type which is a primitive - String, Integer, Float, Double or Long. + * + * @author bratseth + */ +@SuppressWarnings("rawtypes") +public class PrimitiveFieldType extends FieldType { + + private Class primitiveClass; + + PrimitiveFieldType(Class primitiveClass) { + this.primitiveClass=primitiveClass; + } + + public @Override Class getValueClass() { return primitiveClass; } + + public @Override String stringValue() { + return toLowerCase(primitiveClass.getSimpleName()); + } + + public @Override String toString() { return "field type " + stringValue(); } + + public @Override String toInstanceDescription() { + return toLowerCase(primitiveClass.getSimpleName()); + } + + @Override + public Object convertFrom(Object object, CompiledQueryProfileRegistry registry) { + return convertFrom(object, (QueryProfileRegistry)null); + } + + public @Override Object convertFrom(Object object, QueryProfileRegistry registry) { + if (primitiveClass == object.getClass()) return object; + + if (object.getClass() == String.class) return convertFromString((String)object); + if (object instanceof Number) return convertFromNumber((Number)object); + + return null; + } + + private Object convertFromString(String string) { + try { + if (primitiveClass==Integer.class) return Integer.valueOf(string); + if (primitiveClass==Double.class) return Double.valueOf(string); + if (primitiveClass==Float.class) return Float.valueOf(string); + if (primitiveClass==Long.class) return Long.valueOf(string); + if (primitiveClass==Boolean.class) return Boolean.valueOf(string); + } + catch (NumberFormatException e) { + return null; // Handled in caller + } + throw new RuntimeException("Programming error"); + } + + private Object convertFromNumber(Number number) { + if (primitiveClass==Integer.class) return number.intValue(); + if (primitiveClass==Double.class) return number.doubleValue(); + if (primitiveClass==Float.class) return number.floatValue(); + if (primitiveClass==Long.class) return number.longValue(); + if (primitiveClass==String.class) return String.valueOf(number); + throw new RuntimeException("Programming error: Input type is " + number.getClass() + + " primitiveClass is " + primitiveClass); + } + + @Override + public int hashCode() { + return primitiveClass.hashCode(); + } + + @Override + public boolean equals(Object o) { + if (o == this) return true; + if ( ! (o instanceof PrimitiveFieldType)) return false; + PrimitiveFieldType other = (PrimitiveFieldType)o; + return other.primitiveClass.equals(this.primitiveClass); + } + +} diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/types/QueryFieldType.java b/container-search/src/main/java/com/yahoo/search/query/profile/types/QueryFieldType.java new file mode 100644 index 00000000000..a0982fdf0f6 --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/query/profile/types/QueryFieldType.java @@ -0,0 +1,41 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.query.profile.types; + +import com.yahoo.search.query.profile.QueryProfileRegistry; +import com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry; +import com.yahoo.search.yql.YqlQuery; +import com.yahoo.tensor.MapTensor; +import com.yahoo.tensor.Tensor; + +/** + * A YQL query template field type in a query profile + * + * @author bratseth + */ +public class QueryFieldType extends FieldType { + + @Override + public Class getValueClass() { return YqlQuery.class; } + + @Override + public String stringValue() { return "query"; } + + @Override + public String toString() { return "field type " + stringValue(); } + + @Override + public String toInstanceDescription() { return "a YQL query template"; } + + @Override + public Object convertFrom(Object o, QueryProfileRegistry registry) { + if (o instanceof YqlQuery) return o; + if (o instanceof String) return YqlQuery.from((String)o); + return null; + } + + @Override + public Object convertFrom(Object o, CompiledQueryProfileRegistry registry) { + return convertFrom(o, (QueryProfileRegistry)null); + } + +} diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/types/QueryProfileFieldType.java b/container-search/src/main/java/com/yahoo/search/query/profile/types/QueryProfileFieldType.java new file mode 100644 index 00000000000..df52e78c6ef --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/query/profile/types/QueryProfileFieldType.java @@ -0,0 +1,100 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.query.profile.types; + +import com.yahoo.search.query.profile.QueryProfile; +import com.yahoo.search.query.profile.QueryProfileRegistry; +import com.yahoo.search.query.profile.compiled.CompiledQueryProfile; +import com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry; + +/** + * Represents a query profile field type which is a reference to a query profile. + * The reference may optionally specify the type of the referred query profile. + * + * @author bratseth + */ +public class QueryProfileFieldType extends FieldType { + + private final QueryProfileType type; + + public static QueryProfileFieldType fromString(String queryProfileName, QueryProfileTypeRegistry registry) { + if (queryProfileName==null || queryProfileName.equals("")) + return new QueryProfileFieldType(null); + + if (registry==null) + throw new IllegalArgumentException("Can not resolve query profile type '" + queryProfileName + + "' because no registry is provided"); + QueryProfileType queryProfileType=registry.getComponent(queryProfileName); + if (queryProfileType==null) + throw new IllegalArgumentException("Could not resolve query profile type '" + queryProfileName + "'"); + return new QueryProfileFieldType(registry.getComponent(queryProfileName)); + } + + public QueryProfileFieldType() { this(null); } + + public QueryProfileFieldType(QueryProfileType type) { + this.type = type; + } + + /** Returns the query profile type of this, or null if any type works */ + public QueryProfileType getQueryProfileType() { return type; } + + public @Override Class<?> getValueClass() { return QueryProfile.class; } + + public @Override String stringValue() { + return "query-profile" + (type!=null ? ":" + type.getId().getName() : ""); + } + + public @Override String toString() { + return "field type " + stringValue(); + } + + public @Override String toInstanceDescription() { + return "reference to a query profile" + (type!=null ? " of type '" + type.getId().getName() + "'" : ""); + } + + @Override + public CompiledQueryProfile convertFrom(Object object, CompiledQueryProfileRegistry registry) { + String profileId = object.toString(); + if (profileId.startsWith("ref:")) + profileId = profileId.substring("ref:".length()); + CompiledQueryProfile profile = registry.getComponent(profileId); + if (profile == null) return null; + if (type != null && ! type.equals(profile.getType())) return null; + return profile; + } + + @Override + public QueryProfile convertFrom(Object object, QueryProfileRegistry registry) { + QueryProfile profile; + if (object instanceof String) + profile = registry.getComponent((String)object); + else if (object instanceof QueryProfile) + profile = (QueryProfile)object; + else + return null; + + // Verify its type as well + if (type!=null && type!=profile.getType()) return null; + return profile; + } + + @Override + public int hashCode() { + if (type == null) return 17; + return type.hashCode(); + } + + @Override + public boolean equals(Object o) { + if (o == this) return true; + if ( ! (o instanceof QueryProfileFieldType)) return false; + QueryProfileFieldType other = (QueryProfileFieldType)o; + return equals(this.type.getId(), other.type.getId()); + } + + private boolean equals(Object o1, Object o2) { + if (o1 == null) return o2 == null; + return o1.equals(o2); + } + +} diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/types/QueryProfileType.java b/container-search/src/main/java/com/yahoo/search/query/profile/types/QueryProfileType.java new file mode 100644 index 00000000000..ecf60f8723d --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/query/profile/types/QueryProfileType.java @@ -0,0 +1,355 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.query.profile.types; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.yahoo.component.ComponentId; +import com.yahoo.component.provider.FreezableSimpleComponent; +import com.yahoo.processing.request.CompoundName; +import com.yahoo.search.query.profile.QueryProfile; + +import java.util.*; + +import static com.yahoo.text.Lowercase.toLowerCase; + +/** + * Defines a kind of query profiles + * + * @author bratseth + */ +public class QueryProfileType extends FreezableSimpleComponent { + + /** The fields of this query profile type */ + private Map<String, FieldDescription> fields = new HashMap<>(); + + /** The query profile types this inherits */ + private List<QueryProfileType> inherited = new ArrayList<>(); + + /** If this is true, keys which are not declared in this type cannot be set in instances */ + private boolean strict = false; + + /** True if the name of instances of this profile should be matched as path names, see QueryProfileRegistry */ + private boolean matchAsPath = false; + + private boolean builtin = false; + + /** Aliases *from* any strings *to* field names. Aliases are case insensitive */ + private Map<String, String> aliases = null; + + public QueryProfileType(String idString) { + this(new ComponentId(idString)); + } + + public QueryProfileType(ComponentId id) { + super(id); + QueryProfile.validateName(id.getName()); + } + + private QueryProfileType(ComponentId id, Map<String, FieldDescription> fields, List<QueryProfileType> inherited, + boolean strict, boolean matchAsPath, boolean builtin, Map<String,String> aliases) { + super(id); + this.fields = new HashMap<>(fields); + this.inherited = new ArrayList<>(inherited); + this.strict = strict; + this.matchAsPath = matchAsPath; + this.builtin = builtin; + this.aliases = aliases == null ? null : new HashMap<>(aliases); + } + + /** Return this is it is not frozen, returns a modifiable deeply unfrozen copy otherwise */ + public QueryProfileType unfrozen() { + if ( ! isFrozen()) return this; + + // Unfreeze inherited query profile references + List<QueryProfileType> unfrozenInherited = new ArrayList<>(); + for (QueryProfileType inheritedType : inherited) { + unfrozenInherited.add(inheritedType.unfrozen()); + } + + // Unfreeze nested query profile references + Map<String, FieldDescription> unfrozenFields = new HashMap<>(); + for (Map.Entry<String, FieldDescription> field : fields.entrySet()) { + FieldDescription unfrozenFieldValue = field.getValue(); + if (field.getValue().getType() instanceof QueryProfileFieldType) { + QueryProfileFieldType queryProfileFieldType = (QueryProfileFieldType)field.getValue().getType(); + if (queryProfileFieldType.getQueryProfileType() != null) { + QueryProfileFieldType unfrozenType = + new QueryProfileFieldType(queryProfileFieldType.getQueryProfileType().unfrozen()); + unfrozenFieldValue = field.getValue().withType(unfrozenType); + } + } + unfrozenFields.put(field.getKey(), unfrozenFieldValue); + } + + return new QueryProfileType(getId(), unfrozenFields, unfrozenInherited, strict, matchAsPath, builtin, aliases); + } + + /** Mark this type as built into the system. Do not use */ + public void setBuiltin(boolean builtin) { this.builtin=builtin; } + + /** Returns whether this type is built into the system */ + public boolean isBuiltin() { return builtin; } + + /** + * Returns the query profile types inherited from this (never null). + * If this profile type is not frozen, this list can be modified to change the set of inherited types. + * If it is frozen, the returned list is immutable. + */ + public List<QueryProfileType> inherited() { return inherited; } + + /** + * Returns the fields declared in this (i.e not including those inherited) as an immutable map. + * + * @throws IllegalStateException if this is frozen + */ + public Map<String,FieldDescription> declaredFields() { + ensureNotFrozen(); + return Collections.unmodifiableMap(fields); + } + + /** + * Returns true if <i>this</i> is declared strict. + * @throws IllegalStateException if this is frozen + */ + public boolean isDeclaredStrict() { + ensureNotFrozen(); + return strict; + } + + /** + * Returns true if <i>this</i> is declared as match as path. + * @throws IllegalStateException if this is frozen + */ + public boolean getDeclaredMatchAsPath() { + ensureNotFrozen(); + return matchAsPath; + } + + /** Set whether nondeclared fields are permissible. Throws an exception if this is frozen. */ + public void setStrict(boolean strict) { + ensureNotFrozen(); + this.strict=strict; + } + + /** Returns whether field not declared in this type is permissible in instances. Default is false: Additional values are allowed */ + public boolean isStrict() { + if (isFrozen()) return strict; + + // Check if any of this or an inherited is true + if (strict) return true; + for (QueryProfileType inheritedType : inherited) + if (inheritedType.isStrict()) return true; + return false; + } + + /** Returns whether instances of this should be matched as path names. Throws if this is frozen. */ + public void setMatchAsPath(boolean matchAsPath) { + ensureNotFrozen(); + this.matchAsPath=matchAsPath; + } + + /** Returns whether instances of this should be matched as path names. Default is false: Use exact name matching. */ + public boolean getMatchAsPath() { + if (isFrozen()) return matchAsPath; + + // Check if any of this or an inherited is true + if (matchAsPath) return true; + for (QueryProfileType inheritedType : inherited) + if (inheritedType.getMatchAsPath()) return true; + return false; + } + + public void freeze() { + if (isFrozen()) return; + // Flatten the inheritance hierarchy into this to facilitate faster lookup + for (QueryProfileType inheritedType : inherited) { + for (FieldDescription field : inheritedType.fields().values()) + if ( ! fields.containsKey(field.getName())) + fields.put(field.getName(),field); + } + fields = ImmutableMap.copyOf(fields); + inherited = ImmutableList.copyOf(inherited); + strict = isStrict(); + matchAsPath = getMatchAsPath(); + super.freeze(); + } + + /** + * Returns whether the given field name is overridable in this type. + * Default: true (so all non-declared fields returns true) + */ + public boolean isOverridable(String fieldName) { + FieldDescription field=getField(fieldName); + if (field==null) return true; + return field.isOverridable(); + } + + /** + * Returns the permissible class for the value of the given name in this type + * + * @return the permissible class for a value, <code>Object</code> if all types are legal, + * null if no types are legal (i.e if the name is not legal) + */ + public Class<?> getValueClass(String name) { + FieldDescription fieldDescription=getField(name); + if (fieldDescription==null) { + if (strict) + return null; // Undefined -> Not legal + else + return Object.class; // Undefined -> Anything is legal + } + return fieldDescription.getType().getValueClass(); + } + + /** Returns the type of the given query profile type declared as a field in this */ + public QueryProfileType getType(String localName) { + FieldDescription fieldDescription=getField(localName); + if (fieldDescription ==null) return null; + if ( ! (fieldDescription.getType() instanceof QueryProfileFieldType)) return null; + return ((QueryProfileFieldType) fieldDescription.getType()).getQueryProfileType(); + } + + /** + * Returns the description of the field with the given name in this type or an inherited type + * (depth first left to right search). Returns null if the field is not defined in this or an inherited profile. + */ + public FieldDescription getField(String name) { + FieldDescription field=fields.get(name); + if ( field!=null ) return field; + + if ( isFrozen() ) return null; // Inherited are collapsed into this + + for (QueryProfileType inheritedType : this.inherited() ) { + field=inheritedType.getField(name); + if (field!=null) return field; + } + + return null; + } + + /** + * Removes a field from this (not from any inherited profile) + * + * @return the removed field or null if none + * @throws IllegalStateException if this is frozen + */ + public FieldDescription removeField(String fieldName) { + ensureNotFrozen(); + return fields.remove(fieldName); + } + + /** + * Adds a field to this, without associating with a type registry; field descriptions with compound + * is not be supported. + * + * @throws IllegalStateException if this is frozen + */ + public void addField(FieldDescription fieldDescription) { + // Compound names translates to new types, which must be added to a supplied registry + if (fieldDescription.getCompoundName().isCompound()) + throw new IllegalArgumentException("Adding compound names is only legal when supplying a registry"); + addField(fieldDescription, null); + } + + /** + * Adds a field to this + * + * @throws IllegalStateException if this is frozen + */ + public void addField(FieldDescription fieldDescription, QueryProfileTypeRegistry registry) { + CompoundName name = fieldDescription.getCompoundName(); + if (name.isCompound()) { + // Add (/to) a query profile type containing the rest of the name. + // (we do not need the field description settings for intermediate query profile types + // as the leaf entry will enforce them) + QueryProfileType type = getOrCreateQueryProfileType(name.first(), registry); + type.addField(fieldDescription.withName(name.rest()), registry); + } + else { + ensureNotFrozen(); + fields.put(fieldDescription.getName(), fieldDescription); + } + + for (String alias : fieldDescription.getAliases()) + addAlias(alias, fieldDescription.getName()); + } + + private QueryProfileType getOrCreateQueryProfileType(String name, QueryProfileTypeRegistry registry) { + FieldDescription fieldDescription = getField(name); + if (fieldDescription != null) { + if ( ! ( fieldDescription.getType() instanceof QueryProfileFieldType)) + throw new IllegalArgumentException("Cannot use name '" + name + "' as a prefix because it is " + + "already a " + fieldDescription.getType()); + QueryProfileFieldType fieldType = (QueryProfileFieldType) fieldDescription.getType(); + QueryProfileType type = fieldType.getQueryProfileType(); + if (type == null) { // an as-yet untyped reference; add type + type = new QueryProfileType(name); + registry.register(type.getId(), type); + fields.put(name, fieldDescription.withType(new QueryProfileFieldType(type))); + } + return type; + } + else { + QueryProfileType type = new QueryProfileType(name); + registry.register(type.getId(), type); + fields.put(name, new FieldDescription(name, new QueryProfileFieldType(type))); + return type; + } + } + + private void addAlias(String alias,String field) { + ensureNotFrozen(); + if (aliases==null) + aliases=new HashMap<>(); + aliases.put(toLowerCase(alias),field); + } + + /** Returns all the fields of this profile type and all types it inherits as a read-only map */ + public Map<String,FieldDescription> fields() { + if (isFrozen()) return fields; + if (inherited().size()==0) return Collections.unmodifiableMap(fields); + + // Collapse inherited + Map<String,FieldDescription> allFields=new HashMap<>(fields); + for (QueryProfileType inheritedType : inherited) + allFields.putAll(inheritedType.fields()); + return Collections.unmodifiableMap(allFields); + } + + /** + * Returns the alias to field mapping of this type as a read-only map. This is never null. + * Note that all keys are lower-cased because aliases are case-insensitive + */ + public Map<String,String> aliases() { + if (isFrozen()) return aliases; + if (aliases == null) return Collections.emptyMap(); + return Collections.unmodifiableMap(aliases); + } + + /** Returns the field name of an alias or field name */ + public String unalias(String aliasOrField) { + if (aliases==null || aliases.isEmpty()) return aliasOrField; + String field=aliases.get(toLowerCase(aliasOrField)); + if (field!=null) return field; + return aliasOrField; + } + + @Override + public int hashCode() { + return getId().hashCode(); + } + + /** Two types are equal if they have the same id */ + @Override + public boolean equals(Object o) { + if (o == this) return true; + if ( ! (o instanceof QueryProfileType)) return false; + QueryProfileType other = (QueryProfileType)o; + return other.getId().equals(this.getId()); + } + + public String toString() { + return "query profile type '" + getId() + "'"; + } + +} diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/types/QueryProfileTypeRegistry.java b/container-search/src/main/java/com/yahoo/search/query/profile/types/QueryProfileTypeRegistry.java new file mode 100644 index 00000000000..3f64caa7ab1 --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/query/profile/types/QueryProfileTypeRegistry.java @@ -0,0 +1,37 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.query.profile.types; + +import com.yahoo.component.provider.ComponentRegistry; +import com.yahoo.search.Query; +import com.yahoo.search.query.profile.QueryProfileRegistry; + +/** + * A registry of query profile types + * + * @author bratseth + */ +public class QueryProfileTypeRegistry extends ComponentRegistry<QueryProfileType> { + + public QueryProfileTypeRegistry() { + Query.addNativeQueryProfileTypesTo(this); + } + + /** Register this type by its id */ + public void register(QueryProfileType type) { + super.register(type.getId(), type); + } + + @Override + public void freeze() { + if (isFrozen()) return; + for (QueryProfileType queryProfileType : allComponents()) + queryProfileType.freeze(); + } + + public static QueryProfileTypeRegistry emptyFrozen() { + QueryProfileTypeRegistry registry = new QueryProfileTypeRegistry(); + registry.freeze(); + return registry; + } + +} diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/types/TensorFieldType.java b/container-search/src/main/java/com/yahoo/search/query/profile/types/TensorFieldType.java new file mode 100644 index 00000000000..747cf73acb3 --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/query/profile/types/TensorFieldType.java @@ -0,0 +1,59 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.query.profile.types; + +import com.yahoo.search.query.profile.QueryProfileRegistry; +import com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry; +import com.yahoo.tensor.MapTensor; +import com.yahoo.tensor.Tensor; +import com.yahoo.tensor.TensorType; + +import java.util.Optional; + +/** + * A tensor field type in a query profile + * + * @author bratseth + */ +public class TensorFieldType extends FieldType { + + private final Optional<TensorType> type; + + /** Creates a tensor field type with optional information about the kind of tensor this will hold */ + public TensorFieldType(Optional<TensorType> type) { + this.type = type; + } + + /** Returns information about the type of tensor this will hold, or empty to allow any kind of tensor */ + public Optional<TensorType> type() { return type; } + + @Override + public Class getValueClass() { return Tensor.class; } + + @Override + public String stringValue() { return "tensor"; } + + @Override + public String toString() { return "field type " + stringValue(); } + + @Override + public String toInstanceDescription() { return "a tensor"; } + + @Override + public Object convertFrom(Object o, QueryProfileRegistry registry) { + if (o instanceof Tensor) return o; + if (o instanceof String) return MapTensor.from((String)o); + return null; + } + + @Override + public Object convertFrom(Object o, CompiledQueryProfileRegistry registry) { + return convertFrom(o, (QueryProfileRegistry)null); + } + + public static TensorFieldType fromTypeString(String s) { + if (s.equals("tensor")) return genericTensorType; + return new TensorFieldType(Optional.of(TensorType.fromSpec(s))); + } + + +} diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/types/package-info.java b/container-search/src/main/java/com/yahoo/search/query/profile/types/package-info.java new file mode 100644 index 00000000000..1f9fa7a1fb4 --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/query/profile/types/package-info.java @@ -0,0 +1,11 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * Query profile types defines the set of fields a query profile may, can or must have. Query profile + * types may be inherited in a type hierarchy. + */ +@ExportPackage +@PublicApi +package com.yahoo.search.query.profile.types; + +import com.yahoo.api.annotations.PublicApi; +import com.yahoo.osgi.annotation.ExportPackage; |