diff options
156 files changed, 2165 insertions, 848 deletions
diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastSearcher.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastSearcher.java index 90410699748..c93220f0a85 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastSearcher.java @@ -203,7 +203,7 @@ public class FastSearcher extends VespaBackEndSearcher { /** When we only search a single node, doing all grouping in one pass is more efficient */ private void forceSinglePassGrouping(Query query) { - for (GroupingRequest groupingRequest : GroupingRequest.getRequests(query)) + for (GroupingRequest groupingRequest : query.getSelect().getGrouping()) forceSinglePassGrouping(groupingRequest.getRootOperation()); } diff --git a/container-search/src/main/java/com/yahoo/search/Query.java b/container-search/src/main/java/com/yahoo/search/Query.java index cfb1c9a26be..53435941e26 100644 --- a/container-search/src/main/java/com/yahoo/search/Query.java +++ b/container-search/src/main/java/com/yahoo/search/Query.java @@ -972,6 +972,7 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { private void copyPropertiesTo(Query clone) { clone.model = model.cloneFor(clone); + clone.select = select.cloneFor(clone); clone.ranking = (Ranking) ranking.clone(); clone.presentation = (Presentation) presentation.clone(); clone.context = getContext(true).cloneFor(clone); diff --git a/container-search/src/main/java/com/yahoo/search/grouping/Continuation.java b/container-search/src/main/java/com/yahoo/search/grouping/Continuation.java index eda2b449fda..d7ee3fedfc9 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/Continuation.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/Continuation.java @@ -21,4 +21,8 @@ public abstract class Continuation { public static Continuation fromString(String str) { return ContinuationDecoder.decode(str); } + + /** Returns a deep copy of this */ + public abstract Continuation copy(); + } diff --git a/container-search/src/main/java/com/yahoo/search/grouping/GroupingRequest.java b/container-search/src/main/java/com/yahoo/search/grouping/GroupingRequest.java index 8ce0d90dfc5..754d3b562d3 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/GroupingRequest.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/GroupingRequest.java @@ -8,6 +8,7 @@ import com.yahoo.search.Result; import com.yahoo.search.grouping.request.GroupingOperation; import com.yahoo.search.grouping.result.Group; import com.yahoo.search.grouping.result.RootGroup; +import com.yahoo.search.query.Select; import com.yahoo.search.result.Hit; import java.util.*; @@ -19,35 +20,53 @@ import java.util.*; * result {@link Group} using the {@link #getResultGroup(Result)} method. * * @author Simon Thoresen Hult + * @author bratseth */ public class GroupingRequest { - private final static CompoundName PROP_REQUEST = new CompoundName(GroupingRequest.class.getName() + ".Request"); + private final Select parent; private final List<Continuation> continuations = new ArrayList<>(); - private final int requestId; private GroupingOperation root; private TimeZone timeZone; private URI resultId; - private GroupingRequest(int requestId) { - this.requestId = requestId; + private GroupingRequest(Select parent) { + this.parent = parent; + } + + private GroupingRequest(Select parent, + List<Continuation> continuations, + GroupingOperation root, + TimeZone timeZone, + URI resultId) { + this.parent = parent; + continuations.forEach(item -> this.continuations.add(item.copy())); + this.root = root != null ? root.copy(null) : null; + this.timeZone = timeZone; + this.resultId = resultId; + } + + /** Returns a deep copy of this */ + public GroupingRequest copy(Select parentOfCopy) { + return new GroupingRequest(parentOfCopy, continuations, root, timeZone, resultId); } /** - * Returns the id of this GroupingRequest. This id is injected into the {@link RootGroup} of the final result, and + * Returns the id of this GroupingRequest. + * This id is injected into the {@link RootGroup} of the final result, and * allows tracking of per-request meta data. * - * @return The id of this. + * @return the id of this request, or -1 if it has been removed from the query select statement */ public int getRequestId() { - return requestId; + return parent.getGrouping().indexOf(this); } /** * Returns the root {@link GroupingOperation} that defines this request. As long as this remains unset, the request * is void. * - * @return The root operation. + * @return the root operation. */ public GroupingOperation getRootOperation() { return root; @@ -57,8 +76,8 @@ public class GroupingRequest { * Sets the root {@link GroupingOperation} that defines this request. As long as this remains unset, the request is * void. * - * @param root The root operation to set. - * @return This, to allow chaining. + * @param root the root operation to set. + * @return this, to allow chaining. */ public GroupingRequest setRootOperation(GroupingOperation root) { this.root = root; @@ -69,7 +88,7 @@ public class GroupingRequest { * Returns the {@link TimeZone} used when resolving time expressions such as {@link * com.yahoo.search.grouping.request.DayOfMonthFunction} and {@link com.yahoo.search.grouping.request.HourOfDayFunction}. * - * @return The time zone in use. + * @return the time zone in use. */ public TimeZone getTimeZone() { return timeZone; @@ -79,8 +98,8 @@ public class GroupingRequest { * Sets the {@link TimeZone} used when resolving time expressions such as {@link * com.yahoo.search.grouping.request.DayOfMonthFunction} and {@link com.yahoo.search.grouping.request.HourOfDayFunction}. * - * @param timeZone The time zone to set. - * @return This, to allow chaining. + * @param timeZone the time zone to set. + * @return this, to allow chaining. */ public GroupingRequest setTimeZone(TimeZone timeZone) { this.timeZone = timeZone; @@ -92,8 +111,8 @@ public class GroupingRequest { * search returns. Because searchers are allowed to modify both {@link Result} and {@link Hit} objects freely, this * method requires that you pass it the current {@link Result} object as argument. * - * @param result The search result that contains the root group. - * @return The result {@link RootGroup} of this request, or null if not found. + * @param result the search result that contains the root group. + * @return the result {@link RootGroup} of this request, or null if not found. */ public RootGroup getResultGroup(Result result) { Hit root = result.hits().get(resultId, -1); @@ -107,8 +126,8 @@ public class GroupingRequest { * Sets the result {@link RootGroup} of this request. This is used by the executing grouping searcher, and should * not be called by a requesting searcher. * - * @param group The result to set. - * @return This, to allow chaining. + * @param group the result to set. + * @return this, to allow chaining. */ public GroupingRequest setResultGroup(RootGroup group) { this.resultId = group.getId(); @@ -119,46 +138,41 @@ public class GroupingRequest { * Returns the list of {@link Continuation}s of this request. This is used by the executing grouping searcher to * allow pagination of grouping results. * - * @return The list of Continuations. + * @return the list of Continuations. */ public List<Continuation> continuations() { return continuations; } /** - * Creates and attaches a new instance of this class to the given {@link Query}. This is necessary to allow {@link - * #getRequests(Query)} to return all created requests. + * Creates a new grouping request and adds it to the query.getSelect().getGrouping() list * - * @param query The query to attach the request to. + * @param query the query to attach the request to. * @return The created request. */ public static GroupingRequest newInstance(Query query) { - List<GroupingRequest> lst = getRequests(query); - if (lst.isEmpty()) { - lst = new LinkedList<>(); - query.properties().set(PROP_REQUEST, lst); - } - GroupingRequest ret = new GroupingRequest(lst.size()); - lst.add(ret); - return ret; + GroupingRequest newRequest = new GroupingRequest(query.getSelect()); + query.getSelect().getGrouping().add(newRequest); + return newRequest; } /** * Returns all instances of this class that have been attached to the given {@link Query}. If no requests have been * attached to the {@link Query}, this method returns an empty list. * - * @param query The query whose requests to return. - * @return The list of grouping requests. + * @param query the query whose requests to return. + * @return the list of grouping requests. + * @deprecated use query.getSelect().getGrouping() */ @SuppressWarnings({ "unchecked" }) + @Deprecated public static List<GroupingRequest> getRequests(Query query) { - Object lst = query.properties().get(PROP_REQUEST); - if (lst == null) { - return Collections.emptyList(); - } - if (!(lst instanceof List)) { - throw new IllegalArgumentException("Expected " + GroupingRequest.class + ", got " + lst.getClass() + "."); - } - return (List<GroupingRequest>)lst; + return query.getSelect().getGrouping(); } + + @Override + public String toString() { + return root == null ? "(empty)" : root.toString(); + } + } diff --git a/container-search/src/main/java/com/yahoo/search/grouping/GroupingValidator.java b/container-search/src/main/java/com/yahoo/search/grouping/GroupingValidator.java index 6379c43bc87..d96f490909e 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/GroupingValidator.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/GroupingValidator.java @@ -63,9 +63,8 @@ public class GroupingValidator extends Searcher { public Result search(Query query, Execution execution) { if (enabled && query.properties().getBoolean(PARAM_ENABLED, true)) { ExpressionVisitor visitor = new MyVisitor(); - for (GroupingRequest req : GroupingRequest.getRequests(query)) { + for (GroupingRequest req : query.getSelect().getGrouping()) req.getRootOperation().visitExpressions(visitor); - } } return execution.search(query); } diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/AddFunction.java b/container-search/src/main/java/com/yahoo/search/grouping/request/AddFunction.java index 8cb27569e67..420861d2f6c 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/request/AddFunction.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/request/AddFunction.java @@ -2,12 +2,14 @@ package com.yahoo.search.grouping.request; import java.util.List; +import java.util.stream.Collectors; /** * This class represents an add-function in a {@link GroupingExpression}. It evaluates to a number that equals the * result of adding the results of all arguments together in the order they were given to the constructor. * * @author Simon Thoresen Hult + * @author bratseth */ public class AddFunction extends FunctionNode { @@ -19,11 +21,18 @@ public class AddFunction extends FunctionNode { * @param argN The optional arguments, must evaluate to a number. */ public AddFunction(GroupingExpression arg1, GroupingExpression arg2, GroupingExpression... argN) { - this(asList(arg1, arg2, argN)); + this(null, null, asList(arg1, arg2, argN)); } - private AddFunction(List<GroupingExpression> args) { - super("add", args); + private AddFunction(String label, Integer level, List<GroupingExpression> args) { + super("add", label, level, args); + } + + @Override + public AddFunction copy() { + return new AddFunction(getLabel(), + getLevelOrNull(), + args().stream().map(arg -> arg.copy()).collect(Collectors.toList())); } /** @@ -37,6 +46,6 @@ public class AddFunction extends FunctionNode { if (args.size() < 2) { throw new IllegalArgumentException("Expected 2 or more arguments, got " + args.size() + "."); } - return new AddFunction(args); + return new AddFunction(null, null, args); } } diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/AggregatorNode.java b/container-search/src/main/java/com/yahoo/search/grouping/request/AggregatorNode.java index 323f1ac769a..6166833999a 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/request/AggregatorNode.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/request/AggregatorNode.java @@ -12,13 +12,13 @@ public abstract class AggregatorNode extends GroupingExpression { private final GroupingExpression exp; - protected AggregatorNode(String image) { - super(image + "()"); + protected AggregatorNode(String image, String label, Integer level) { + super(image + "()", label, level); this.exp = null; } - protected AggregatorNode(String image, GroupingExpression exp) { - super(image + "(" + exp.toString() + ")"); + protected AggregatorNode(String image, String label, Integer level, GroupingExpression exp) { + super(image + "(" + exp.toString() + ")", label, level); this.exp = exp; } diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/AllOperation.java b/container-search/src/main/java/com/yahoo/search/grouping/request/AllOperation.java index b809706331f..93b46d95d89 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/request/AllOperation.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/request/AllOperation.java @@ -1,11 +1,16 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.grouping.request; +import java.util.List; +import java.util.Map; +import java.util.Set; + /** * This is a grouping operation that processes the input list as a whole, as opposed to {@link EachOperation} which * processes each element of that list separately. * * @author Simon Thoresen Hult + * @author bratseth */ public class AllOperation extends GroupingOperation { @@ -13,6 +18,44 @@ public class AllOperation extends GroupingOperation { * Constructs a new instance of this class. */ public AllOperation() { - super("all"); + super("all", null); + } + + private AllOperation(GroupingOperation parentOfCopy, + String image, + String label, + List<GroupingExpression> orderBy, + List<GroupingExpression> outputs, + List<GroupingOperation> children, + Map<String, GroupingExpression> aliases, + Set<String> hints, + GroupingExpression groupBy, + String where, + boolean forceSinglePass, + double accuracy, + int precision, + int level, + int max) { + super(parentOfCopy, image, label, orderBy, outputs, children, aliases, hints, groupBy, where, forceSinglePass, accuracy, precision, level, max); } + + @Override + public AllOperation copy(GroupingOperation parentOfCopy) { + return new AllOperation(parentOfCopy, + getImage(), + getLabel(), + getOrderBy(), + getOutputs(), + getChildren(), + getAliases(), + getHints(), + getGroupBy(), + getWhere(), + getForceSinglePass(), + getAccuracy(), + getPrecision(), + getLevel(), + getMax()); + } + } diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/AndFunction.java b/container-search/src/main/java/com/yahoo/search/grouping/request/AndFunction.java index 5f2617acf16..569f0d06987 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/request/AndFunction.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/request/AndFunction.java @@ -2,12 +2,14 @@ package com.yahoo.search.grouping.request; import java.util.List; +import java.util.stream.Collectors; /** * This class represents an and-function in a {@link GroupingExpression}. It evaluates to a long that equals the result * of and'ing the results of all arguments together in the order they were given to the constructor. * * @author Simon Thoresen Hult + * @author bratseth */ public class AndFunction extends FunctionNode { @@ -19,11 +21,18 @@ public class AndFunction extends FunctionNode { * @param argN The optional arguments, must evaluate to a long. */ public AndFunction(GroupingExpression arg1, GroupingExpression arg2, GroupingExpression... argN) { - this(asList(arg1, arg2, argN)); + this(null, null, asList(arg1, arg2, argN)); } - private AndFunction(List<GroupingExpression> args) { - super("and", args); + private AndFunction(String label, Integer level, List<GroupingExpression> args) { + super("and", label, level, args); + } + + @Override + public AndFunction copy() { + return new AndFunction(getLabel(), + getLevelOrNull(), + args().stream().map(arg -> arg.copy()).collect(Collectors.toList())); } /** @@ -37,6 +46,6 @@ public class AndFunction extends FunctionNode { if (args.size() < 2) { throw new IllegalArgumentException("Expected 2 or more arguments, got " + args.size() + "."); } - return new AndFunction(args); + return new AndFunction(null, null, args); } } diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/ArrayAtLookup.java b/container-search/src/main/java/com/yahoo/search/grouping/request/ArrayAtLookup.java index ed69e8aaee7..c8e0a6be3e8 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/request/ArrayAtLookup.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/request/ArrayAtLookup.java @@ -19,34 +19,36 @@ import com.google.common.annotations.Beta; public class ArrayAtLookup extends DocumentValue { private final String attributeName; - private final GroupingExpression arg2; + private final GroupingExpression indexArgument; /** * Constructs a new instance of this class. * - * @param attributeName The attribute name to assign to this. + * @param attributeName the attribute name to assign to this. */ public ArrayAtLookup(String attributeName, GroupingExpression indexArg) { - super("array.at(" + attributeName + ", " + indexArg + ")"); + this(null, null, attributeName, indexArg); + } + + private ArrayAtLookup(String label, Integer level, String attributeName, GroupingExpression indexArgument) { + super("array.at(" + attributeName + ", " + indexArgument + ")", label, level); this.attributeName = attributeName; - this.arg2 = indexArg; + this.indexArgument = indexArgument; } - /** - * Returns the name of the attribute to retrieve from the input hit. - * - * @return The attribute name. - */ + @Override + public ArrayAtLookup copy() { + return new ArrayAtLookup(getLabel(), getLevelOrNull(), getAttributeName(), getIndexArgument().copy()); + } + + /** Returns the name of the attribute to retrieve from the input hit */ public String getAttributeName() { return attributeName; } - /** - * get the expression to evaluate before indexing - * @return grouping expression argument - */ + /** Return the expression to evaluate before indexing */ public GroupingExpression getIndexArgument() { - return arg2; + return indexArgument; } } diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/AttributeFunction.java b/container-search/src/main/java/com/yahoo/search/grouping/request/AttributeFunction.java index faf565647c7..d609800cc1c 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/request/AttributeFunction.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/request/AttributeFunction.java @@ -6,6 +6,7 @@ package com.yahoo.search.grouping.request; * named attribute in the input {@link com.yahoo.search.result.Hit}. * * @author Simon Thoresen Hult + * @author bratseth */ public class AttributeFunction extends DocumentValue { @@ -17,8 +18,17 @@ public class AttributeFunction extends DocumentValue { * @param attributeName the attribute name to assign to this. */ public AttributeFunction(String attributeName) { - super("attribute(" + attributeName + ")"); - name = attributeName; + this(null, null, attributeName); + } + + private AttributeFunction(String label, Integer level, String attributeName) { + super("attribute(" + attributeName + ")", label, level); + this.name = attributeName; + } + + @Override + public AttributeFunction copy() { + return new AttributeFunction(getLabel(), getLevelOrNull(), getAttributeName()); } /** diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/AttributeValue.java b/container-search/src/main/java/com/yahoo/search/grouping/request/AttributeValue.java index d83888b3107..1afac32d488 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/request/AttributeValue.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/request/AttributeValue.java @@ -6,6 +6,7 @@ package com.yahoo.search.grouping.request; * named attribute in the input {@link com.yahoo.search.result.Hit}. * * @author Simon Thoresen Hult + * @author bratseth */ public class AttributeValue extends DocumentValue { @@ -17,8 +18,17 @@ public class AttributeValue extends DocumentValue { * @param attributeName the attribute name to assign to this. */ public AttributeValue(String attributeName) { - super(attributeName); - name = attributeName; + this(null, null, attributeName); + } + + private AttributeValue(String label, Integer level, String attributeName) { + super(attributeName, label, level); + this.name = attributeName; + } + + @Override + public AttributeValue copy() { + return new AttributeValue(getLabel(), getLevelOrNull(), getAttributeName()); } /** diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/AvgAggregator.java b/container-search/src/main/java/com/yahoo/search/grouping/request/AvgAggregator.java index d7e3232ea5f..bf11da5d2b9 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/request/AvgAggregator.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/request/AvgAggregator.java @@ -6,15 +6,26 @@ package com.yahoo.search.grouping.request; * the contained expression evaluated to over all the inputs. * * @author Simon Thoresen Hult + * @author bratseth */ public class AvgAggregator extends AggregatorNode { /** * Constructs a new instance of this class. * - * @param exp The expression to aggregate on. + * @param expression the expression to aggregate on. */ - public AvgAggregator(GroupingExpression exp) { - super("avg", exp); + public AvgAggregator(GroupingExpression expression) { + this(null, null, expression); } + + private AvgAggregator(String label, Integer level, GroupingExpression expression) { + super("avg", label, level, expression); + } + + @Override + public AvgAggregator copy() { + return new AvgAggregator(getLabel(), getLevelOrNull(), getExpression().copy()); + } + } diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/AvgFunction.java b/container-search/src/main/java/com/yahoo/search/grouping/request/AvgFunction.java index 55c4d355589..775f6c48d66 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/request/AvgFunction.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/request/AvgFunction.java @@ -2,12 +2,14 @@ package com.yahoo.search.grouping.request; import java.util.List; +import java.util.stream.Collectors; /** * This class represents a min-function in a {@link GroupingExpression}. It evaluates to a number that equals the * average of the results of all arguments. * * @author Simon Thoresen Hult + * @author bratseth */ public class AvgFunction extends FunctionNode { @@ -19,11 +21,18 @@ public class AvgFunction extends FunctionNode { * @param argN The optional arguments, must evaluate to a number. */ public AvgFunction(GroupingExpression arg1, GroupingExpression arg2, GroupingExpression... argN) { - this(asList(arg1, arg2, argN)); + this(null, null, asList(arg1, arg2, argN)); } - private AvgFunction(List<GroupingExpression> args) { - super("avg", args); + private AvgFunction(String label, Integer level, List<GroupingExpression> args) { + super("avg", label, level, args); + } + + @Override + public AvgFunction copy() { + return new AvgFunction(getLabel(), + getLevelOrNull(), + args().stream().map(arg -> arg.copy()).collect(Collectors.toList())); } /** @@ -37,6 +46,6 @@ public class AvgFunction extends FunctionNode { if (args.size() < 2) { throw new IllegalArgumentException("Expected 2 or more arguments, got " + args.size() + "."); } - return new AvgFunction(args); + return new AvgFunction(null, null, args); } } diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/BooleanValue.java b/container-search/src/main/java/com/yahoo/search/grouping/request/BooleanValue.java index 8b8903c9a8d..32ef3c16d2e 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/request/BooleanValue.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/request/BooleanValue.java @@ -4,7 +4,8 @@ package com.yahoo.search.grouping.request; /** * This class represents a constant {@link Boolean} value in a {@link GroupingExpression}. * - * @author <a href="mailto:lulf@yahoo-inc.com">Ulf Lilleengen</a> + * @author Ulf Lilleengen + * @author bratseth */ public class BooleanValue extends ConstantValue<Boolean> { @@ -14,6 +15,16 @@ public class BooleanValue extends ConstantValue<Boolean> { * @param value The immutable value to assign to this. */ public BooleanValue(Boolean value) { - super(value); + super(null, null, value); } + + private BooleanValue(String label, Integer level, Boolean value) { + super(label, level, value); + } + + @Override + public BooleanValue copy() { + return new BooleanValue(getLabel(), getLevelOrNull(), getValue()); + } + } diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/BucketValue.java b/container-search/src/main/java/com/yahoo/search/grouping/request/BucketValue.java index c73b7199394..392c6386eb7 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/request/BucketValue.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/request/BucketValue.java @@ -14,8 +14,8 @@ public class BucketValue extends GroupingExpression implements Comparable<Bucket private final ConstantValue<?> to; private final ConstantValueComparator comparator = new ConstantValueComparator(); - protected BucketValue(ConstantValue<?> inclusiveFrom, ConstantValue<?> exclusiveTo) { - super("bucket[" + asImage(inclusiveFrom) + ", " + asImage(exclusiveTo) + ">"); + protected BucketValue(String label, Integer level, ConstantValue<?> inclusiveFrom, ConstantValue<?> exclusiveTo) { + super("bucket[" + asImage(inclusiveFrom) + ", " + asImage(exclusiveTo) + ">", label, level); if (comparator.compare(exclusiveTo, inclusiveFrom) < 0) { throw new IllegalArgumentException("Bucket to-value can not be less than from-value."); } @@ -23,6 +23,11 @@ public class BucketValue extends GroupingExpression implements Comparable<Bucket to = exclusiveTo; } + @Override + public BucketValue copy() { + return new BucketValue(getLabel(), getLevelOrNull(), getFrom().copy(), getTo().copy()); + } + /** * Returns the inclusive-from value of this bucket. * diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/CatFunction.java b/container-search/src/main/java/com/yahoo/search/grouping/request/CatFunction.java index 5e51c8e35a0..dddc3df0542 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/request/CatFunction.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/request/CatFunction.java @@ -2,12 +2,14 @@ package com.yahoo.search.grouping.request; import java.util.List; +import java.util.stream.Collectors; /** * This class represents a cat-function in a {@link GroupingExpression}. It evaluates to a byte array that equals the * concatenation of the binary result of all arguments in the order they were given to the constructor. * * @author Simon Thoresen Hult + * @author bratseth */ public class CatFunction extends FunctionNode { @@ -19,11 +21,18 @@ public class CatFunction extends FunctionNode { * @param argN The optional arguments. */ public CatFunction(GroupingExpression arg1, GroupingExpression arg2, GroupingExpression... argN) { - this(asList(arg1, arg2, argN)); + this(null, null, asList(arg1, arg2, argN)); } - private CatFunction(List<GroupingExpression> args) { - super("cat", args); + private CatFunction(String label, Integer level, List<GroupingExpression> args) { + super("cat", label, level, args); + } + + @Override + public CatFunction copy() { + return new CatFunction(getLabel(), + getLevelOrNull(), + args().stream().map(arg -> arg.copy()).collect(Collectors.toList())); } /** @@ -37,6 +46,6 @@ public class CatFunction extends FunctionNode { if (args.size() < 2) { throw new IllegalArgumentException("Expected 2 or more arguments, got " + args.size() + "."); } - return new CatFunction(args); + return new CatFunction(null, null, args); } } diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/ConstantValue.java b/container-search/src/main/java/com/yahoo/search/grouping/request/ConstantValue.java index ad8c1ef1cc9..81712424829 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/request/ConstantValue.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/request/ConstantValue.java @@ -7,23 +7,24 @@ package com.yahoo.search.grouping.request; * data types are represented as subclasses of this. * * @author Simon Thoresen Hult + * @author bratseth */ @SuppressWarnings("rawtypes") public abstract class ConstantValue<T extends Comparable> extends GroupingExpression { private final T value; - protected ConstantValue(T value) { - super(asImage(value)); + protected ConstantValue(String label, Integer level, T value) { + super(asImage(value), label, level); this.value = value; } - /** - * Returns the constant value of this. - * - * @return The value. - */ + @Override + public abstract ConstantValue copy(); + + /** Returns the constant value of this */ public T getValue() { return value; } + } diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/ConstantValueComparator.java b/container-search/src/main/java/com/yahoo/search/grouping/request/ConstantValueComparator.java index 543a1f495c6..61118e04784 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/request/ConstantValueComparator.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/request/ConstantValueComparator.java @@ -7,10 +7,11 @@ import java.util.Comparator; * This class compares two constant values, and takes into account that one of * the arguments may be the very special infinity value. * - * @author <a href="mailto:lulf@yahoo-inc.com">Ulf Lilleengen</a> + * @author Ulf Lilleengen */ @SuppressWarnings("rawtypes") public class ConstantValueComparator implements Comparator<ConstantValue> { + @SuppressWarnings("unchecked") @Override public int compare(ConstantValue lhs, ConstantValue rhs) { diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/CountAggregator.java b/container-search/src/main/java/com/yahoo/search/grouping/request/CountAggregator.java index 19a6c939087..b9d691500ad 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/request/CountAggregator.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/request/CountAggregator.java @@ -6,13 +6,22 @@ package com.yahoo.search.grouping.request; * there are in the input. * * @author Simon Thoresen Hult + * @author bratseth */ public class CountAggregator extends AggregatorNode { - /** - * Constructs a new instance of this class. - */ + /** Constructs a new instance of this class. */ public CountAggregator() { - super("count"); + this(null, null); } + + private CountAggregator(String label, Integer level) { + super("count", label, level); + } + + @Override + public CountAggregator copy() { + return new CountAggregator(getLabel(), getLevelOrNull()); + } + } diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/DateFunction.java b/container-search/src/main/java/com/yahoo/search/grouping/request/DateFunction.java index b3a4c451b6b..0c2c4928a91 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/request/DateFunction.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/request/DateFunction.java @@ -2,12 +2,14 @@ package com.yahoo.search.grouping.request; import java.util.Arrays; +import java.util.stream.Collectors; /** * This class represents a timestamp-formatter function in a {@link GroupingExpression}. It evaluates to a string on the * form "YYYY-MM-DD" of the result of the argument. * * @author Simon Thoresen Hult + * @author bratseth */ public class DateFunction extends FunctionNode { @@ -17,6 +19,18 @@ public class DateFunction extends FunctionNode { * @param exp The expression to evaluate, must evaluate to a long. */ public DateFunction(GroupingExpression exp) { - super("time.date", Arrays.asList(exp)); + this(null, null, exp); } + + private DateFunction(String label, Integer level, GroupingExpression exp) { + super("time.date", label, level, Arrays.asList(exp)); + } + + @Override + public DateFunction copy() { + return new DateFunction(getLabel(), + getLevelOrNull(), + getArg(0).copy()); + } + } diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/DayOfMonthFunction.java b/container-search/src/main/java/com/yahoo/search/grouping/request/DayOfMonthFunction.java index 5d82a80352b..62c23804344 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/request/DayOfMonthFunction.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/request/DayOfMonthFunction.java @@ -2,12 +2,14 @@ package com.yahoo.search.grouping.request; import java.util.Arrays; +import java.util.stream.Collectors; /** * This class represents a day-of-month timestamp-function in a {@link GroupingExpression}. It evaluates to a long that * equals the day of month (1-31) of the result of the argument. * * @author Simon Thoresen Hult + * @author bratseth */ public class DayOfMonthFunction extends FunctionNode { @@ -17,6 +19,18 @@ public class DayOfMonthFunction extends FunctionNode { * @param exp The expression to evaluate, must evaluate to a long. */ public DayOfMonthFunction(GroupingExpression exp) { - super("time.dayofmonth", Arrays.asList(exp)); + this(null, null, exp); } + + private DayOfMonthFunction(String label, Integer level, GroupingExpression exp) { + super("time.dayofmonth", label, level, Arrays.asList(exp)); + } + + @Override + public DayOfMonthFunction copy() { + return new DayOfMonthFunction(getLabel(), + getLevelOrNull(), + getArg(0).copy()); + } + } diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/DayOfWeekFunction.java b/container-search/src/main/java/com/yahoo/search/grouping/request/DayOfWeekFunction.java index f2135427bf5..6730db2e8d6 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/request/DayOfWeekFunction.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/request/DayOfWeekFunction.java @@ -8,6 +8,7 @@ import java.util.Arrays; * equals the day of week (0 - 6) of the result of the argument, Monday being 0. * * @author Simon Thoresen Hult + * @author bratseth */ public class DayOfWeekFunction extends FunctionNode { @@ -17,6 +18,18 @@ public class DayOfWeekFunction extends FunctionNode { * @param exp The expression to evaluate, must evaluate to a long. */ public DayOfWeekFunction(GroupingExpression exp) { - super("time.dayofweek", Arrays.asList(exp)); + this(null, null, exp); } + + private DayOfWeekFunction(String label, Integer level, GroupingExpression exp) { + super("time.dayofweek", label, level, Arrays.asList(exp)); + } + + @Override + public DayOfWeekFunction copy() { + return new DayOfWeekFunction(getLabel(), + getLevelOrNull(), + getArg(0).copy()); + } + } diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/DayOfYearFunction.java b/container-search/src/main/java/com/yahoo/search/grouping/request/DayOfYearFunction.java index 15f158ef2f1..9ff59c26fe5 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/request/DayOfYearFunction.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/request/DayOfYearFunction.java @@ -8,6 +8,7 @@ import java.util.Arrays; * equals the day of year (0-365) of the result of the argument. * * @author Simon Thoresen Hult + * @author bratseth */ public class DayOfYearFunction extends FunctionNode { @@ -17,6 +18,18 @@ public class DayOfYearFunction extends FunctionNode { * @param exp The expression to evaluate, must evaluate to a long. */ public DayOfYearFunction(GroupingExpression exp) { - super("time.dayofyear", Arrays.asList(exp)); + this(null, null, exp); } + + private DayOfYearFunction(String label, Integer level, GroupingExpression exp) { + super("time.dayofyear", label, level, Arrays.asList(exp)); + } + + @Override + public DayOfYearFunction copy() { + return new DayOfYearFunction(getLabel(), + getLevelOrNull(), + getArg(0).copy()); + } + } diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/DebugWaitFunction.java b/container-search/src/main/java/com/yahoo/search/grouping/request/DebugWaitFunction.java index be0bb66f91f..5cad8ea55ec 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/request/DebugWaitFunction.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/request/DebugWaitFunction.java @@ -2,13 +2,14 @@ package com.yahoo.search.grouping.request; import java.util.Arrays; +import java.util.stream.Collectors; /** * This class represents debug_wait function in a {@link GroupingExpression}. For each hit evaluated, * it waits for the time specified as the second argument. The third argument specifies if the wait * should be a busy-wait or not. The first argument is then evaluated. * - * @author <a href="mailto:lulf@yahoo-inc.com">Ulf Lilleengen</a> + * @author Ulf Lilleengen */ public class DebugWaitFunction extends FunctionNode { @@ -20,7 +21,20 @@ public class DebugWaitFunction extends FunctionNode { * @param arg3 The third compulsory argument, specifying busy wait or not. */ public DebugWaitFunction(GroupingExpression arg1, DoubleValue arg2, BooleanValue arg3) { - super("debugwait", Arrays.asList(arg1, arg2, arg3)); + this(null, null, arg1, arg2, arg3); + } + + private DebugWaitFunction(String label, Integer level, GroupingExpression arg1, DoubleValue arg2, BooleanValue arg3) { + super("debugwait", label, level, Arrays.asList(arg1, arg2, arg3)); + } + + @Override + public DebugWaitFunction copy() { + return new DebugWaitFunction(getLabel(), + getLevelOrNull(), + getArg(0).copy(), + (DoubleValue)getArg(1).copy(), + (BooleanValue)getArg(2).copy()); } /** diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/DivFunction.java b/container-search/src/main/java/com/yahoo/search/grouping/request/DivFunction.java index f50dcde2d56..f559d911ff4 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/request/DivFunction.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/request/DivFunction.java @@ -2,6 +2,7 @@ package com.yahoo.search.grouping.request; import java.util.List; +import java.util.stream.Collectors; /** * This class represents a div-function in a {@link GroupingExpression}. It evaluates to a number that equals the result @@ -9,6 +10,7 @@ import java.util.List; * second, result by third, ...). * * @author Simon Thoresen Hult + * @author bratseth */ public class DivFunction extends FunctionNode { @@ -20,11 +22,18 @@ public class DivFunction extends FunctionNode { * @param argN The optional arguments, must evaluate to a number. */ public DivFunction(GroupingExpression arg1, GroupingExpression arg2, GroupingExpression... argN) { - this(asList(arg1, arg2, argN)); + this(null, null, asList(arg1, arg2, argN)); } - private DivFunction(List<GroupingExpression> args) { - super("div", args); + private DivFunction(String label, Integer level, List<GroupingExpression> args) { + super("div", label, level, args); + } + + @Override + public DivFunction copy() { + return new DivFunction(getLabel(), + getLevelOrNull(), + args().stream().map(arg -> arg.copy()).collect(Collectors.toList())); } /** @@ -38,6 +47,6 @@ public class DivFunction extends FunctionNode { if (args.size() < 2) { throw new IllegalArgumentException("Expected 2 or more arguments, got " + args.size() + "."); } - return new DivFunction(args); + return new DivFunction(null, null, args); } } diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/DocIdNsSpecificValue.java b/container-search/src/main/java/com/yahoo/search/grouping/request/DocIdNsSpecificValue.java index 188e642e7d7..d9a9daa2db5 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/request/DocIdNsSpecificValue.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/request/DocIdNsSpecificValue.java @@ -6,6 +6,7 @@ package com.yahoo.search.grouping.request; * specific value of the document id of the input {@link com.yahoo.search.result.Hit}. * * @author Simon Thoresen Hult + * @author bratseth */ public class DocIdNsSpecificValue extends DocumentValue { @@ -13,7 +14,16 @@ public class DocIdNsSpecificValue extends DocumentValue { * Constructs a new instance of this class. */ public DocIdNsSpecificValue() { - super("docidnsspecific()"); + this(null, null); + } + + private DocIdNsSpecificValue(String label, Integer level) { + super("docidnsspecific()", label, level); + } + + @Override + public DocIdNsSpecificValue copy() { + return new DocIdNsSpecificValue(getLabel(), getLevelOrNull()); } } diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/DocumentValue.java b/container-search/src/main/java/com/yahoo/search/grouping/request/DocumentValue.java index f8756b81163..f8b18e4acd6 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/request/DocumentValue.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/request/DocumentValue.java @@ -9,8 +9,8 @@ package com.yahoo.search.grouping.request; */ public abstract class DocumentValue extends GroupingExpression { - protected DocumentValue(String image) { - super(image); + protected DocumentValue(String image, String label, Integer level) { + super(image, label, level); } @Override diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/DoubleBucket.java b/container-search/src/main/java/com/yahoo/search/grouping/request/DoubleBucket.java index d71ec8093fd..f0bcfead39a 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/request/DoubleBucket.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/request/DoubleBucket.java @@ -26,7 +26,7 @@ public class DoubleBucket extends BucketValue { * @param to The to-value to assign to this. */ public DoubleBucket(double from, double to) { - super(new DoubleValue(from), new DoubleValue(to)); + super(null, null, new DoubleValue(from), new DoubleValue(to)); } /** @@ -36,6 +36,16 @@ public class DoubleBucket extends BucketValue { * @param to The to-value to assign to this. */ public DoubleBucket(ConstantValue<?> from, ConstantValue<?> to) { - super(from, to); + super(null, null, from, to); } + + private DoubleBucket(String label, Integer level, ConstantValue<?> from, ConstantValue<?> to) { + super(label, level, from, to); + } + + @Override + public DoubleBucket copy() { + return new DoubleBucket(getLabel(), getLevelOrNull(), getFrom().copy(), getTo().copy()); + } + } diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/DoublePredefined.java b/container-search/src/main/java/com/yahoo/search/grouping/request/DoublePredefined.java index d43e8412623..0409709f2a2 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/request/DoublePredefined.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/request/DoublePredefined.java @@ -2,6 +2,7 @@ package com.yahoo.search.grouping.request; import java.util.List; +import java.util.stream.Collectors; /** * This class represents a predefined bucket-function in a {@link GroupingExpression} for expressions that evaluate to a @@ -19,11 +20,22 @@ public class DoublePredefined extends PredefinedFunction { * @param argN The optional buckets. */ public DoublePredefined(GroupingExpression exp, DoubleBucket arg1, DoubleBucket... argN) { - this(exp, asList(arg1, argN)); + this(null, null, exp, asList(arg1, argN)); } - private DoublePredefined(GroupingExpression exp, List<DoubleBucket> args) { - super(exp, args); + private DoublePredefined(String label, Integer level, GroupingExpression exp, List<DoubleBucket> args) { + super(label, level, exp, args); + } + + @Override + public DoublePredefined copy() { + return new DoublePredefined(getLabel(), + getLevelOrNull(), + getArg(0), + args().stream().skip(1) + .map(DoubleBucket.class::cast) + .map(arg -> arg.copy()) + .collect(Collectors.toList())); } @Override @@ -43,6 +55,6 @@ public class DoublePredefined extends PredefinedFunction { if (args.isEmpty()) { throw new IllegalArgumentException("Expected at least one bucket, got none."); } - return new DoublePredefined(exp, args); + return new DoublePredefined(null, null, exp, args); } } diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/DoubleValue.java b/container-search/src/main/java/com/yahoo/search/grouping/request/DoubleValue.java index 5dad65aaabf..318cc680f2e 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/request/DoubleValue.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/request/DoubleValue.java @@ -5,6 +5,7 @@ package com.yahoo.search.grouping.request; * This class represents a constant {@link Double} value in a {@link GroupingExpression}. * * @author Simon Thoresen Hult + * @author bratseth */ public class DoubleValue extends ConstantValue<Double> { @@ -14,6 +15,16 @@ public class DoubleValue extends ConstantValue<Double> { * @param value The immutable value to assign to this. */ public DoubleValue(double value) { - super(value); + super(null, null, value); } + + private DoubleValue(String label, Integer level, Double value) { + super(label, level, value); + } + + @Override + public DoubleValue copy() { + return new DoubleValue(getLabel(), getLevelOrNull(), getValue()); + } + } diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/EachOperation.java b/container-search/src/main/java/com/yahoo/search/grouping/request/EachOperation.java index 1a7074a2c6e..2d830fd214e 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/request/EachOperation.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/request/EachOperation.java @@ -1,11 +1,16 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.grouping.request; +import java.util.List; +import java.util.Map; +import java.util.Set; + /** * This is a grouping operation that processes each element of the input list separately, as opposed to {@link * AllOperation} which processes that list as a whole. * * @author Simon Thoresen Hult + * @author bratseth */ public class EachOperation extends GroupingOperation { @@ -13,7 +18,44 @@ public class EachOperation extends GroupingOperation { * Constructs a new instance of this class. */ public EachOperation() { - super("each"); + super("each", null); + } + + private EachOperation(GroupingOperation parentOfCopy, + String image, + String label, + List<GroupingExpression> orderBy, + List<GroupingExpression> outputs, + List<GroupingOperation> children, + Map<String, GroupingExpression> aliases, + Set<String> hints, + GroupingExpression groupBy, + String where, + boolean forceSinglePass, + double accuracy, + int precision, + int level, + int max) { + super(parentOfCopy, image, label, orderBy, outputs, children, aliases, hints, groupBy, where, forceSinglePass, accuracy, precision, level, max); + } + + @Override + public EachOperation copy(GroupingOperation parentOfCopy) { + return new EachOperation(parentOfCopy, + getImage(), + getLabel(), + getOrderBy(), + getOutputs(), + getChildren(), + getAliases(), + getHints(), + getGroupBy(), + getWhere(), + getForceSinglePass(), + getAccuracy(), + getPrecision(), + getLevel(), + getMax()); } @Override diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/ExpressionVisitor.java b/container-search/src/main/java/com/yahoo/search/grouping/request/ExpressionVisitor.java index a341bee2568..89d698694a8 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/request/ExpressionVisitor.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/request/ExpressionVisitor.java @@ -13,7 +13,8 @@ public interface ExpressionVisitor { /** * This method is called for every {@link GroupingExpression} object in the targeted {@link GroupingOperation}. * - * @param exp The expression being visited. + * @param exp the expression being visited. */ - public void visitExpression(GroupingExpression exp); + void visitExpression(GroupingExpression exp); + } diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/FixedWidthFunction.java b/container-search/src/main/java/com/yahoo/search/grouping/request/FixedWidthFunction.java index f44394c6193..3b397634a11 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/request/FixedWidthFunction.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/request/FixedWidthFunction.java @@ -18,7 +18,17 @@ public class FixedWidthFunction extends FunctionNode { * @param width The width of each bucket. */ public FixedWidthFunction(GroupingExpression exp, Number width) { - super("fixedwidth", Arrays.asList(exp, width instanceof Double ? new DoubleValue(width.doubleValue()) : new LongValue(width.longValue()))); + this(null, null, exp, + width instanceof Double ? new DoubleValue(width.doubleValue()) : new LongValue(width.longValue())); + } + + private FixedWidthFunction(String label, Integer level, GroupingExpression exp, ConstantValue width) { + super("fixedwidth", label, level, Arrays.asList(exp, width)); + } + + @Override + public FixedWidthFunction copy() { + return new FixedWidthFunction(getLabel(), getLevelOrNull(), getArg(0).copy(), (ConstantValue)getArg(1).copy()); } /** diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/FunctionNode.java b/container-search/src/main/java/com/yahoo/search/grouping/request/FunctionNode.java index 1a059d79b1a..9e09baf66ab 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/request/FunctionNode.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/request/FunctionNode.java @@ -14,8 +14,8 @@ public abstract class FunctionNode extends GroupingExpression implements Iterabl private final List<GroupingExpression> args = new ArrayList<>(); - protected FunctionNode(String image, List<GroupingExpression> args) { - super(image + "(" + asString(args) + ")"); + protected FunctionNode(String image, String label, Integer level, List<GroupingExpression> args) { + super(image + "(" + asString(args) + ")", label, level); this.args.addAll(args); } @@ -39,6 +39,10 @@ public abstract class FunctionNode extends GroupingExpression implements Iterabl return args.get(i); } + /** Returns the arguments of this as a list which cannot be modified */ + // Note: If this is made public the returned list must be immutable + protected List<GroupingExpression> args() { return args; } + @Override public Iterator<GroupingExpression> iterator() { return Collections.unmodifiableList(args).iterator(); @@ -75,4 +79,5 @@ public abstract class FunctionNode extends GroupingExpression implements Iterabl ret.addAll(bar); return ret; } + } diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/GroupingExpression.java b/container-search/src/main/java/com/yahoo/search/grouping/request/GroupingExpression.java index 2f7691156e5..ad97f5519ee 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/request/GroupingExpression.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/request/GroupingExpression.java @@ -15,10 +15,14 @@ public abstract class GroupingExpression extends GroupingNode { private Integer level = null; - protected GroupingExpression(String image) { - super(image); + protected GroupingExpression(String image, String label, Integer level) { + super(image, label); + this.level = level; } + /** Returns a deep copy of this */ + public abstract GroupingExpression copy(); + /** * Resolves the conceptual level of this expression. This level represents the type of data that is consumed by this * expression, where level 0 is a single hit, level 1 is a group, level 2 is a list of groups, and so forth. This @@ -39,8 +43,8 @@ public abstract class GroupingExpression extends GroupingNode { /** * Returns the conceptual level of this expression. * - * @return The level. - * @throws IllegalArgumentException Thrown if the level of this expression has not been resolved. + * @return the level. + * @throws IllegalArgumentException thrown if the level of this expression has not been resolved. * @see #resolveLevel(int) */ public int getLevel() { @@ -50,6 +54,8 @@ public abstract class GroupingExpression extends GroupingNode { return level; } + protected final Integer getLevelOrNull() { return level; } + /** * Recursively calls {@link ExpressionVisitor#visitExpression(GroupingExpression)} for this expression and all of * its argument expressions. diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/GroupingNode.java b/container-search/src/main/java/com/yahoo/search/grouping/request/GroupingNode.java index 148b4243a4b..bd5db3d1fdb 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/request/GroupingNode.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/request/GroupingNode.java @@ -10,10 +10,11 @@ package com.yahoo.search.grouping.request; public abstract class GroupingNode { private final String image; - private String label = null; + private String label; - protected GroupingNode(String image) { + protected GroupingNode(String image, String label) { this.image = image; + this.label = label; } /** @@ -25,6 +26,8 @@ public abstract class GroupingNode { return label; } + protected String getImage() { return image; } + /** * Assigns a label to this grouping expression. The label is applied to the results of this expression so that they * can be identified by the caller when processing the output. @@ -41,4 +44,5 @@ public abstract class GroupingNode { public String toString() { return image; } + } diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/GroupingOperation.java b/container-search/src/main/java/com/yahoo/search/grouping/request/GroupingOperation.java index 4c29ca7d98d..93619913b4f 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/request/GroupingOperation.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/request/GroupingOperation.java @@ -23,7 +23,7 @@ public abstract class GroupingOperation extends GroupingNode { private final List<GroupingExpression> orderBy = new ArrayList<>(); private final List<GroupingExpression> outputs = new ArrayList<>(); private final List<GroupingOperation> children = new ArrayList<>(); - private final Map<String, GroupingExpression> alias = LazyMap.newHashMap(); + private final Map<String, GroupingExpression> aliases = LazyMap.newHashMap(); private final Set<String> hints = LazySet.newHashSet(); private GroupingExpression groupBy = null; @@ -35,10 +35,44 @@ public abstract class GroupingOperation extends GroupingNode { private int level = -1; private int max = -1; - protected GroupingOperation(String image) { - super(image); + protected GroupingOperation(String image, String label) { + super(image, label); + } + + protected GroupingOperation(GroupingOperation parentOfCopy, + String image, + String label, + List<GroupingExpression> orderBy, + List<GroupingExpression> outputs, + List<GroupingOperation> children, + Map<String, GroupingExpression> aliases, + Set<String> hints, + GroupingExpression groupBy, + String where, + boolean forceSinglePass, + double accuracy, + int precision, + int level, + int max) { + super(image, label); + this.parent = parentOfCopy; + orderBy.forEach(item -> this.orderBy.add(item.copy())); + outputs.forEach(item -> this.outputs.add(item.copy())); + children.forEach(item -> this.children.add(item.copy(this))); + aliases.forEach((key, value) -> this.aliases.put(key, value.copy())); + this.hints.addAll(hints); + if (groupBy != null) this.groupBy = groupBy.copy(); + this.where = where; + this.forceSinglePass = forceSinglePass; + this.accuracy = accuracy; + this.precision = precision; + this.level = level; + this.max = max; } + /** Returns a deep copy of this */ + public abstract GroupingOperation copy(GroupingOperation parentOfCopy); + /** * Registers an alias with this operation. An alias is made available to expressions in both this node and all child * nodes. @@ -48,7 +82,7 @@ public abstract class GroupingOperation extends GroupingNode { * @return This, to allow chaining. */ public GroupingOperation putAlias(String id, GroupingExpression exp) { - alias.put(id, exp); + aliases.put(id, exp); return this; } @@ -60,8 +94,8 @@ public abstract class GroupingOperation extends GroupingNode { * @return The expression associated with the id. */ public GroupingExpression getAlias(String id) { - if (alias.containsKey(id)) { - return alias.get(id); + if (aliases.containsKey(id)) { + return aliases.get(id); } else if (parent != null) { return parent.getAlias(id); } else { @@ -69,6 +103,11 @@ public abstract class GroupingOperation extends GroupingNode { } } + /** Returns a direct, mutable copy of the aliases of this, never null */ + protected Map<String, GroupingExpression> getAliases() { + return aliases; + } + /** * Adds a hint to this. * @@ -225,6 +264,9 @@ public abstract class GroupingOperation extends GroupingNode { } } + /** Returns the parent of this, or null if none */ + protected GroupingOperation getParent() { return parent; } + public GroupingOperation setForceSinglePass(boolean forceSinglePass) { this.forceSinglePass = forceSinglePass; return this; @@ -446,7 +488,7 @@ public abstract class GroupingOperation extends GroupingNode { * @param visitor The visitor to call. */ public void visitExpressions(ExpressionVisitor visitor) { - for (GroupingExpression exp : alias.values()) { + for (GroupingExpression exp : aliases.values()) { exp.visit(visitor); } for (GroupingExpression exp : outputs) { diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/HourOfDayFunction.java b/container-search/src/main/java/com/yahoo/search/grouping/request/HourOfDayFunction.java index faf1c1e0289..7adebcbfbdb 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/request/HourOfDayFunction.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/request/HourOfDayFunction.java @@ -8,6 +8,7 @@ import java.util.Arrays; * equals the hour of day (0-23) of the result of the argument. * * @author Simon Thoresen Hult + * @author bratseth */ public class HourOfDayFunction extends FunctionNode { @@ -17,6 +18,16 @@ public class HourOfDayFunction extends FunctionNode { * @param exp The expression to evaluate, must evaluate to a long. */ public HourOfDayFunction(GroupingExpression exp) { - super("time.hourofday", Arrays.asList(exp)); + this(null, null, exp); } + + private HourOfDayFunction(String label, Integer level, GroupingExpression exp) { + super("time.hourofday", label, level, Arrays.asList(exp)); + } + + @Override + public HourOfDayFunction copy() { + return new HourOfDayFunction(getLabel(), getLevelOrNull(), getArg(0).copy()); + } + } diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/Infinite.java b/container-search/src/main/java/com/yahoo/search/grouping/request/Infinite.java index 53f5d3256ed..766571914cc 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/request/Infinite.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/request/Infinite.java @@ -3,12 +3,13 @@ package com.yahoo.search.grouping.request; /** * This class represents an Infinite value that may be used as a bucket - * size specifier. + * size specifier. This is immutable. * - * @author <a href="mailto:lulf@yahoo-inc.com">Ulf Lilleengen</a> + * @author Ulf Lilleengen */ @SuppressWarnings("rawtypes") public class Infinite implements Comparable { + private final boolean negative; /** diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/InfiniteValue.java b/container-search/src/main/java/com/yahoo/search/grouping/request/InfiniteValue.java index 7c2730992df..dac6259595d 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/request/InfiniteValue.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/request/InfiniteValue.java @@ -4,7 +4,8 @@ package com.yahoo.search.grouping.request; /** * This class represents an infinite value in a {@link GroupingExpression}. * - * @author <a href="mailto:lulf@yahoo-inc.com">Ulf Lilleengen</a> + * @author Ulf Lilleengen + * @author bratseth */ public class InfiniteValue extends ConstantValue<Infinite> { @@ -14,6 +15,16 @@ public class InfiniteValue extends ConstantValue<Infinite> { * @param value The immutable value to assign to this. */ public InfiniteValue(Infinite value) { - super(value); + super(null, null, value); } + + private InfiniteValue(String label, Integer level, Infinite value) { + super(label, level, value); + } + + @Override + public InfiniteValue copy() { + return new InfiniteValue(getLabel(), getLevelOrNull(), getValue()); + } + } diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/InterpolatedLookup.java b/container-search/src/main/java/com/yahoo/search/grouping/request/InterpolatedLookup.java index dc8b6d68dc3..279434f1df2 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/request/InterpolatedLookup.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/request/InterpolatedLookup.java @@ -18,34 +18,37 @@ import com.google.common.annotations.Beta; public class InterpolatedLookup extends DocumentValue { private final String attributeName; - private final GroupingExpression arg2; + private final GroupingExpression lookupArgument; /** * Constructs a new instance of this class. * - * @param attributeName The attribute name the lookup should happen in - * @param lookupArg Expression giving a floating-point value for the lookup argument + * @param attributeName the attribute name to assign to this. + * @param lookupArgument Expression giving a floating-point value for the lookup argument */ - public InterpolatedLookup(String attributeName, GroupingExpression lookupArg) { - super("interpolatedlookup(" + attributeName + ", " + lookupArg + ")"); + public InterpolatedLookup(String attributeName, GroupingExpression lookupArgument) { + this(null, null, attributeName, lookupArgument); + } + + private InterpolatedLookup(String label, Integer level, String attributeName, GroupingExpression lookupArgument) { + super("interpolatedlookup(" + attributeName + ", " + lookupArgument + ")", label, level); this.attributeName = attributeName; - this.arg2 = lookupArg; + this.lookupArgument = lookupArgument; } - /** - * Get the name of the attribute to be retrieved from the input hit. - * @return The attribute name. - */ + @Override + public InterpolatedLookup copy() { + return new InterpolatedLookup(getLabel(), getLevelOrNull(), getAttributeName(), getLookupArgument().copy()); + } + + /** Returns the name of the attribute to retrieve from the input hit */ public String getAttributeName() { return attributeName; } - /** - * Get the expression that will be evaluated before lookup. - * @return grouping expression argument - */ + /** Return the expression to evaluate before lookup */ public GroupingExpression getLookupArgument() { - return arg2; + return lookupArgument; } } diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/LongBucket.java b/container-search/src/main/java/com/yahoo/search/grouping/request/LongBucket.java index fd363320033..a0fdf8f9526 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/request/LongBucket.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/request/LongBucket.java @@ -12,7 +12,7 @@ public class LongBucket extends BucketValue { * Gives the next distinct long value. * * @param value the base value. - * @return the nextt value. + * @return the next value. */ public static LongValue nextValue(LongValue value) { long v = value.getValue(); @@ -26,7 +26,7 @@ public class LongBucket extends BucketValue { * @param to The to-value to assign to this. */ public LongBucket(long from, long to) { - super(new LongValue(from), new LongValue(to)); + super(null, null, new LongValue(from), new LongValue(to)); } /** @@ -37,6 +37,16 @@ public class LongBucket extends BucketValue { */ @SuppressWarnings("rawtypes") public LongBucket(ConstantValue from, ConstantValue to) { - super(from, to); + super(null, null, from, to); } + + private LongBucket(String label, Integer level, ConstantValue<?> from, ConstantValue<?> to) { + super(label, level, from, to); + } + + @Override + public LongBucket copy() { + return new LongBucket(getLabel(), getLevelOrNull(), getFrom().copy(), getTo().copy()); + } + } diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/LongPredefined.java b/container-search/src/main/java/com/yahoo/search/grouping/request/LongPredefined.java index 98acc800ed7..fe428d26db7 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/request/LongPredefined.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/request/LongPredefined.java @@ -2,12 +2,14 @@ package com.yahoo.search.grouping.request; import java.util.List; +import java.util.stream.Collectors; /** * This class represents a predefined bucket-function in a {@link GroupingExpression} for expressions that evaluate to a * long. * * @author Simon Thoresen Hult + * @author bratseth */ public class LongPredefined extends PredefinedFunction { @@ -19,11 +21,23 @@ public class LongPredefined extends PredefinedFunction { * @param argN The optional buckets. */ public LongPredefined(GroupingExpression exp, LongBucket arg1, LongBucket... argN) { - this(exp, asList(arg1, argN)); + this(null, null, exp, asList(arg1, argN)); } - private LongPredefined(GroupingExpression exp, List<LongBucket> args) { - super(exp, args); + private LongPredefined(String label, Integer level, GroupingExpression exp, List<LongBucket> args) { + super(label, level, exp, args); + } + + @Override + public LongPredefined copy() { + return new LongPredefined(getLabel(), + getLevelOrNull(), + getArg(0), + args().stream() + .skip(1) + .map(LongBucket.class::cast) + .map(arg -> arg.copy()) + .collect(Collectors.toList())); } @Override @@ -43,6 +57,6 @@ public class LongPredefined extends PredefinedFunction { if (args.isEmpty()) { throw new IllegalArgumentException("Expected at least one bucket, got none."); } - return new LongPredefined(exp, args); + return new LongPredefined(null, null, exp, args); } } diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/LongValue.java b/container-search/src/main/java/com/yahoo/search/grouping/request/LongValue.java index 4c4eb1c409d..705036af78e 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/request/LongValue.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/request/LongValue.java @@ -5,6 +5,7 @@ package com.yahoo.search.grouping.request; * This class represents a constant {@link Long} value in a {@link GroupingExpression}. * * @author Simon Thoresen Hult + * @author bratseth */ public class LongValue extends ConstantValue<Long> { @@ -14,7 +15,16 @@ public class LongValue extends ConstantValue<Long> { * @param value the immutable value to assign to this. */ public LongValue(long value) { - super(value); + super(null, null, value); + } + + private LongValue(String label, Integer level, Long value) { + super(label, level, value); + } + + @Override + public LongValue copy() { + return new LongValue(getLabel(), getLevelOrNull(), getValue()); } } diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/MathACosFunction.java b/container-search/src/main/java/com/yahoo/search/grouping/request/MathACosFunction.java index 58d7275e8eb..24ba449e517 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/request/MathACosFunction.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/request/MathACosFunction.java @@ -5,14 +5,26 @@ import java.util.Arrays; /** * @author baldersheim + * @author bratseth */ public class MathACosFunction extends FunctionNode { + /** * Constructs a new instance of this class. * * @param exp The expression to evaluate, double value will be requested. */ public MathACosFunction(GroupingExpression exp) { - super("math.acos", Arrays.asList(exp)); + this(null, null, exp); + } + + private MathACosFunction(String label, Integer level, GroupingExpression exp) { + super("math.acos", label, level, Arrays.asList(exp)); } + + @Override + public MathACosFunction copy() { + return new MathACosFunction(getLabel(), getLevelOrNull(), getArg(0).copy()); + } + } diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/MathACosHFunction.java b/container-search/src/main/java/com/yahoo/search/grouping/request/MathACosHFunction.java index eca8fd7ebf7..40de58ba0be 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/request/MathACosHFunction.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/request/MathACosHFunction.java @@ -5,14 +5,26 @@ import java.util.Arrays; /** * @author baldersheim + * @author bratseth */ public class MathACosHFunction extends FunctionNode { -/** + + /** * Constructs a new instance of this class. * * @param exp The expression to evaluate, double value will be requested. */ public MathACosHFunction(GroupingExpression exp) { - super("math.acosh", Arrays.asList(exp)); + this(null, null, exp); + } + + private MathACosHFunction(String label, Integer level, GroupingExpression exp) { + super("math.acosh", label, level, Arrays.asList(exp)); } + + @Override + public MathACosHFunction copy() { + return new MathACosHFunction(getLabel(), getLevelOrNull(), getArg(0).copy()); + } + } diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/MathASinFunction.java b/container-search/src/main/java/com/yahoo/search/grouping/request/MathASinFunction.java index 1189bccd5bc..48a2641b660 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/request/MathASinFunction.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/request/MathASinFunction.java @@ -5,14 +5,26 @@ import java.util.Arrays; /** * @author baldersheim + * @author bratseth */ public class MathASinFunction extends FunctionNode { + /** * Constructs a new instance of this class. * * @param exp The expression to evaluate, double value will be requested. */ public MathASinFunction(GroupingExpression exp) { - super("math.asin", Arrays.asList(exp)); + this(null, null, exp); + } + + private MathASinFunction(String label, Integer level, GroupingExpression exp) { + super("math.asin", label, level, Arrays.asList(exp)); } + + @Override + public MathASinFunction copy() { + return new MathASinFunction(getLabel(), getLevelOrNull(), getArg(0).copy()); + } + } diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/MathASinHFunction.java b/container-search/src/main/java/com/yahoo/search/grouping/request/MathASinHFunction.java index 7619ab4f1b3..687bf1c6c34 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/request/MathASinHFunction.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/request/MathASinHFunction.java @@ -5,14 +5,26 @@ import java.util.Arrays; /** * @author baldersheim + * @author bratseth */ public class MathASinHFunction extends FunctionNode { -/** + + /** * Constructs a new instance of this class. * * @param exp The expression to evaluate, double value will be requested. */ public MathASinHFunction(GroupingExpression exp) { - super("math.asinh", Arrays.asList(exp)); + this(null, null, exp); + } + + private MathASinHFunction(String label, Integer level, GroupingExpression exp) { + super("math.asinh", label, level, Arrays.asList(exp)); } + + @Override + public MathASinHFunction copy() { + return new MathASinHFunction(getLabel(), getLevelOrNull(), getArg(0).copy()); + } + } diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/MathATanFunction.java b/container-search/src/main/java/com/yahoo/search/grouping/request/MathATanFunction.java index 114fd24644b..6612c9cc70f 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/request/MathATanFunction.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/request/MathATanFunction.java @@ -5,14 +5,26 @@ import java.util.Arrays; /** * @author baldersheim + * @author bratseth */ public class MathATanFunction extends FunctionNode { + /** * Constructs a new instance of this class. * * @param exp The expression to evaluate, double value will be requested. */ public MathATanFunction(GroupingExpression exp) { - super("math.atan", Arrays.asList(exp)); + this(null, null, exp); + } + + private MathATanFunction(String label, Integer level, GroupingExpression exp) { + super("math.atan", label, level, Arrays.asList(exp)); } + + @Override + public MathATanFunction copy() { + return new MathATanFunction(getLabel(), getLevelOrNull(), getArg(0).copy()); + } + } diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/MathATanHFunction.java b/container-search/src/main/java/com/yahoo/search/grouping/request/MathATanHFunction.java index 08f7956e855..fa9844ec17b 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/request/MathATanHFunction.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/request/MathATanHFunction.java @@ -5,14 +5,26 @@ import java.util.Arrays; /** * @author baldersheim + * @author bratseth */ public class MathATanHFunction extends FunctionNode { -/** + + /** * Constructs a new instance of this class. * * @param exp The expression to evaluate, double value will be requested. */ public MathATanHFunction(GroupingExpression exp) { - super("math.atanh", Arrays.asList(exp)); + this(null, null, exp); + } + + private MathATanHFunction(String label, Integer level, GroupingExpression exp) { + super("math.atanh", label, level, Arrays.asList(exp)); } + + @Override + public MathATanHFunction copy() { + return new MathATanHFunction(getLabel(), getLevelOrNull(), getArg(0).copy()); + } + } diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/MathCbrtFunction.java b/container-search/src/main/java/com/yahoo/search/grouping/request/MathCbrtFunction.java index 6978e89ba71..04595003a03 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/request/MathCbrtFunction.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/request/MathCbrtFunction.java @@ -5,14 +5,26 @@ import java.util.Arrays; /** * @author baldersheim + * @author bratseth */ public class MathCbrtFunction extends FunctionNode { -/** + + /** * Constructs a new instance of this class. * * @param exp The expression to evaluate, double value will be requested. */ public MathCbrtFunction(GroupingExpression exp) { - super("math.cbrt", Arrays.asList(exp)); + this(null, null, exp); + } + + private MathCbrtFunction(String label, Integer level, GroupingExpression exp) { + super("math.cbrt", label, level, Arrays.asList(exp)); } + + @Override + public MathCbrtFunction copy() { + return new MathCbrtFunction(getLabel(), getLevelOrNull(), getArg(0).copy()); + } + } diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/MathCosFunction.java b/container-search/src/main/java/com/yahoo/search/grouping/request/MathCosFunction.java index 846660c2370..95140ce06d0 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/request/MathCosFunction.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/request/MathCosFunction.java @@ -5,14 +5,26 @@ import java.util.Arrays; /** * @author baldersheim + * @author bratseth */ public class MathCosFunction extends FunctionNode { + /** * Constructs a new instance of this class. * * @param exp The expression to evaluate, double value will be requested. */ public MathCosFunction(GroupingExpression exp) { - super("math.cos", Arrays.asList(exp)); + this(null, null, exp); + } + + private MathCosFunction(String label, Integer level, GroupingExpression exp) { + super("math.cos", label, level, Arrays.asList(exp)); } + + @Override + public MathCosFunction copy() { + return new MathCosFunction(getLabel(), getLevelOrNull(), getArg(0).copy()); + } + } diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/MathCosHFunction.java b/container-search/src/main/java/com/yahoo/search/grouping/request/MathCosHFunction.java index d370678d3f5..f0384666ffe 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/request/MathCosHFunction.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/request/MathCosHFunction.java @@ -5,14 +5,26 @@ import java.util.Arrays; /** * @author baldersheim + * @author bratseth */ public class MathCosHFunction extends FunctionNode { -/** + + /** * Constructs a new instance of this class. * * @param exp The expression to evaluate, double value will be requested. */ public MathCosHFunction(GroupingExpression exp) { - super("math.cosh", Arrays.asList(exp)); + this(null, null, exp); + } + + private MathCosHFunction(String label, Integer level, GroupingExpression exp) { + super("math.cosh", label, level, Arrays.asList(exp)); } + + @Override + public MathCosHFunction copy() { + return new MathCosHFunction(getLabel(), getLevelOrNull(), getArg(0).copy()); + } + } diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/MathExpFunction.java b/container-search/src/main/java/com/yahoo/search/grouping/request/MathExpFunction.java index 812fd98faa9..ef2a53d90b0 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/request/MathExpFunction.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/request/MathExpFunction.java @@ -5,14 +5,26 @@ import java.util.Arrays; /** * @author baldersheim + * @author bratseth */ public class MathExpFunction extends FunctionNode { + /** * Constructs a new instance of this class. * * @param exp The expression to evaluate, double value will be requested. */ public MathExpFunction(GroupingExpression exp) { - super("math.exp", Arrays.asList(exp)); + this(null, null, exp); + } + + private MathExpFunction(String label, Integer level, GroupingExpression exp) { + super("math.exp", label, level, Arrays.asList(exp)); } + + @Override + public MathExpFunction copy() { + return new MathExpFunction(getLabel(), getLevelOrNull(), getArg(0).copy()); + } + } diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/MathFloorFunction.java b/container-search/src/main/java/com/yahoo/search/grouping/request/MathFloorFunction.java index 31e5e99d974..64dd931f9e4 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/request/MathFloorFunction.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/request/MathFloorFunction.java @@ -3,14 +3,30 @@ package com.yahoo.search.grouping.request; import java.util.Arrays; -/** represents the math.floor(expression) function */ +/** + * Represents the math.floor(expression) function + * + * @author baldersheim + * @author bratseth + */ public class MathFloorFunction extends FunctionNode { + /** * Constructs a new instance of this class. * * @param exp The expression to evaluate, double value will be requested. */ public MathFloorFunction(GroupingExpression exp) { - super("math.floor", Arrays.asList(exp)); + this(null, null, exp); + } + + private MathFloorFunction(String label, Integer level, GroupingExpression exp) { + super("math.floor", label, level, Arrays.asList(exp)); } + + @Override + public MathFloorFunction copy() { + return new MathFloorFunction(getLabel(), getLevelOrNull(), getArg(0).copy()); + } + } diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/MathFunctions.java b/container-search/src/main/java/com/yahoo/search/grouping/request/MathFunctions.java index 6e3e52c9563..5fc806a90fe 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/request/MathFunctions.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/request/MathFunctions.java @@ -5,6 +5,7 @@ package com.yahoo.search.grouping.request; * @author baldersheim */ public abstract class MathFunctions { + /** * Defines the different types of math functions that are available. */ diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/MathHypotFunction.java b/container-search/src/main/java/com/yahoo/search/grouping/request/MathHypotFunction.java index 596c7501af6..d6062124619 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/request/MathHypotFunction.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/request/MathHypotFunction.java @@ -5,8 +5,10 @@ import java.util.Arrays; /** * @author baldersheim + * @author bratseth */ public class MathHypotFunction extends FunctionNode { + /** * Constructs a new instance of this class. * @@ -14,6 +16,16 @@ public class MathHypotFunction extends FunctionNode { * @param y The expression to evaluate for y exponent, double value will be requested. */ public MathHypotFunction(GroupingExpression x, GroupingExpression y) { - super("math.hypot", Arrays.asList(x, y)); + this(null, null, x, y); + } + + private MathHypotFunction(String label, Integer level, GroupingExpression x, GroupingExpression y) { + super("math.hypot", label, level, Arrays.asList(x, y)); } + + @Override + public MathHypotFunction copy() { + return new MathHypotFunction(getLabel(), getLevelOrNull(), getArg(0).copy(), getArg(1).copy()); + } + } diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/MathLog10Function.java b/container-search/src/main/java/com/yahoo/search/grouping/request/MathLog10Function.java index 6a19de43f1c..b64e5d03a97 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/request/MathLog10Function.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/request/MathLog10Function.java @@ -5,14 +5,26 @@ import java.util.Arrays; /** * @author baldersheim + * @author bratseth */ public class MathLog10Function extends FunctionNode { + /** * Constructs a new instance of this class. * * @param exp The expression to evaluate, double value will be requested. */ public MathLog10Function(GroupingExpression exp) { - super("math.log10", Arrays.asList(exp)); + this(null, null, exp); + } + + private MathLog10Function(String label, Integer level, GroupingExpression exp) { + super("math.log10", label, level, Arrays.asList(exp)); } + + @Override + public MathLog10Function copy() { + return new MathLog10Function(getLabel(), getLevelOrNull(), getArg(0).copy()); + } + } diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/MathLog1pFunction.java b/container-search/src/main/java/com/yahoo/search/grouping/request/MathLog1pFunction.java index 06fc2424f41..7df97e6b0d5 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/request/MathLog1pFunction.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/request/MathLog1pFunction.java @@ -5,14 +5,26 @@ import java.util.Arrays; /** * @author baldersheim + * @author bratseth */ public class MathLog1pFunction extends FunctionNode { + /** * Constructs a new instance of this class. * * @param exp The expression to evaluate, double value will be requested. */ public MathLog1pFunction(GroupingExpression exp) { - super("math.log1p", Arrays.asList(exp)); + this(null, null, exp); + } + + private MathLog1pFunction(String label, Integer level, GroupingExpression exp) { + super("math.log1p", label, level, Arrays.asList(exp)); } + + @Override + public MathLog1pFunction copy() { + return new MathLog1pFunction(getLabel(), getLevelOrNull(), getArg(0).copy()); + } + } diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/MathLogFunction.java b/container-search/src/main/java/com/yahoo/search/grouping/request/MathLogFunction.java index 0b71ad66bd9..323a03ad152 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/request/MathLogFunction.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/request/MathLogFunction.java @@ -5,14 +5,26 @@ import java.util.Arrays; /** * @author baldersheim + * @author bratseth */ public class MathLogFunction extends FunctionNode { - /** + + /** * Constructs a new instance of this class. * * @param exp The expression to evaluate, double value will be requested. */ public MathLogFunction(GroupingExpression exp) { - super("math.log", Arrays.asList(exp)); + this(null, null, exp); + } + + private MathLogFunction(String label, Integer level, GroupingExpression exp) { + super("math.log", label, level, Arrays.asList(exp)); } + + @Override + public MathLogFunction copy() { + return new MathLogFunction(getLabel(), getLevelOrNull(), getArg(0).copy()); + } + } diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/MathPowFunction.java b/container-search/src/main/java/com/yahoo/search/grouping/request/MathPowFunction.java index ab0d6ba2139..084fdc82646 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/request/MathPowFunction.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/request/MathPowFunction.java @@ -5,8 +5,10 @@ import java.util.Arrays; /** * @author baldersheim + * @author bratseth */ public class MathPowFunction extends FunctionNode { + /** * Constructs a new instance of this class. * @@ -14,6 +16,19 @@ public class MathPowFunction extends FunctionNode { * @param y The expression to evaluate for the exponent, double value will be requested. */ public MathPowFunction(GroupingExpression x, GroupingExpression y) { - super("math.pow", Arrays.asList(x,y)); + this(null, null, x, y); + } + + private MathPowFunction(String label, Integer level, GroupingExpression x, GroupingExpression y) { + super("math.pow", label, level, Arrays.asList(x, y)); } + + @Override + public MathPowFunction copy() { + return new MathPowFunction(getLabel(), + getLevelOrNull(), + getArg(0).copy(), + getArg(1).copy()); + } + } diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/MathSinFunction.java b/container-search/src/main/java/com/yahoo/search/grouping/request/MathSinFunction.java index dd914807e7a..ea9a0d97887 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/request/MathSinFunction.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/request/MathSinFunction.java @@ -5,14 +5,26 @@ import java.util.Arrays; /** * @author baldersheim + * @author bratseth */ public class MathSinFunction extends FunctionNode { + /** * Constructs a new instance of this class. * * @param exp The expression to evaluate, double value will be requested. */ public MathSinFunction(GroupingExpression exp) { - super("math.sin", Arrays.asList(exp)); + this(null, null, exp); + } + + private MathSinFunction(String label, Integer level, GroupingExpression exp) { + super("math.sin", label, level, Arrays.asList(exp)); } + + @Override + public MathSinFunction copy() { + return new MathSinFunction(getLabel(), getLevelOrNull(), getArg(0).copy()); + } + } diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/MathSinHFunction.java b/container-search/src/main/java/com/yahoo/search/grouping/request/MathSinHFunction.java index b600d947489..3a02de24e6a 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/request/MathSinHFunction.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/request/MathSinHFunction.java @@ -5,14 +5,26 @@ import java.util.Arrays; /** * @author baldersheim + * @author bratseth */ public class MathSinHFunction extends FunctionNode { -/** + + /** * Constructs a new instance of this class. * * @param exp The expression to evaluate, double value will be requested. */ public MathSinHFunction(GroupingExpression exp) { - super("math.sinh", Arrays.asList(exp)); + this(null, null, exp); + } + + private MathSinHFunction(String label, Integer level, GroupingExpression exp) { + super("math.sinh", label, level, Arrays.asList(exp)); } + + @Override + public MathSinHFunction copy() { + return new MathSinHFunction(getLabel(), getLevelOrNull(), getArg(0).copy()); + } + } diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/MathSqrtFunction.java b/container-search/src/main/java/com/yahoo/search/grouping/request/MathSqrtFunction.java index e6a0c04e7ee..ee66b71e070 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/request/MathSqrtFunction.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/request/MathSqrtFunction.java @@ -5,14 +5,26 @@ import java.util.Arrays; /** * @author baldersheim + * @author bratseth */ public class MathSqrtFunction extends FunctionNode { + /** * Constructs a new instance of this class. * * @param exp The expression to evaluate, double value will be requested. */ public MathSqrtFunction(GroupingExpression exp) { - super("math.sqrt", Arrays.asList(exp)); + this(null, null, exp); + } + + private MathSqrtFunction(String label, Integer level, GroupingExpression exp) { + super("math.sqrt", label, level, Arrays.asList(exp)); } + + @Override + public MathSqrtFunction copy() { + return new MathSqrtFunction(getLabel(), getLevelOrNull(), getArg(0).copy()); + } + } diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/MathTanFunction.java b/container-search/src/main/java/com/yahoo/search/grouping/request/MathTanFunction.java index f66492f87ad..003ef9c6a00 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/request/MathTanFunction.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/request/MathTanFunction.java @@ -5,14 +5,26 @@ import java.util.Arrays; /** * @author baldersheim + * @author bratseth */ public class MathTanFunction extends FunctionNode { + /** * Constructs a new instance of this class. * * @param exp The expression to evaluate, double value will be requested. */ public MathTanFunction(GroupingExpression exp) { - super("math.tan", Arrays.asList(exp)); + this(null, null, exp); + } + + private MathTanFunction(String label, Integer level, GroupingExpression exp) { + super("math.tan", label, level, Arrays.asList(exp)); } + + @Override + public MathTanFunction copy() { + return new MathTanFunction(getLabel(), getLevelOrNull(), getArg(0).copy()); + } + } diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/MathTanHFunction.java b/container-search/src/main/java/com/yahoo/search/grouping/request/MathTanHFunction.java index 93e7c1c1d49..86ccbbdfcf9 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/request/MathTanHFunction.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/request/MathTanHFunction.java @@ -5,14 +5,26 @@ import java.util.Arrays; /** * @author baldersheim + * @author bratseth */ public class MathTanHFunction extends FunctionNode { -/** + + /** * Constructs a new instance of this class. * * @param exp The expression to evaluate, double value will be requested. */ public MathTanHFunction(GroupingExpression exp) { - super("math.tanh", Arrays.asList(exp)); + this(null, null, exp); + } + + private MathTanHFunction(String label, Integer level, GroupingExpression exp) { + super("math.tanh", label, level, Arrays.asList(exp)); } + + @Override + public MathTanHFunction copy() { + return new MathTanHFunction(getLabel(), getLevelOrNull(), getArg(0).copy()); + } + } diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/MaxAggregator.java b/container-search/src/main/java/com/yahoo/search/grouping/request/MaxAggregator.java index 54dc7c6b3bd..513cba8fb19 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/request/MaxAggregator.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/request/MaxAggregator.java @@ -6,15 +6,26 @@ package com.yahoo.search.grouping.request; * the contained expression evaluated to over all the inputs. * * @author Simon Thoresen Hult + * @author bratseth */ public class MaxAggregator extends AggregatorNode { /** * Constructs a new instance of this class. * - * @param exp The expression to aggregate on. + * @param expression the expression to aggregate on. */ - public MaxAggregator(GroupingExpression exp) { - super("max", exp); + public MaxAggregator(GroupingExpression expression) { + this(null, null, expression); } + + private MaxAggregator(String label, Integer level, GroupingExpression expression) { + super("max", label, level, expression); + } + + @Override + public MaxAggregator copy() { + return new MaxAggregator(getLabel(), getLevelOrNull(), getExpression().copy()); + } + } diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/MaxFunction.java b/container-search/src/main/java/com/yahoo/search/grouping/request/MaxFunction.java index 731c1c4eba4..bff8cb481b8 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/request/MaxFunction.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/request/MaxFunction.java @@ -2,6 +2,7 @@ package com.yahoo.search.grouping.request; import java.util.List; +import java.util.stream.Collectors; /** * This class represents a max-function in a {@link GroupingExpression}. It evaluates to a number that equals the @@ -19,11 +20,18 @@ public class MaxFunction extends FunctionNode { * @param argN The optional arguments, must evaluate to a number. */ public MaxFunction(GroupingExpression arg1, GroupingExpression arg2, GroupingExpression... argN) { - this(asList(arg1, arg2, argN)); + this(null, null, asList(arg1, arg2, argN)); } - private MaxFunction(List<GroupingExpression> args) { - super("max", args); + private MaxFunction(String label, Integer level, List<GroupingExpression> args) { + super("max", label, level, args); + } + + @Override + public MaxFunction copy() { + return new MaxFunction(getLabel(), + getLevelOrNull(), + args().stream().map(arg -> arg.copy()).collect(Collectors.toList())); } /** @@ -37,7 +45,7 @@ public class MaxFunction extends FunctionNode { if (args.size() < 2) { throw new IllegalArgumentException("Expected 2 or more arguments, got " + args.size() + "."); } - return new MaxFunction(args); + return new MaxFunction(null, null, args); } } diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/Md5Function.java b/container-search/src/main/java/com/yahoo/search/grouping/request/Md5Function.java index 6ae33b804e9..1426d141ef7 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/request/Md5Function.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/request/Md5Function.java @@ -18,7 +18,16 @@ public class Md5Function extends FunctionNode { * @param numBits The number of bits of the md5 to include. */ public Md5Function(GroupingExpression exp, int numBits) { - super("md5", Arrays.asList(exp, new LongValue(numBits))); + this(null, null, exp, new LongValue(numBits)); + } + + private Md5Function(String label, Integer level, GroupingExpression exp, LongValue numBits) { + super("md5", label, level, Arrays.asList(exp, numBits)); + } + + @Override + public Md5Function copy() { + return new Md5Function(getLabel(), getLevelOrNull(), getArg(0).copy(), (LongValue)getArg(1).copy()); } /** diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/MinAggregator.java b/container-search/src/main/java/com/yahoo/search/grouping/request/MinAggregator.java index 1a24527dcfc..07164ba4053 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/request/MinAggregator.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/request/MinAggregator.java @@ -6,15 +6,26 @@ package com.yahoo.search.grouping.request; * the contained expression evaluated to over all the inputs. * * @author Simon Thoresen Hult + * @author bratseth */ public class MinAggregator extends AggregatorNode { /** * Constructs a new instance of this class. * - * @param exp The expression to aggregate on. + * @param expression the expression to aggregate on. */ - public MinAggregator(GroupingExpression exp) { - super("min", exp); + public MinAggregator(GroupingExpression expression) { + this(null, null, expression); } + + private MinAggregator(String label, Integer level, GroupingExpression expression) { + super("min", label, level, expression); + } + + @Override + public MinAggregator copy() { + return new MinAggregator(getLabel(), getLevelOrNull(), getExpression().copy()); + } + } diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/MinFunction.java b/container-search/src/main/java/com/yahoo/search/grouping/request/MinFunction.java index 478684dd73e..40fbda5a98a 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/request/MinFunction.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/request/MinFunction.java @@ -1,7 +1,9 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.grouping.request; +import java.util.Arrays; import java.util.List; +import java.util.stream.Collectors; /** * This class represents a min-function in a {@link GroupingExpression}. It evaluates to a number that equals the @@ -19,11 +21,18 @@ public class MinFunction extends FunctionNode { * @param argN The optional arguments, must evaluate to a number. */ public MinFunction(GroupingExpression arg1, GroupingExpression arg2, GroupingExpression... argN) { - this(asList(arg1, arg2, argN)); + this(null, null, asList(arg1, arg2, argN)); } - private MinFunction(List<GroupingExpression> args) { - super("min", args); + private MinFunction(String label, Integer level, List<GroupingExpression> args) { + super("min", label, level, args); + } + + @Override + public MinFunction copy() { + return new MinFunction(getLabel(), + getLevelOrNull(), + args().stream().map(arg -> arg.copy()).collect(Collectors.toList())); } /** @@ -37,7 +46,7 @@ public class MinFunction extends FunctionNode { if (args.size() < 2) { throw new IllegalArgumentException("Expected 2 or more arguments, got " + args.size() + "."); } - return new MinFunction(args); + return new MinFunction(null, null, args); } } diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/MinuteOfHourFunction.java b/container-search/src/main/java/com/yahoo/search/grouping/request/MinuteOfHourFunction.java index d2442b8455e..97dfb5b0d37 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/request/MinuteOfHourFunction.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/request/MinuteOfHourFunction.java @@ -8,6 +8,7 @@ import java.util.Arrays; * that equals the minute of hour (0-59) of the result of the argument. * * @author Simon Thoresen Hult + * @author bratseth */ public class MinuteOfHourFunction extends FunctionNode { @@ -17,6 +18,16 @@ public class MinuteOfHourFunction extends FunctionNode { * @param exp The expression to evaluate, must evaluate to a long. */ public MinuteOfHourFunction(GroupingExpression exp) { - super("time.minuteofhour", Arrays.asList(exp)); + this(null, null, exp); } + + private MinuteOfHourFunction(String label, Integer level, GroupingExpression exp) { + super("time.minuteofhour", label, level, Arrays.asList(exp)); + } + + @Override + public MinuteOfHourFunction copy() { + return new MinuteOfHourFunction(getLabel(), getLevelOrNull(), getArg(0).copy()); + } + } diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/ModFunction.java b/container-search/src/main/java/com/yahoo/search/grouping/request/ModFunction.java index a3fd1041e6e..33b5c7bf655 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/request/ModFunction.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/request/ModFunction.java @@ -2,6 +2,7 @@ package com.yahoo.search.grouping.request; import java.util.List; +import java.util.stream.Collectors; /** * This class represents a mod-function in a {@link GroupingExpression}. It evaluates to a number that equals the result @@ -20,11 +21,18 @@ public class ModFunction extends FunctionNode { * @param argN The optional arguments, must evaluate to a number. */ public ModFunction(GroupingExpression arg1, GroupingExpression arg2, GroupingExpression... argN) { - this(asList(arg1, arg2, argN)); + this(null, null, asList(arg1, arg2, argN)); } - private ModFunction(List<GroupingExpression> args) { - super("mod", args); + private ModFunction(String label, Integer level, List<GroupingExpression> args) { + super("mod", label, level, args); + } + + @Override + public ModFunction copy() { + return new ModFunction(getLabel(), + getLevelOrNull(), + args().stream().map(arg -> arg.copy()).collect(Collectors.toList())); } /** @@ -38,6 +46,6 @@ public class ModFunction extends FunctionNode { if (args.size() < 2) { throw new IllegalArgumentException("Expected 2 or more arguments, got " + args.size() + "."); } - return new ModFunction(args); + return new ModFunction(null, null, args); } } diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/MonthOfYearFunction.java b/container-search/src/main/java/com/yahoo/search/grouping/request/MonthOfYearFunction.java index 96e08562e42..c6df8dbd408 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/request/MonthOfYearFunction.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/request/MonthOfYearFunction.java @@ -8,6 +8,7 @@ import java.util.Arrays; * equals the month of year (1-12) of the result of the argument. * * @author Simon Thoresen Hult + * @author bratseth */ public class MonthOfYearFunction extends FunctionNode { @@ -17,6 +18,16 @@ public class MonthOfYearFunction extends FunctionNode { * @param exp The expression to evaluate, must evaluate to a long. */ public MonthOfYearFunction(GroupingExpression exp) { - super("time.monthofyear", Arrays.asList(exp)); + this(null, null, exp); } + + private MonthOfYearFunction(String label, Integer level, GroupingExpression exp) { + super("time.monthofyear", label, level, Arrays.asList(exp)); + } + + @Override + public MonthOfYearFunction copy() { + return new MonthOfYearFunction(getLabel(), getLevelOrNull(), getArg(0).copy()); + } + } diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/MulFunction.java b/container-search/src/main/java/com/yahoo/search/grouping/request/MulFunction.java index 84584270f58..7af8d22955e 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/request/MulFunction.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/request/MulFunction.java @@ -2,6 +2,7 @@ package com.yahoo.search.grouping.request; import java.util.List; +import java.util.stream.Collectors; /** * This class represents a mul-function in a {@link GroupingExpression}. It evaluates to a number that equals the result @@ -19,11 +20,18 @@ public class MulFunction extends FunctionNode { * @param argN The optional arguments, must evaluate to a number. */ public MulFunction(GroupingExpression arg1, GroupingExpression arg2, GroupingExpression... argN) { - this(asList(arg1, arg2, argN)); + this(null, null, asList(arg1, arg2, argN)); } - private MulFunction(List<GroupingExpression> args) { - super("mul", args); + private MulFunction(String label, Integer level, List<GroupingExpression> args) { + super("mul", label, level, args); + } + + @Override + public MulFunction copy() { + return new MulFunction(getLabel(), + getLevelOrNull(), + args().stream().map(arg -> arg.copy()).collect(Collectors.toList())); } /** @@ -37,6 +45,6 @@ public class MulFunction extends FunctionNode { if (args.size() < 2) { throw new IllegalArgumentException("Expected 2 or more arguments, got " + args.size() + "."); } - return new MulFunction(args); + return new MulFunction(null, null, args); } } diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/NegFunction.java b/container-search/src/main/java/com/yahoo/search/grouping/request/NegFunction.java index 7e05b02b59e..ad3b892970c 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/request/NegFunction.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/request/NegFunction.java @@ -8,6 +8,7 @@ import java.util.Arrays; * negative of the results of the argument. * * @author Simon Thoresen Hult + * @author bratseth */ public class NegFunction extends FunctionNode { @@ -17,7 +18,17 @@ public class NegFunction extends FunctionNode { * @param exp The expression to evaluate, must evaluate to a number. */ public NegFunction(GroupingExpression exp) { - super("neg", Arrays.asList(exp)); + this(null, null, exp); } + + private NegFunction(String label, Integer level, GroupingExpression exp) { + super("neg", label, level, Arrays.asList(exp)); + } + + @Override + public NegFunction copy() { + return new NegFunction(getLabel(), getLevelOrNull(), getArg(0).copy()); + } + } diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/NormalizeSubjectFunction.java b/container-search/src/main/java/com/yahoo/search/grouping/request/NormalizeSubjectFunction.java index 54a0062e2c9..8bf0a03141a 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/request/NormalizeSubjectFunction.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/request/NormalizeSubjectFunction.java @@ -13,7 +13,17 @@ public class NormalizeSubjectFunction extends FunctionNode { * @param exp The expression to evaluate, must evaluate to a string. */ public NormalizeSubjectFunction(GroupingExpression exp) { - super("normalizesubject", Arrays.asList(exp)); + this(null, null, exp); } + + private NormalizeSubjectFunction(String label, Integer level, GroupingExpression exp) { + super("normalizesubject", label, level, Arrays.asList(exp)); + } + + @Override + public NormalizeSubjectFunction copy() { + return new NormalizeSubjectFunction(getLabel(), getLevelOrNull(), getArg(0).copy()); + } + } diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/NowFunction.java b/container-search/src/main/java/com/yahoo/search/grouping/request/NowFunction.java index 24dd888e98d..75715bad37b 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/request/NowFunction.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/request/NowFunction.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.grouping.request; +import java.util.Arrays; import java.util.Collections; /** @@ -8,6 +9,7 @@ import java.util.Collections; * of seconds since midnight, January 1, 1970 UTC. * * @author Simon Thoresen Hult + * @author bratseth */ public class NowFunction extends FunctionNode { @@ -15,7 +17,17 @@ public class NowFunction extends FunctionNode { * Constructs a new instance of this class. */ public NowFunction() { - super("now", Collections.<GroupingExpression>emptyList()); + this(null, null); } + + private NowFunction(String label, Integer level) { + super("now", label, level, Collections.emptyList()); + } + + @Override + public NowFunction copy() { + return new NowFunction(getLabel(), getLevelOrNull()); + } + } diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/OrFunction.java b/container-search/src/main/java/com/yahoo/search/grouping/request/OrFunction.java index ad1809f312f..43456a4f281 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/request/OrFunction.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/request/OrFunction.java @@ -2,12 +2,14 @@ package com.yahoo.search.grouping.request; import java.util.List; +import java.util.stream.Collectors; /** * This class represents an or-function in a {@link GroupingExpression}. It evaluates to a long that equals the result * of or'ing the results of all arguments together in the order they were given to the constructor. * * @author Simon Thoresen Hult + * @author bratseth */ public class OrFunction extends FunctionNode { @@ -19,11 +21,18 @@ public class OrFunction extends FunctionNode { * @param argN The optional arguments, must evaluate to a long. */ public OrFunction(GroupingExpression arg1, GroupingExpression arg2, GroupingExpression... argN) { - this(asList(arg1, arg2, argN)); + this(null, null, asList(arg1, arg2, argN)); } - private OrFunction(List<GroupingExpression> args) { - super("or", args); + private OrFunction(String label, Integer level, List<GroupingExpression> args) { + super("or", label, level, args); + } + + @Override + public OrFunction copy() { + return new OrFunction(getLabel(), + getLevelOrNull(), + args().stream().map(arg -> arg.copy()).collect(Collectors.toList())); } /** @@ -37,7 +46,7 @@ public class OrFunction extends FunctionNode { if (args.size() < 2) { throw new IllegalArgumentException("Expected 2 or more arguments, got " + args.size() + "."); } - return new OrFunction(args); + return new OrFunction(null, null, args); } } diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/PredefinedFunction.java b/container-search/src/main/java/com/yahoo/search/grouping/request/PredefinedFunction.java index ebae4b238b4..5818c5034b6 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/request/PredefinedFunction.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/request/PredefinedFunction.java @@ -4,6 +4,7 @@ package com.yahoo.search.grouping.request; import java.util.Iterator; import java.util.LinkedList; import java.util.List; +import java.util.stream.Collectors; /** * This class represents a predefined bucket-function in a {@link GroupingExpression}. It maps the input into one of the @@ -13,8 +14,8 @@ import java.util.List; */ public abstract class PredefinedFunction extends FunctionNode { - protected PredefinedFunction(GroupingExpression exp, List<? extends BucketValue> args) { - super("predefined", asList(exp, args)); + protected PredefinedFunction(String label, Integer level, GroupingExpression exp, List<? extends BucketValue> args) { + super("predefined", label, level, asList(exp, args)); Iterator<? extends BucketValue> it = args.iterator(); BucketValue prev = it.next(); while (it.hasNext()) { @@ -47,12 +48,12 @@ public abstract class PredefinedFunction extends FunctionNode { return (BucketValue)getArg(i + 1); } - private static - List<GroupingExpression> asList(GroupingExpression exp, List<? extends BucketValue> args) { + private static List<GroupingExpression> asList(GroupingExpression exp, List<? extends BucketValue> args) { List<GroupingExpression> ret = new LinkedList<>(); ret.add(exp); ret.addAll(args); return ret; } + } diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/RawBucket.java b/container-search/src/main/java/com/yahoo/search/grouping/request/RawBucket.java index 32be9233445..415dbe2adff 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/request/RawBucket.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/request/RawBucket.java @@ -4,7 +4,7 @@ package com.yahoo.search.grouping.request; /** * This class represents a {@link RawValue} bucket in a {@link PredefinedFunction}. * - * @author <a href="mailto:lulf@yahoo-inc.com">Ulf Lilleengen</a> + * @author Ulf Lilleengen */ public class RawBucket extends BucketValue { @@ -25,7 +25,7 @@ public class RawBucket extends BucketValue { * @param to The to-value to assign to this. */ public RawBucket(RawBuffer from, RawBuffer to) { - super(new RawValue(from), new RawValue(to)); + super(null, null, new RawValue(from), new RawValue(to)); } /** @@ -35,6 +35,16 @@ public class RawBucket extends BucketValue { * @param to The to-value to assign to this. */ public RawBucket(ConstantValue<?> from, ConstantValue<?> to) { - super(from, to); + super(null, null, from, to); } + + private RawBucket(String label, Integer level, ConstantValue<?> from, ConstantValue<?> to) { + super(label, level, from, to); + } + + @Override + public RawBucket copy() { + return new RawBucket(getLabel(), getLevelOrNull(), getFrom().copy(), getTo().copy()); + } + } diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/RawBuffer.java b/container-search/src/main/java/com/yahoo/search/grouping/request/RawBuffer.java index 460c547f1d6..9b18d7b374c 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/request/RawBuffer.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/request/RawBuffer.java @@ -7,9 +7,10 @@ import java.util.ArrayList; * This class represents a buffer of byte values to be used as a backing buffer * for raw buckets. * - * @author <a href="mailto:lulf@yahoo-inc.com">Ulf Lilleengen</a> + * @author Ulf Lilleengen */ public class RawBuffer implements Comparable<RawBuffer>, Cloneable { + private final ArrayList<Byte> buffer; /** diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/RawPredefined.java b/container-search/src/main/java/com/yahoo/search/grouping/request/RawPredefined.java index 7f27a428e74..567892a0377 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/request/RawPredefined.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/request/RawPredefined.java @@ -2,12 +2,13 @@ package com.yahoo.search.grouping.request; import java.util.List; +import java.util.stream.Collectors; /** * This class represents a predefined bucket-function in a {@link GroupingExpression} for expressions that evaluate to a * raw. * - * @author <a href="mailto:lulf@yahoo-inc.com">Ulf Lilleengen</a> + * @author Ulf Lilleengen */ public class RawPredefined extends PredefinedFunction { @@ -19,11 +20,23 @@ public class RawPredefined extends PredefinedFunction { * @param argN The optional buckets. */ public RawPredefined(GroupingExpression exp, RawBucket arg1, RawBucket... argN) { - this(exp, asList(arg1, argN)); + this(null, null, exp, asList(arg1, argN)); } - private RawPredefined(GroupingExpression exp, List<RawBucket> args) { - super(exp, args); + private RawPredefined(String label, Integer level, GroupingExpression exp, List<RawBucket> args) { + super(label, level, exp, args); + } + + @Override + public RawPredefined copy() { + return new RawPredefined(getLabel(), + getLevelOrNull(), + getArg(0), + args().stream() + .skip(1) + .map(RawBucket.class::cast) + .map(arg -> arg.copy()) + .collect(Collectors.toList())); } @Override @@ -43,6 +56,6 @@ public class RawPredefined extends PredefinedFunction { if (args.isEmpty()) { throw new IllegalArgumentException("Expected at least one bucket, got none."); } - return new RawPredefined(exp, args); + return new RawPredefined(null, null, exp, args); } } diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/RawValue.java b/container-search/src/main/java/com/yahoo/search/grouping/request/RawValue.java index f66ee362599..c8215b1b389 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/request/RawValue.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/request/RawValue.java @@ -4,7 +4,8 @@ package com.yahoo.search.grouping.request; /** * This class represents a raw value in a {@link GroupingExpression}. * - * @author <a href="mailto:lulf@yahoo-inc.com">Ulf Lilleengen</a> + * @author Ulf Lilleengen + * @author bratseth */ public class RawValue extends ConstantValue<RawBuffer> { @@ -14,6 +15,16 @@ public class RawValue extends ConstantValue<RawBuffer> { * @param value The immutable value to assign to this. */ public RawValue(RawBuffer value) { - super(value); + super(null, null, value); } + + private RawValue(String label, Integer level, RawBuffer value) { + super(label, level, value); + } + + @Override + public RawValue copy() { + return new RawValue(getLabel(), getLevelOrNull(), getValue().clone()); + } + } diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/RelevanceValue.java b/container-search/src/main/java/com/yahoo/search/grouping/request/RelevanceValue.java index 01735c0bdab..30cdf2c6f5c 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/request/RelevanceValue.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/request/RelevanceValue.java @@ -6,6 +6,7 @@ package com.yahoo.search.grouping.request; * the input {@link com.yahoo.search.result.Hit}. * * @author Simon Thoresen Hult + * @author bratseth */ public class RelevanceValue extends DocumentValue { @@ -13,7 +14,16 @@ public class RelevanceValue extends DocumentValue { * Constructs a new instance of this class. */ public RelevanceValue() { - super("relevance()"); + this(null, null); + } + + private RelevanceValue(String label, Integer level) { + super("relevance()", label, level); + } + + @Override + public RelevanceValue copy() { + return new RelevanceValue(getLabel(), getLevelOrNull()); } } diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/ReverseFunction.java b/container-search/src/main/java/com/yahoo/search/grouping/request/ReverseFunction.java index 8283806b5c1..8937018c28f 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/request/ReverseFunction.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/request/ReverseFunction.java @@ -8,6 +8,7 @@ import java.util.Arrays; * result of the argument, sorted in descending order. * * @author baldersheim + * @author bratseth */ public class ReverseFunction extends FunctionNode { @@ -17,6 +18,16 @@ public class ReverseFunction extends FunctionNode { * @param exp The expression to evaluate, must evaluate to a list. */ public ReverseFunction(GroupingExpression exp) { - super("reverse", Arrays.asList(exp)); + this(null, null, exp); } + + private ReverseFunction(String label, Integer level, GroupingExpression exp) { + super("reverse", label, level, Arrays.asList(exp)); + } + + @Override + public ReverseFunction copy() { + return new ReverseFunction(getLabel(), getLevelOrNull(), getArg(0).copy()); + } + } diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/SecondOfMinuteFunction.java b/container-search/src/main/java/com/yahoo/search/grouping/request/SecondOfMinuteFunction.java index b83065af629..7b01d820d89 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/request/SecondOfMinuteFunction.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/request/SecondOfMinuteFunction.java @@ -8,6 +8,7 @@ import java.util.Arrays; * that equals the second of minute (0-59) of the result of the argument. * * @author Simon Thoresen Hult + * @author bratseth */ public class SecondOfMinuteFunction extends FunctionNode { @@ -17,6 +18,16 @@ public class SecondOfMinuteFunction extends FunctionNode { * @param exp The expression to evaluate, must evaluate to a long. */ public SecondOfMinuteFunction(GroupingExpression exp) { - super("time.secondofminute", Arrays.asList(exp)); + this(null, null, exp); } + + private SecondOfMinuteFunction(String label, Integer level, GroupingExpression exp) { + super("time.secondofminute", label, level, Arrays.asList(exp)); + } + + @Override + public SecondOfMinuteFunction copy() { + return new SecondOfMinuteFunction(getLabel(), getLevelOrNull(), getArg(0).copy()); + } + } diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/SizeFunction.java b/container-search/src/main/java/com/yahoo/search/grouping/request/SizeFunction.java index 4b7055ffb89..a9deb2b8e64 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/request/SizeFunction.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/request/SizeFunction.java @@ -8,6 +8,7 @@ import java.util.Arrays; * number of elements in the result of the argument (e.g. the number of elements in an array). * * @author Simon Thoresen Hult + * @author bratseth */ public class SizeFunction extends FunctionNode { @@ -17,7 +18,17 @@ public class SizeFunction extends FunctionNode { * @param exp The expression to evaluate. */ public SizeFunction(GroupingExpression exp) { - super("size", Arrays.asList(exp)); + this(null, null, exp); } + + private SizeFunction(String label, Integer level, GroupingExpression exp) { + super("size", label, level, Arrays.asList(exp)); + } + + @Override + public SizeFunction copy() { + return new SizeFunction(getLabel(), getLevelOrNull(), getArg(0).copy()); + } + } diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/SortFunction.java b/container-search/src/main/java/com/yahoo/search/grouping/request/SortFunction.java index 6ee4644344e..57489b8ee1a 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/request/SortFunction.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/request/SortFunction.java @@ -8,6 +8,7 @@ import java.util.Arrays; * result of the argument, sorted in ascending order. * * @author baldersheim + * @author bratseth */ public class SortFunction extends FunctionNode { @@ -17,6 +18,16 @@ public class SortFunction extends FunctionNode { * @param exp The expression to evaluate, must evaluate to a list. */ public SortFunction(GroupingExpression exp) { - super("sort", Arrays.asList(exp)); + this(null, null, exp); } + + private SortFunction(String label, Integer level, GroupingExpression exp) { + super("sort", label, level, Arrays.asList(exp)); + } + + @Override + public SortFunction copy() { + return new SortFunction(getLabel(), getLevelOrNull(), getArg(0).copy()); + } + } diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/StandardDeviationAggregator.java b/container-search/src/main/java/com/yahoo/search/grouping/request/StandardDeviationAggregator.java index 62bc0085422..d96b840dde9 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/request/StandardDeviationAggregator.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/request/StandardDeviationAggregator.java @@ -6,13 +6,26 @@ package com.yahoo.search.grouping.request; * of the values that the contained expression evaluated to over all the inputs. * * @author bjorncs + * @author bratseth */ public class StandardDeviationAggregator extends AggregatorNode { /** - * @param exp The expression to aggregate on. + * Constructs a new instance of this class. + * + * @param expression the expression to aggregate on. */ - public StandardDeviationAggregator(GroupingExpression exp) { - super("stddev", exp); + public StandardDeviationAggregator(GroupingExpression expression) { + this(null, null, expression); } + + private StandardDeviationAggregator(String label, Integer level, GroupingExpression expression) { + super("stddev", label, level, expression); + } + + @Override + public StandardDeviationAggregator copy() { + return new StandardDeviationAggregator(getLabel(), getLevelOrNull(), getExpression().copy()); + } + } diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/StrCatFunction.java b/container-search/src/main/java/com/yahoo/search/grouping/request/StrCatFunction.java index 97dcc909254..3b266b884bf 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/request/StrCatFunction.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/request/StrCatFunction.java @@ -2,12 +2,14 @@ package com.yahoo.search.grouping.request; import java.util.List; +import java.util.stream.Collectors; /** * This class represents a strcat-function in a {@link GroupingExpression}. It evaluates to a string that equals the * contatenation of the string results of all arguments in the order they were given to the constructor. * * @author Simon Thoresen Hult + * @author bratseth */ public class StrCatFunction extends FunctionNode { @@ -19,11 +21,18 @@ public class StrCatFunction extends FunctionNode { * @param argN The optional arguments, must evaluate to a string. */ public StrCatFunction(GroupingExpression arg1, GroupingExpression arg2, GroupingExpression... argN) { - this(asList(arg1, arg2, argN)); + this(null, null, asList(arg1, arg2, argN)); } - private StrCatFunction(List<GroupingExpression> args) { - super("strcat", args); + private StrCatFunction(String label, Integer level, List<GroupingExpression> args) { + super("strcat", label, level, args); + } + + @Override + public StrCatFunction copy() { + return new StrCatFunction(getLabel(), + getLevelOrNull(), + args().stream().map(arg -> arg.copy()).collect(Collectors.toList())); } /** @@ -37,7 +46,7 @@ public class StrCatFunction extends FunctionNode { if (args.size() < 2) { throw new IllegalArgumentException("Expected 2 or more arguments, got " + args.size() + "."); } - return new StrCatFunction(args); + return new StrCatFunction(null, null, args); } } diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/StrLenFunction.java b/container-search/src/main/java/com/yahoo/search/grouping/request/StrLenFunction.java index e9ca48b3db3..d784e37b337 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/request/StrLenFunction.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/request/StrLenFunction.java @@ -8,6 +8,7 @@ import java.util.Arrays; * number of bytes in the string result of the argument. * * @author Simon Thoresen Hult + * @author bratseth */ public class StrLenFunction extends FunctionNode { @@ -17,7 +18,17 @@ public class StrLenFunction extends FunctionNode { * @param exp The expression to evaluate, must evaluate to a string. */ public StrLenFunction(GroupingExpression exp) { - super("strlen", Arrays.asList(exp)); + this(null, null, exp); } + + private StrLenFunction(String label, Integer level, GroupingExpression exp) { + super("strlen", label, level, Arrays.asList(exp)); + } + + @Override + public StrLenFunction copy() { + return new StrLenFunction(getLabel(), getLevelOrNull(), getArg(0).copy()); + } + } diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/StringBucket.java b/container-search/src/main/java/com/yahoo/search/grouping/request/StringBucket.java index 80e85ed9e78..5558f1443db 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/request/StringBucket.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/request/StringBucket.java @@ -25,7 +25,7 @@ public class StringBucket extends BucketValue { * @param to The to-value to assign to this. */ public StringBucket(String from, String to) { - super(new StringValue(from), new StringValue(to)); + super(null, null, new StringValue(from), new StringValue(to)); } /** @@ -35,6 +35,16 @@ public class StringBucket extends BucketValue { * @param to The to-value to assign to this. */ public StringBucket(ConstantValue<?> from, ConstantValue<?> to) { - super(from, to); + super(null, null, from, to); } + + private StringBucket(String label, Integer level, ConstantValue<?> from, ConstantValue<?> to) { + super(label, level, from, to); + } + + @Override + public StringBucket copy() { + return new StringBucket(getLabel(), getLevelOrNull(), getFrom().copy(), getTo().copy()); + } + } diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/StringPredefined.java b/container-search/src/main/java/com/yahoo/search/grouping/request/StringPredefined.java index 465ff96157d..cae9bc113c9 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/request/StringPredefined.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/request/StringPredefined.java @@ -2,6 +2,7 @@ package com.yahoo.search.grouping.request; import java.util.List; +import java.util.stream.Collectors; /** * This class represents a predefined bucket-function in a {@link GroupingExpression} for expressions that evaluate to a @@ -19,11 +20,23 @@ public class StringPredefined extends PredefinedFunction { * @param argN The optional buckets. */ public StringPredefined(GroupingExpression exp, StringBucket arg1, StringBucket... argN) { - this(exp, asList(arg1, argN)); + this(null, null, exp, asList(arg1, argN)); } - private StringPredefined(GroupingExpression exp, List<StringBucket> args) { - super(exp, args); + private StringPredefined(String label, Integer level, GroupingExpression exp, List<StringBucket> args) { + super(label, level, exp, args); + } + + @Override + public StringPredefined copy() { + return new StringPredefined(getLabel(), + getLevelOrNull(), + getArg(0), + args().stream() + .skip(1) + .map(StringBucket.class::cast) + .map(arg -> arg.copy()) + .collect(Collectors.toList())); } @Override @@ -43,6 +56,6 @@ public class StringPredefined extends PredefinedFunction { if (args.isEmpty()) { throw new IllegalArgumentException("Expected at least one bucket, got none."); } - return new StringPredefined(exp, args); + return new StringPredefined(null, null, exp, args); } } diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/StringValue.java b/container-search/src/main/java/com/yahoo/search/grouping/request/StringValue.java index 4fc630c7b1b..473e6b73f95 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/request/StringValue.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/request/StringValue.java @@ -5,6 +5,7 @@ package com.yahoo.search.grouping.request; * This class represents a constant {@link String} value in a {@link GroupingExpression}. * * @author Simon Thoresen Hult + * @author bratseth */ public class StringValue extends ConstantValue<String> { @@ -14,6 +15,16 @@ public class StringValue extends ConstantValue<String> { * @param value The immutable value to assign to this. */ public StringValue(String value) { - super(value); + super(null, null, value); } + + private StringValue(String label, Integer level, String value) { + super(label, level, value); + } + + @Override + public StringValue copy() { + return new StringValue(getLabel(), getLevelOrNull(), getValue()); + } + } diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/SubFunction.java b/container-search/src/main/java/com/yahoo/search/grouping/request/SubFunction.java index 1d0456ffcf2..e0fe533e361 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/request/SubFunction.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/request/SubFunction.java @@ -2,6 +2,7 @@ package com.yahoo.search.grouping.request; import java.util.List; +import java.util.stream.Collectors; /** * This class represents a div-function in a {@link GroupingExpression}. It evaluates to a number that equals the result @@ -20,11 +21,18 @@ public class SubFunction extends FunctionNode { * @param argN The optional arguments, must evaluate to a number. */ public SubFunction(GroupingExpression arg1, GroupingExpression arg2, GroupingExpression... argN) { - this(asList(arg1, arg2, argN)); + this(null, null, asList(arg1, arg2, argN)); } - private SubFunction(List<GroupingExpression> args) { - super("sub", args); + private SubFunction(String label, Integer level, List<GroupingExpression> args) { + super("sub", label, level, args); + } + + @Override + public SubFunction copy() { + return new SubFunction(getLabel(), + getLevelOrNull(), + args().stream().map(arg -> arg.copy()).collect(Collectors.toList())); } /** @@ -38,6 +46,6 @@ public class SubFunction extends FunctionNode { if (args.size() < 2) { throw new IllegalArgumentException("Expected 2 or more arguments, got " + args.size() + "."); } - return new SubFunction(args); + return new SubFunction(null, null, args); } } diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/SumAggregator.java b/container-search/src/main/java/com/yahoo/search/grouping/request/SumAggregator.java index b50dd84e27e..0b34b0660d5 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/request/SumAggregator.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/request/SumAggregator.java @@ -6,15 +6,26 @@ package com.yahoo.search.grouping.request; * the contained expression evaluated to over all the inputs. * * @author Simon Thoresen Hult + * @author bratseth */ public class SumAggregator extends AggregatorNode { /** * Constructs a new instance of this class. * - * @param exp The expression to aggregate on. + * @param expression the expression to aggregate on. */ - public SumAggregator(GroupingExpression exp) { - super("sum", exp); + public SumAggregator(GroupingExpression expression) { + this(null, null, expression); } + + private SumAggregator(String label, Integer level, GroupingExpression expression) { + super("sum", label, level, expression); + } + + @Override + public SumAggregator copy() { + return new SumAggregator(getLabel(), getLevelOrNull(), getExpression().copy()); + } + } diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/SummaryValue.java b/container-search/src/main/java/com/yahoo/search/grouping/request/SummaryValue.java index 6cdaa3b2e4a..68641d6e6e6 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/request/SummaryValue.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/request/SummaryValue.java @@ -15,8 +15,7 @@ public class SummaryValue extends DocumentValue { * Constructs a new instance of this class, using the default summary class. */ public SummaryValue() { - super("summary()"); - name = null; + this(null, null, null); } /** @@ -25,10 +24,19 @@ public class SummaryValue extends DocumentValue { * @param summaryName The name of the summary class to assign to this. */ public SummaryValue(String summaryName) { - super("summary(" + summaryName + ")"); + this(null, null, summaryName); + } + + private SummaryValue(String label, Integer level, String summaryName) { + super("summary(" + (summaryName == null ? "" : summaryName) + ")", label, level); name = summaryName; } + @Override + public SummaryValue copy() { + return new SummaryValue(getLabel(), getLevelOrNull(), getSummaryName()); + } + /** * Returns the name of the summary class used to retrieve the hit from the search node. * @@ -37,4 +45,5 @@ public class SummaryValue extends DocumentValue { public String getSummaryName() { return name; } + } diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/ToDoubleFunction.java b/container-search/src/main/java/com/yahoo/search/grouping/request/ToDoubleFunction.java index c6816dda621..8e7490d0cc6 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/request/ToDoubleFunction.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/request/ToDoubleFunction.java @@ -8,6 +8,7 @@ import java.util.Arrays; * a double. If the argument can not be converted, this function returns 0. * * @author baldersheim + * @author bratseth */ public class ToDoubleFunction extends FunctionNode { @@ -17,6 +18,18 @@ public class ToDoubleFunction extends FunctionNode { * @param exp The expression to evaluate. */ public ToDoubleFunction(GroupingExpression exp) { - super("todouble", Arrays.asList(exp)); + this(null, null, exp); } + + private ToDoubleFunction(String label, Integer level, GroupingExpression exp) { + super("todouble", label, level, Arrays.asList(exp)); + } + + @Override + public ToDoubleFunction copy() { + return new ToDoubleFunction(getLabel(), getLevelOrNull(), getArg(0).copy()); + } + } + + diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/ToLongFunction.java b/container-search/src/main/java/com/yahoo/search/grouping/request/ToLongFunction.java index c513fbd76e6..77470487a6e 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/request/ToLongFunction.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/request/ToLongFunction.java @@ -8,6 +8,7 @@ import java.util.Arrays; * long. If the argument can not be converted, this function returns 0. * * @author baldersheim + * @author bratseth */ public class ToLongFunction extends FunctionNode { @@ -17,6 +18,16 @@ public class ToLongFunction extends FunctionNode { * @param exp The expression to evaluate. */ public ToLongFunction(GroupingExpression exp) { - super("tolong", Arrays.asList(exp)); + this(null, null, exp); } + + private ToLongFunction(String label, Integer level, GroupingExpression exp) { + super("tolong", label, level, Arrays.asList(exp)); + } + + @Override + public ToLongFunction copy() { + return new ToLongFunction(getLabel(), getLevelOrNull(), getArg(0).copy()); + } + } diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/ToRawFunction.java b/container-search/src/main/java/com/yahoo/search/grouping/request/ToRawFunction.java index 3c1c335b8ef..2f32b0fe139 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/request/ToRawFunction.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/request/ToRawFunction.java @@ -8,7 +8,8 @@ import java.util.Arrays; * converts the result of the argument to a raw type. If the argument can not * be converted, this function returns null. * - * @author <a href="mailto:lulf@yahoo-inc.com">Ulf Lilleengen</a> + * @author Ulf Lilleengen + * @author bratseth */ public class ToRawFunction extends FunctionNode { @@ -18,6 +19,16 @@ public class ToRawFunction extends FunctionNode { * @param exp The expression to evaluate. */ public ToRawFunction(GroupingExpression exp) { - super("toraw", Arrays.asList(exp)); + this(null, null, exp); } + + private ToRawFunction(String label, Integer level, GroupingExpression exp) { + super("toraw", label, level, Arrays.asList(exp)); + } + + @Override + public ToRawFunction copy() { + return new ToRawFunction(getLabel(), getLevelOrNull(), getArg(0).copy()); + } + } diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/ToStringFunction.java b/container-search/src/main/java/com/yahoo/search/grouping/request/ToStringFunction.java index 7f4e48c8d81..dd7bd0b5250 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/request/ToStringFunction.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/request/ToStringFunction.java @@ -8,6 +8,7 @@ import java.util.Arrays; * long. If the argument can not be converted, this function returns 0. * * @author baldersheim + * @author bratseth */ public class ToStringFunction extends FunctionNode { @@ -17,6 +18,16 @@ public class ToStringFunction extends FunctionNode { * @param exp The expression to evaluate. */ public ToStringFunction(GroupingExpression exp) { - super("tostring", Arrays.asList(exp)); + this(null, null, exp); } + + private ToStringFunction(String label, Integer level, GroupingExpression exp) { + super("tostring", label, level, Arrays.asList(exp)); + } + + @Override + public ToStringFunction copy() { + return new ToStringFunction(getLabel(), getLevelOrNull(), getArg(0).copy()); + } + } diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/UcaFunction.java b/container-search/src/main/java/com/yahoo/search/grouping/request/UcaFunction.java index 2efecc235ac..787d7aa5863 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/request/UcaFunction.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/request/UcaFunction.java @@ -2,18 +2,20 @@ package com.yahoo.search.grouping.request; import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; /** * This class represents an uca-function in a {@link GroupingExpression}. * - * @author <a href="mailto:lulf@yahoo-inc.com">Ulf Lilleengen</a> + * @author Ulf Lilleengen + * @author bratseth */ public class UcaFunction extends FunctionNode { private final String locale; private final String strength; - /** * Constructs a new instance of this class. * @@ -21,9 +23,7 @@ public class UcaFunction extends FunctionNode { * @param locale The locale to used for sorting. */ public UcaFunction(GroupingExpression exp, String locale) { - super("uca", Arrays.asList(exp, new StringValue(locale))); - this.locale = locale; - this.strength = "TERTIARY"; + this(null, null, Arrays.asList(exp, new StringValue(locale))); } /** @@ -34,12 +34,22 @@ public class UcaFunction extends FunctionNode { * @param strength The strength level to use. */ public UcaFunction(GroupingExpression exp, String locale, String strength) { - super("uca", Arrays.asList(exp, new StringValue(locale), new StringValue(strength))); - if (!validStrength(strength)) { + this(null, null, Arrays.asList(exp, new StringValue(locale), new StringValue(strength))); + if ( ! validStrength(strength)) throw new IllegalArgumentException("Not a valid UCA strength: " + strength); - } - this.locale = locale; - this.strength = strength; + } + + private UcaFunction(String label, Integer level, List<GroupingExpression> args) { + super("uca", label, level, args); + this.locale = ((StringValue)args.get(1)).getValue(); + this.strength = args.size() > 2 ? ((StringValue)args.get(1)).getValue() : "TERTIARY"; + } + + @Override + public UcaFunction copy() { + return new UcaFunction(getLabel(), + getLevelOrNull(), + args().stream().map(arg -> arg.copy()).collect(Collectors.toList())); } private boolean validStrength(String strength) { diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/XorAggregator.java b/container-search/src/main/java/com/yahoo/search/grouping/request/XorAggregator.java index 18ce7abd96c..e68303b564c 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/request/XorAggregator.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/request/XorAggregator.java @@ -6,15 +6,26 @@ package com.yahoo.search.grouping.request; * the contained expression evaluated to over all the inputs. * * @author Simon Thoresen Hult + * @author bratseth */ public class XorAggregator extends AggregatorNode { /** * Constructs a new instance of this class. * - * @param exp The expression to aggregate on. + * @param expression the expression to aggregate on. */ - public XorAggregator(GroupingExpression exp) { - super("xor", exp); + public XorAggregator(GroupingExpression expression) { + this(null, null, expression); } + + private XorAggregator(String label, Integer level, GroupingExpression expression) { + super("xor", label, level, expression); + } + + @Override + public XorAggregator copy() { + return new XorAggregator(getLabel(), getLevelOrNull(), getExpression().copy()); + } + } diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/XorBitFunction.java b/container-search/src/main/java/com/yahoo/search/grouping/request/XorBitFunction.java index 2f6061a3aa2..8fdd2dc5839 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/request/XorBitFunction.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/request/XorBitFunction.java @@ -2,12 +2,15 @@ package com.yahoo.search.grouping.request; import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; /** * This class represents an xor-function in a {@link GroupingExpression}. It evaluates to a long that equals the xor of * 'width' bits over the binary representation of the result of the argument. * * @author Simon Thoresen Hult + * @author bratseth */ public class XorBitFunction extends FunctionNode { @@ -18,7 +21,18 @@ public class XorBitFunction extends FunctionNode { * @param numBits The number of bits of the expression value to xor. */ public XorBitFunction(GroupingExpression exp, int numBits) { - super("xorbit", Arrays.asList(exp, new LongValue(numBits))); + this(null, null, Arrays.asList(exp, new LongValue(numBits))); + } + + private XorBitFunction(String label, Integer level, List<GroupingExpression> exp) { + super("xorbit", label, level, exp); + } + + @Override + public XorBitFunction copy() { + return new XorBitFunction(getLabel(), + getLevelOrNull(), + args().stream().map(arg -> arg.copy()).collect(Collectors.toList())); } /** diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/XorFunction.java b/container-search/src/main/java/com/yahoo/search/grouping/request/XorFunction.java index c3a5f5d88ad..a7a1e4d2a8f 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/request/XorFunction.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/request/XorFunction.java @@ -2,12 +2,14 @@ package com.yahoo.search.grouping.request; import java.util.List; +import java.util.stream.Collectors; /** * This class represents an xor-function in a {@link GroupingExpression}. It evaluates to a long that equals the result * of and'ing the results of all arguments together in the order they were given to the constructor. * * @author Simon Thoresen Hult + * @author bratseth */ public class XorFunction extends FunctionNode { @@ -19,11 +21,18 @@ public class XorFunction extends FunctionNode { * @param argN The optional arguments, must evaluate to a long. */ public XorFunction(GroupingExpression arg1, GroupingExpression arg2, GroupingExpression... argN) { - this(asList(arg1, arg2, argN)); + this(null, null, asList(arg1, arg2, argN)); } - private XorFunction(List<GroupingExpression> args) { - super("xor", args); + private XorFunction(String label, Integer level, List<GroupingExpression> args) { + super("xor", label, level, args); + } + + @Override + public XorFunction copy() { + return new XorFunction(getLabel(), + getLevelOrNull(), + args().stream().map(arg -> arg.copy()).collect(Collectors.toList())); } /** @@ -37,7 +46,7 @@ public class XorFunction extends FunctionNode { if (args.size() < 2) { throw new IllegalArgumentException("Expected 2 or more arguments, got " + args.size() + "."); } - return new XorFunction(args); + return new XorFunction(null, null, args); } } diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/YearFunction.java b/container-search/src/main/java/com/yahoo/search/grouping/request/YearFunction.java index 5bda8231929..737597a585c 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/request/YearFunction.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/request/YearFunction.java @@ -8,6 +8,7 @@ import java.util.Arrays; * the full year (e.g. 2010) of the result of the argument. * * @author Simon Thoresen Hult + * @author bratseth */ public class YearFunction extends FunctionNode { @@ -17,6 +18,16 @@ public class YearFunction extends FunctionNode { * @param exp The expression to evaluate, must evaluate to a long. */ public YearFunction(GroupingExpression exp) { - super("time.year", Arrays.asList(exp)); + this(null, null, exp); } + + private YearFunction(String label, Integer level, GroupingExpression exp) { + super("time.year", label, level, Arrays.asList(exp)); + } + + @Override + public YearFunction copy() { + return new YearFunction(getLabel(), getLevelOrNull(), getArg(0).copy()); + } + } diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/YmumValue.java b/container-search/src/main/java/com/yahoo/search/grouping/request/YmumValue.java index c5749c4673d..8ca772b9188 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/request/YmumValue.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/request/YmumValue.java @@ -6,6 +6,7 @@ package com.yahoo.search.grouping.request; * input {@link com.yahoo.search.result.Hit}. * * @author Simon Thoresen Hult + * @author bratseth */ public class YmumValue extends DocumentValue { @@ -13,7 +14,17 @@ public class YmumValue extends DocumentValue { * Constructs a new instance of this class. */ public YmumValue() { - super("ymum()"); + this(null, null); } + + private YmumValue(String label, Integer level) { + super("ymum()", label, level); + } + + @Override + public YmumValue copy() { + return new YmumValue(getLabel(), getLevelOrNull()); + } + } diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/ZCurveXFunction.java b/container-search/src/main/java/com/yahoo/search/grouping/request/ZCurveXFunction.java index bcd6f2ba71a..5ab057ef8a8 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/request/ZCurveXFunction.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/request/ZCurveXFunction.java @@ -5,14 +5,26 @@ import java.util.Arrays; /** * @author baldersheim + * @author bratseth */ public class ZCurveXFunction extends FunctionNode { + /** * Constructs a new instance of this class. * * @param exp The expression to evaluate, must evaluate to a long or long[]. */ public ZCurveXFunction(GroupingExpression exp) { - super("zcurve.x", Arrays.asList(exp)); + this(null, null, exp); + } + + private ZCurveXFunction(String label, Integer level, GroupingExpression exp) { + super("zcurve.x", label, level, Arrays.asList(exp)); } + + @Override + public ZCurveXFunction copy() { + return new ZCurveXFunction(getLabel(), getLevelOrNull(), getArg(0).copy()); + } + } diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/ZCurveYFunction.java b/container-search/src/main/java/com/yahoo/search/grouping/request/ZCurveYFunction.java index 57c55d41209..fa645eb9141 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/request/ZCurveYFunction.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/request/ZCurveYFunction.java @@ -5,6 +5,7 @@ import java.util.Arrays; /** * @author baldersheim + * @author bratseth */ public class ZCurveYFunction extends FunctionNode { /** @@ -13,6 +14,16 @@ public class ZCurveYFunction extends FunctionNode { * @param exp The expression to evaluate, must evaluate to a long or long[]. */ public ZCurveYFunction(GroupingExpression exp) { - super("zcurve.y", Arrays.asList(exp)); + this(null, null, exp); } + + private ZCurveYFunction(String label, Integer level, GroupingExpression exp) { + super("zcurve.y", label, level, Arrays.asList(exp)); + } + + @Override + public ZCurveYFunction copy() { + return new ZCurveYFunction(getLabel(), getLevelOrNull(), getArg(0).copy()); + } + } diff --git a/container-search/src/main/java/com/yahoo/search/grouping/vespa/CompositeContinuation.java b/container-search/src/main/java/com/yahoo/search/grouping/vespa/CompositeContinuation.java index a02b80db16f..7a12a200f5b 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/vespa/CompositeContinuation.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/vespa/CompositeContinuation.java @@ -10,10 +10,17 @@ import java.util.List; /** * @author Simon Thoresen Hult */ -class CompositeContinuation extends EncodableContinuation implements Iterable<EncodableContinuation> { +final class CompositeContinuation extends EncodableContinuation implements Iterable<EncodableContinuation> { private final List<EncodableContinuation> children = new ArrayList<>(); + @Override + public CompositeContinuation copy() { + CompositeContinuation copy = new CompositeContinuation(); + this.children.forEach(child -> copy.add(child.copy())); + return copy; + } + public CompositeContinuation add(EncodableContinuation child) { children.add(child); return this; @@ -48,4 +55,5 @@ class CompositeContinuation extends EncodableContinuation implements Iterable<En } return ret; } + } diff --git a/container-search/src/main/java/com/yahoo/search/grouping/vespa/EncodableContinuation.java b/container-search/src/main/java/com/yahoo/search/grouping/vespa/EncodableContinuation.java index 2ec371475c2..df24b7129ff 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/vespa/EncodableContinuation.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/vespa/EncodableContinuation.java @@ -11,9 +11,13 @@ abstract class EncodableContinuation extends Continuation { public abstract void encode(IntegerEncoder out); @Override + public abstract EncodableContinuation copy(); + + @Override public final String toString() { IntegerEncoder encoder = new IntegerEncoder(); encode(encoder); return encoder.toString(); } + } diff --git a/container-search/src/main/java/com/yahoo/search/grouping/vespa/GroupingExecutor.java b/container-search/src/main/java/com/yahoo/search/grouping/vespa/GroupingExecutor.java index bf7eb8dc12e..61856467652 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/vespa/GroupingExecutor.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/vespa/GroupingExecutor.java @@ -46,7 +46,8 @@ import com.yahoo.vespa.objects.ObjectPredicate; public class GroupingExecutor extends Searcher { public final static String COMPONENT_NAME = "GroupingExecutor"; - private final static CompoundName PROP_GROUPINGLIST = newCompoundName("GroupingList"); + private final static String GROUPING_LIST = "GroupingList"; + private final static CompoundName PROP_GROUPINGLIST = newCompoundName(GROUPING_LIST); private final static Logger log = Logger.getLogger(GroupingExecutor.class.getName()); /** @@ -69,26 +70,19 @@ public class GroupingExecutor extends Searcher { @Override public Result search(Query query, Execution execution) { String error = QueryCanonicalizer.canonicalize(query); - if (error != null) { - return new Result(query, ErrorMessage.createIllegalQuery(error)); - } + if (error != null) return new Result(query, ErrorMessage.createIllegalQuery(error)); + query.prepare(); - // Retrieve grouping requests from query. - List<GroupingRequest> reqList = GroupingRequest.getRequests(query); - if (reqList.isEmpty()) { - return execution.search(query); - } + if (query.getSelect().getGrouping().isEmpty()) return execution.search(query); // Convert requests to Vespa style grouping. Map<Integer, Grouping> groupingMap = new HashMap<>(); List<RequestContext> requestContextList = new LinkedList<>(); - for (GroupingRequest grpRequest : reqList) { - requestContextList.add(convertRequest(query, grpRequest, groupingMap)); - } - if (groupingMap.isEmpty()) { - return execution.search(query); - } + for (int i = 0; i < query.getSelect().getGrouping().size(); i++) + requestContextList.add(convertRequest(query, query.getSelect().getGrouping().get(i), i, groupingMap)); + + if (groupingMap.isEmpty()) return execution.search(query); // Perform the necessary passes to execute grouping. Result result = performSearch(query, execution, groupingMap); @@ -156,8 +150,8 @@ public class GroupingExecutor extends Searcher { * @param map The grouping map to write to. * @return The context required to identify the request results. */ - private RequestContext convertRequest(Query query, GroupingRequest req, Map<Integer, Grouping> map) { - RequestBuilder builder = new RequestBuilder(req.getRequestId()); + private RequestContext convertRequest(Query query, GroupingRequest req, int requestId, Map<Integer, Grouping> map) { + RequestBuilder builder = new RequestBuilder(requestId); builder.setRootOperation(req.getRootOperation()); builder.setDefaultSummaryName(query.getPresentation().getSummary()); builder.setTimeZone(req.getTimeZone()); diff --git a/container-search/src/main/java/com/yahoo/search/grouping/vespa/OffsetContinuation.java b/container-search/src/main/java/com/yahoo/search/grouping/vespa/OffsetContinuation.java index 6adec94132e..324f3eb0b6e 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/vespa/OffsetContinuation.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/vespa/OffsetContinuation.java @@ -20,6 +20,11 @@ class OffsetContinuation extends EncodableContinuation { this.flags = flags; } + @Override + public OffsetContinuation copy() { + return this; // immutable + } + public ResultId getResultId() { return resultId; } @@ -81,4 +86,5 @@ class OffsetContinuation extends EncodableContinuation { int flags = in.next(); return new OffsetContinuation(resultId, tag, offset, flags); } + } diff --git a/container-search/src/main/java/com/yahoo/search/grouping/vespa/ResultId.java b/container-search/src/main/java/com/yahoo/search/grouping/vespa/ResultId.java index 0d09e7ab69d..82784bcf9dd 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/vespa/ResultId.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/vespa/ResultId.java @@ -4,6 +4,8 @@ package com.yahoo.search.grouping.vespa; import java.util.Arrays; /** + * Immutable + * * @author Simon Thoresen Hult */ class ResultId { diff --git a/container-search/src/main/java/com/yahoo/search/query/Model.java b/container-search/src/main/java/com/yahoo/search/query/Model.java index bd0c229085b..cbc15bf39a1 100644 --- a/container-search/src/main/java/com/yahoo/search/query/Model.java +++ b/container-search/src/main/java/com/yahoo/search/query/Model.java @@ -237,12 +237,13 @@ public class Model implements Cloneable { public void setQueryString(String queryString) { if (queryString == null) queryString=""; this.queryString = queryString; - queryTree = null; // Cause parsing of the new query string next time the tree is accessed + clearQueryTree(); } /** * Returns the query string which caused the original query tree of this model to come about. * Note that changes to the query tree are <b>not</b> reflected in this query string. + * Note that changes to the query tree are <b>not</b> reflected in this query string. * * @return the original (or reassigned) query string - never null */ @@ -265,6 +266,14 @@ public class Model implements Cloneable { } /** + * Clears the parsed query such that it will be created anew from the textual representation (a query string or + * select.where expression) on the next access. + */ + public void clearQueryTree() { + queryTree = null; + } + + /** * Returns the filter string set for this query. * The filter is included in the query tree at the time the query tree is parsed */ @@ -337,25 +346,25 @@ public class Model implements Cloneable { QueryHelper.combineHash(encoding,filter,language,getQueryTree(),sources,restrict,defaultIndex,type,searchPath); } - + @Override public Object clone() { try { - Model clone = (Model) super.clone(); + Model clone = (Model)super.clone(); if (queryTree != null) clone.queryTree = this.queryTree.clone(); - if (sources !=null) + if (sources != null) clone.sources = new LinkedHashSet<>(this.sources); - if (restrict !=null) + if (restrict != null) clone.restrict = new LinkedHashSet<>(this.restrict); return clone; } catch (CloneNotSupportedException e) { - throw new RuntimeException("Someone inserted a noncloneable superclass",e); + throw new RuntimeException("Someone inserted a noncloneable superclass", e); } } public Model cloneFor(Query q) { - Model model = (Model) this.clone(); + Model model = (Model)this.clone(); model.setParent(q); return model; } @@ -365,7 +374,7 @@ public class Model implements Cloneable { /** Assigns the query owning this */ public void setParent(Query parent) { - if (parent==null) throw new NullPointerException("A query models owner cannot be null"); + if (parent == null) throw new NullPointerException("A query models owner cannot be null"); this.parent = parent; } @@ -403,7 +412,7 @@ public class Model implements Cloneable { /** Sets the execution working on this. For internal use. */ public void setExecution(Execution execution) { - if (execution==this.execution) return; + if (execution == this.execution) return; // If not already coupled, bind the trace of the new execution into the existing execution trace if (execution.trace().traceNode().isRoot() @@ -425,7 +434,7 @@ public class Model implements Cloneable { /** Returns the Execution working on this, or a null execution if none. For internal use. */ public Execution getExecution() { return execution; } - private void setFromString(String string,Set<String> set) { + private void setFromString(String string, Set<String> set) { set.clear(); for (String item : string.split(",")) set.add(item.trim()); @@ -520,7 +529,6 @@ public class Model implements Cloneable { return false; } - /** * Set the YTrace header value to use when transmitting this model to a * search backend (of some kind). diff --git a/container-search/src/main/java/com/yahoo/search/query/Select.java b/container-search/src/main/java/com/yahoo/search/query/Select.java index 3ffc6bddb24..bbc152c6391 100644 --- a/container-search/src/main/java/com/yahoo/search/query/Select.java +++ b/container-search/src/main/java/com/yahoo/search/query/Select.java @@ -10,6 +10,11 @@ import com.yahoo.search.query.profile.types.FieldDescription; import com.yahoo.search.query.profile.types.QueryProfileType; import com.yahoo.search.yql.VespaGroupingStep; +import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Objects; /** @@ -27,67 +32,75 @@ public class Select implements Cloneable { public static final String WHERE = "where"; public static final String GROUPING = "grouping"; + private final Query parent; + private final List<GroupingRequest> groupingRequests; - private static Model model; - private Query parent; - private String where = ""; - private String grouping = ""; + private String where; + private String grouping; static { argumentType = new QueryProfileType(SELECT); argumentType.setStrict(true); argumentType.setBuiltin(true); - argumentType.addField(new FieldDescription(WHERE, "string", "where")); - argumentType.addField(new FieldDescription(GROUPING, "string", "grouping")); + argumentType.addField(new FieldDescription(WHERE, "string")); + argumentType.addField(new FieldDescription(GROUPING, "string")); argumentType.freeze(); - argumentTypeName=new CompoundName(argumentType.getId().getName()); + argumentTypeName = new CompoundName(argumentType.getId().getName()); } public static QueryProfileType getArgumentType() { return argumentType; } - public Select(String where, String grouping){ - this.where = where; - this.grouping = grouping; - } - + /** Creates an empty select statement */ public Select(Query query) { - setParent(query); - model = query.getModel(); + this("", "", query); } + public Select(String where, String grouping, Query query) { + this(where, grouping, query, Collections.emptyList()); + } - /** Returns the query owning this, never null */ - private Query getParent() { return parent; } - - - /** Assigns the query owning this */ - public void setParent(Query parent) { - if (parent==null) throw new NullPointerException("A query models owner cannot be null"); - this.parent = parent; + private Select(String where, String grouping, Query query, List<GroupingRequest> groupingRequests) { + this.where = Objects.requireNonNull(where, "A Select must have a where string (possibly the empty string)"); + this.grouping = Objects.requireNonNull(grouping, "A Select must have a select string (possibly the empty string)"); + this.parent = Objects.requireNonNull(query, "A Select must have a parent query"); + this.groupingRequests = deepCopy(groupingRequests, this); } + private static List<GroupingRequest> deepCopy(List<GroupingRequest> groupingRequests, Select parentOfCopy) { + List<GroupingRequest> copy = new ArrayList<>(groupingRequests.size()); + for (GroupingRequest request : groupingRequests) + copy.add(request.copy(parentOfCopy)); + return copy; + } - /** Set the where-clause for the query. Must be a JSON-string, with the format described in the Select Reference doc - https://docs.vespa.ai/documentation/reference/select-reference.html. */ - public void setWhere(String where) { + /** + * Sets the document selection criterion of the query. + * + * @param where the documents to select as a JSON string on the format specified in + * <a href="https://docs.vespa.ai/documentation/reference/select-reference.html">the select reference doc</a> + */ + public void setWhereString(String where) { this.where = where; - model.setType(SELECT); + parent.getModel().setType(SELECT); - // Setting the queryTree to null - model.setQueryString(null); + // This replaces the current query + parent.getModel().clearQueryTree(); } + /** Returns the where clause string previously assigned, or an empty string if none */ + public String getWhereString(){ return where; } - /** Returns the where-clause in the query */ - public String getWhereString(){ - return this.where; - } - - /** Set the grouping-string for the query. Must be a JSON-string, with the format described in the Select Reference doc - https://docs.vespa.ai/documentation/reference/select-reference.html. */ - public void setGrouping(String grouping){ + /** + * Sets the grouping operation of the query. + * + * @param grouping the grouping to perform as a JSON string on the format specified in + * <a href="https://docs.vespa.ai/documentation/reference/select-reference.html">the select reference doc</a> + */ + public void setGroupingString(String grouping) { + groupingRequests.clear(); this.grouping = grouping; SelectParser parser = (SelectParser) ParserFactory.newInstance(Query.Type.SELECT, new ParserEnvironment()); - for (VespaGroupingStep step : parser.getGroupingSteps(grouping)) { GroupingRequest.newInstance(parent) .setRootOperation(step.getOperation()) @@ -98,13 +111,28 @@ public class Select implements Cloneable { /** Returns the grouping in the query */ public String getGroupingString(){ - return this.grouping; + return grouping; } + /** + * Returns the query's {@link GroupingRequest} as a mutable list. Changing this directly changes the grouping + * operations which will be performed by this query. + */ + public List<GroupingRequest> getGrouping() { return groupingRequests; } + @Override public String toString() { return "where: [" + where + "], grouping: [" + grouping+ "]"; } + @Override + public Object clone() { + return new Select(where, grouping, parent, groupingRequests); + } + + public Select cloneFor(Query parent) { + return new Select(where, grouping, parent, groupingRequests); + } + } diff --git a/container-search/src/main/java/com/yahoo/search/query/properties/QueryProperties.java b/container-search/src/main/java/com/yahoo/search/query/properties/QueryProperties.java index 71002166b11..559a7279f83 100644 --- a/container-search/src/main/java/com/yahoo/search/query/properties/QueryProperties.java +++ b/container-search/src/main/java/com/yahoo/search/query/properties/QueryProperties.java @@ -3,10 +3,11 @@ package com.yahoo.search.query.properties; import com.yahoo.processing.request.CompoundName; import com.yahoo.search.Query; +import com.yahoo.search.grouping.GroupingRequest; +import com.yahoo.search.grouping.vespa.GroupingExecutor; import com.yahoo.search.query.*; import com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry; import com.yahoo.search.query.profile.types.FieldDescription; -import com.yahoo.search.query.profile.types.QueryProfileFieldType; import com.yahoo.search.query.profile.types.QueryProfileType; import com.yahoo.search.query.ranking.Diversity; import com.yahoo.search.query.ranking.MatchPhase; @@ -14,11 +15,11 @@ import com.yahoo.search.query.ranking.Matching; import com.yahoo.search.query.ranking.SoftTimeout; import com.yahoo.tensor.Tensor; -import java.util.ArrayList; -import java.util.Collections; import java.util.List; import java.util.Map; + + /** * Maps between the query model and text properties. * This can be done simpler by using reflection but the performance penalty was not worth it, @@ -139,7 +140,9 @@ public class QueryProperties extends Properties { if (key.toString().equals(Model.MODEL)) return query.getModel(); if (key.toString().equals(Ranking.RANKING)) return query.getRanking(); if (key.toString().equals(Presentation.PRESENTATION)) return query.getPresentation(); + } + return super.get(key, context, substitution); } @@ -253,9 +256,9 @@ public class QueryProperties extends Properties { } else if (key.size()==2 && key.first().equals(Select.SELECT)) { if (key.last().equals(Select.WHERE)){ - query.getSelect().setWhere(asString(value, "")); + query.getSelect().setWhereString(asString(value, "")); } else if (key.last().equals(Select.GROUPING)) { - query.getSelect().setGrouping(asString(value, "")); + query.getSelect().setGroupingString(asString(value, "")); } } else if (key.first().equals("rankfeature") || key.first().equals("featureoverride") ) { // featureoverride is deprecated @@ -277,8 +280,7 @@ public class QueryProperties extends Properties { query.setGroupingSessionCache(asBoolean(value, false)); else super.set(key,value,context); - } - else + } else super.set(key,value,context); } catch (Exception e) { // Make sure error messages are informative. This should be moved out of this properties implementation diff --git a/container-search/src/main/java/com/yahoo/search/querytransform/SortingDegrader.java b/container-search/src/main/java/com/yahoo/search/querytransform/SortingDegrader.java index 12a775e0a62..58eafef79de 100644 --- a/container-search/src/main/java/com/yahoo/search/querytransform/SortingDegrader.java +++ b/container-search/src/main/java/com/yahoo/search/querytransform/SortingDegrader.java @@ -37,7 +37,6 @@ import java.util.Set; // We are checking if there is a grouping expression, not if there is a raw grouping instruction property, // so we must run after the property is transferred to a grouping expression @After(GroupingQueryParser.SELECT_PARAMETER_PARSING) - public class SortingDegrader extends Searcher { /** Set this to false in query.properties to turn off degrading. Default: on */ @@ -56,7 +55,7 @@ public class SortingDegrader extends Searcher { private boolean shouldBeDegraded(Query query, IndexFacts.Session indexFacts) { if (query.getRanking().getSorting() == null) return false; if (query.getRanking().getSorting().fieldOrders().isEmpty()) return false; - if ( ! GroupingRequest.getRequests(query).isEmpty()) return false; + if ( ! query.getSelect().getGrouping().isEmpty()) return false; if ( ! query.properties().getBoolean(DEGRADING, true)) return false; Index index = indexFacts.getIndex(query.getRanking().getSorting().fieldOrders().get(0).getFieldName()); diff --git a/container-search/src/main/java/com/yahoo/search/yql/VespaSerializer.java b/container-search/src/main/java/com/yahoo/search/yql/VespaSerializer.java index 075df0413b7..ac1c2ee4a6c 100644 --- a/container-search/src/main/java/com/yahoo/search/yql/VespaSerializer.java +++ b/container-search/src/main/java/com/yahoo/search/yql/VespaSerializer.java @@ -1164,7 +1164,7 @@ public class VespaSerializer { public static String serialize(Query query) { StringBuilder out = new StringBuilder(); serialize(query.getModel().getQueryTree().getRoot(), out); - for (GroupingRequest request : GroupingRequest.getRequests(query)) { + for (GroupingRequest request : query.getSelect().getGrouping()) { out.append(" | "); serialize(request, out); } diff --git a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/FastSearcherTestCase.java b/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/FastSearcherTestCase.java index ec586e67763..c9773008e05 100644 --- a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/FastSearcherTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/FastSearcherTestCase.java @@ -491,7 +491,7 @@ public class FastSearcherTestCase { } private void assertForceSinglePassIs(boolean expected, Query query) { - for (GroupingRequest request : GroupingRequest.getRequests(query)) + for (GroupingRequest request : query.getSelect().getGrouping()) assertForceSinglePassIs(expected, request.getRootOperation()); } diff --git a/container-search/src/test/java/com/yahoo/search/grouping/GroupingQueryParserTestCase.java b/container-search/src/test/java/com/yahoo/search/grouping/GroupingQueryParserTestCase.java index 8de704a8a0e..90459fe614f 100644 --- a/container-search/src/test/java/com/yahoo/search/grouping/GroupingQueryParserTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/grouping/GroupingQueryParserTestCase.java @@ -105,6 +105,6 @@ public class GroupingQueryParserTestCase { query.properties().set(GroupingQueryParser.PARAM_CONTINUE, continuation); query.properties().set(GroupingQueryParser.PARAM_TIMEZONE, timeZone); new Execution(new GroupingQueryParser(), Execution.Context.createContextStub()).search(query); - return GroupingRequest.getRequests(query); + return query.getSelect().getGrouping(); } } diff --git a/container-search/src/test/java/com/yahoo/search/grouping/GroupingRequestTestCase.java b/container-search/src/test/java/com/yahoo/search/grouping/GroupingRequestTestCase.java index 494602be7b3..8b1bbb6842e 100644 --- a/container-search/src/test/java/com/yahoo/search/grouping/GroupingRequestTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/grouping/GroupingRequestTestCase.java @@ -10,8 +10,10 @@ import com.yahoo.search.result.Hit; import org.junit.Test; import java.lang.reflect.Field; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.List; import static org.junit.Assert.*; @@ -26,7 +28,10 @@ public class GroupingRequestTestCase { assertTrue(req.continuations().isEmpty()); Continuation foo = new Continuation() { - + @Override + public Continuation copy() { + return null; + } }; req.continuations().add(foo); assertEquals(Arrays.asList(foo), req.continuations()); @@ -105,32 +110,22 @@ public class GroupingRequestTestCase { @Test public void requireThatGetRequestsReturnsAllRequests() { Query query = new Query(); - assertEquals(Collections.emptyList(), GroupingRequest.getRequests(query)); + assertEquals(Collections.emptyList(), query.getSelect().getGrouping()); GroupingRequest foo = GroupingRequest.newInstance(query); - assertEquals(Arrays.asList(foo), GroupingRequest.getRequests(query)); + assertEquals(Arrays.asList(foo), query.getSelect().getGrouping()); GroupingRequest bar = GroupingRequest.newInstance(query); - assertEquals(Arrays.asList(foo, bar), GroupingRequest.getRequests(query)); - } - - @Test - public void requireThatGetRequestThrowsIllegalArgumentOnBadProperty() throws Exception { - Query query = new Query(); - Field propName = GroupingRequest.class.getDeclaredField("PROP_REQUEST"); - propName.setAccessible(true); - query.properties().set((CompoundName)propName.get(null), new Object()); - try { - GroupingRequest.getRequests(query); - fail(); - } catch (IllegalArgumentException e) { - - } + assertEquals(Arrays.asList(foo, bar), query.getSelect().getGrouping()); } + private static RootGroup newRootGroup(int id) { return new RootGroup(id, new Continuation() { - + @Override + public Continuation copy() { + return null; + } }); } } diff --git a/container-search/src/test/java/com/yahoo/search/grouping/UniqueGroupingSearcherTestCase.java b/container-search/src/test/java/com/yahoo/search/grouping/UniqueGroupingSearcherTestCase.java index b570adc51c6..22e5196c0c6 100644 --- a/container-search/src/test/java/com/yahoo/search/grouping/UniqueGroupingSearcherTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/grouping/UniqueGroupingSearcherTestCase.java @@ -210,7 +210,7 @@ public class UniqueGroupingSearcherTestCase { Result result = new Result(query); if (addGroupingData) { result.hits().add(resultGroup); - GroupingRequest.getRequests(query).get(0).setResultGroup(resultGroup); + query.getSelect().getGrouping().get(0).setResultGroup(resultGroup); result.setTotalHitCount(totalHitCount); } return result; diff --git a/container-search/src/test/java/com/yahoo/search/grouping/result/GroupListTestCase.java b/container-search/src/test/java/com/yahoo/search/grouping/result/GroupListTestCase.java index 651321a3b69..b8cc25930e2 100644 --- a/container-search/src/test/java/com/yahoo/search/grouping/result/GroupListTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/grouping/result/GroupListTestCase.java @@ -31,5 +31,11 @@ public class GroupListTestCase { private static class MyContinuation extends Continuation { + @Override + public Continuation copy() { + return null; + } + } + } diff --git a/container-search/src/test/java/com/yahoo/search/grouping/result/HitListTestCase.java b/container-search/src/test/java/com/yahoo/search/grouping/result/HitListTestCase.java index 05a78b71cac..8203691e11e 100644 --- a/container-search/src/test/java/com/yahoo/search/grouping/result/HitListTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/grouping/result/HitListTestCase.java @@ -31,5 +31,11 @@ public class HitListTestCase { private static class MyContinuation extends Continuation { + @Override + public Continuation copy() { + return null; + } + } + } diff --git a/container-search/src/test/java/com/yahoo/search/grouping/result/HitRendererTestCase.java b/container-search/src/test/java/com/yahoo/search/grouping/result/HitRendererTestCase.java index 85b8ff7a6d5..657c169720e 100644 --- a/container-search/src/test/java/com/yahoo/search/grouping/result/HitRendererTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/grouping/result/HitRendererTestCase.java @@ -167,8 +167,15 @@ public class HitRendererTestCase { } @Override + public Continuation copy() { + return null; + } + + @Override public String toString() { return str; } + } + } diff --git a/container-search/src/test/java/com/yahoo/search/grouping/vespa/CompositeContinuationTestCase.java b/container-search/src/test/java/com/yahoo/search/grouping/vespa/CompositeContinuationTestCase.java index 6a6741ae1b4..b9887495940 100644 --- a/container-search/src/test/java/com/yahoo/search/grouping/vespa/CompositeContinuationTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/grouping/vespa/CompositeContinuationTestCase.java @@ -109,6 +109,11 @@ public class CompositeContinuationTestCase { private static class MyContinuation extends EncodableContinuation { @Override + public EncodableContinuation copy() { + return null; + } + + @Override public void encode(IntegerEncoder out) { } diff --git a/container-search/src/test/java/com/yahoo/search/grouping/vespa/GroupingTransformTestCase.java b/container-search/src/test/java/com/yahoo/search/grouping/vespa/GroupingTransformTestCase.java index 038df42bd28..161c3a24801 100644 --- a/container-search/src/test/java/com/yahoo/search/grouping/vespa/GroupingTransformTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/grouping/vespa/GroupingTransformTestCase.java @@ -162,7 +162,10 @@ public class GroupingTransformTestCase { GroupingTransform transform = newTransform(); try { transform.addContinuation(new Continuation() { - + @Override + public Continuation copy() { + return null; + } }); fail(); } catch (UnsupportedOperationException e) { diff --git a/container-search/src/test/java/com/yahoo/search/rendering/JsonRendererTestCase.java b/container-search/src/test/java/com/yahoo/search/rendering/JsonRendererTestCase.java index caad1c76362..726d5cb76be 100644 --- a/container-search/src/test/java/com/yahoo/search/rendering/JsonRendererTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/rendering/JsonRendererTestCase.java @@ -790,6 +790,11 @@ public class JsonRendererTestCase { Result r = newEmptyResult(); RootGroup rg = new RootGroup(0, new Continuation() { @Override + public Continuation copy() { + return null; + } + + @Override public String toString() { return "AAAA"; } @@ -797,12 +802,22 @@ public class JsonRendererTestCase { GroupList gl = new GroupList("customer"); gl.continuations().put("prev", new Continuation() { @Override + public Continuation copy() { + return null; + } + + @Override public String toString() { return "BBBB"; } }); gl.continuations().put("next", new Continuation() { @Override + public Continuation copy() { + return null; + } + + @Override public String toString() { return "CCCC"; } @@ -860,6 +875,11 @@ public class JsonRendererTestCase { Result r = newEmptyResult(); RootGroup rg = new RootGroup(0, new Continuation() { @Override + public Continuation copy() { + return null; + } + + @Override public String toString() { return "AAAA"; } diff --git a/container-search/src/test/java/com/yahoo/search/yql/MinimalQueryInserterTestCase.java b/container-search/src/test/java/com/yahoo/search/yql/MinimalQueryInserterTestCase.java index e5dbe10a0e4..b23e25e173e 100644 --- a/container-search/src/test/java/com/yahoo/search/yql/MinimalQueryInserterTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/yql/MinimalQueryInserterTestCase.java @@ -329,9 +329,8 @@ public class MinimalQueryInserterTestCase { private static void assertGrouping(String expected, Query query) { List<String> actual = new ArrayList<>(); - for (GroupingRequest request : GroupingRequest.getRequests(query)) { + for (GroupingRequest request : query.getSelect().getGrouping()) actual.add(request.continuations().toString() + request.getRootOperation()); - } assertEquals(expected, actual.toString()); } diff --git a/container-search/src/test/java/com/yahoo/select/SelectParserTestCase.java b/container-search/src/test/java/com/yahoo/select/SelectTestCase.java index 031ba386ad4..6871e26b96a 100644 --- a/container-search/src/test/java/com/yahoo/select/SelectParserTestCase.java +++ b/container-search/src/test/java/com/yahoo/select/SelectTestCase.java @@ -15,14 +15,14 @@ import com.yahoo.prelude.query.WeakAndItem; import com.yahoo.prelude.query.WordAlternativesItem; import com.yahoo.prelude.query.WordItem; import com.yahoo.search.Query; -import com.yahoo.search.federation.ProviderConfig; +import com.yahoo.search.grouping.GroupingRequest; +import com.yahoo.search.grouping.request.AllOperation; import com.yahoo.search.query.QueryTree; import com.yahoo.search.query.Select; import com.yahoo.search.query.SelectParser; import com.yahoo.search.query.parser.Parsable; import com.yahoo.search.query.parser.ParserEnvironment; import com.yahoo.search.yql.VespaGroupingStep; -import org.apache.http.client.utils.URIBuilder; import org.json.JSONException; import org.json.JSONObject; import org.junit.Test; @@ -33,24 +33,23 @@ import java.util.List; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotSame; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertSame; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; - /** - * Specification for the conversion of Select expressions to Vespa search queries. + * Tests Query.Select * * @author henrhoi + * @author bratseth */ - -public class SelectParserTestCase { +public class SelectTestCase { private final SelectParser parser = new SelectParser(new ParserEnvironment()); - - /** WHERE TESTS */ + //------------------------------------------------------------------- "where" tests @Test public void test_contains() throws Exception { @@ -136,7 +135,6 @@ public class SelectParserTestCase { "+title:madonna -title:saint"); } - @Test public void testLessThan() throws JSONException { JSONObject range_json = new JSONObject(); @@ -165,7 +163,6 @@ public class SelectParserTestCase { "price:>500"); } - @Test public void testLessThanOrEqual() throws JSONException { JSONObject range_json = new JSONObject(); @@ -326,7 +323,6 @@ public class SelectParserTestCase { assertEquals(3, origin.end); } - @Test public void testSameElement() { assertParse("{ \"contains\": [ \"baz\", {\"sameElement\" : [ { \"contains\" : [\"f1\", \"a\"] }, { \"contains\" : [\"f2\", \"b\"] } ]} ] }", @@ -420,7 +416,7 @@ public class SelectParserTestCase { WordItem first = (WordItem)root.getItem(0); WordItem second = (WordItem)root.getItem(1); WordItem third = (WordItem)root.getItem(2); - assertTrue(first.getConnectedItem() == third); + assertEquals(third, first.getConnectedItem()); assertEquals(first.getConnectivity(), 7.0d, 1E-6); assertNull(second.getConnectedItem()); @@ -557,17 +553,17 @@ public class SelectParserTestCase { @Test public void testAffixItems() { assertRootClass("{ \"contains\" : { \"children\" : [\"baz\", \"colors\"], \"attributes\" : {\"suffix\": true} } }", - SuffixItem.class); + SuffixItem.class); assertRootClass("{ \"contains\" : { \"children\" : [\"baz\", \"colors\"], \"attributes\" : {\"prefix\": true} } }", - PrefixItem.class); + PrefixItem.class); assertRootClass("{ \"contains\" : { \"children\" : [\"baz\", \"colors\"], \"attributes\" : {\"substring\": true} } }", - SubstringItem.class); + SubstringItem.class); assertParseFail("{ \"contains\" : { \"children\" : [\"baz\", \"colors\"], \"attributes\" : {\"suffix\": true, \"prefix\" : true} } }", - new IllegalArgumentException("Only one of prefix, substring and suffix can be set.")); + new IllegalArgumentException("Only one of prefix, substring and suffix can be set.")); assertParseFail("{ \"contains\" : { \"children\" : [\"baz\", \"colors\"], \"attributes\" : {\"suffix\": true, \"substring\" : true} } }", - new IllegalArgumentException("Only one of prefix, substring and suffix can be set.")); + new IllegalArgumentException("Only one of prefix, substring and suffix can be set.")); } @Test @@ -641,7 +637,7 @@ public class SelectParserTestCase { checkWordAlternativesContent(alternatives); } - /** GROUPING TESTS */ + //------------------------------------------------------------------- grouping tests @Test public void testGrouping(){ @@ -659,9 +655,7 @@ public class SelectParserTestCase { assertGrouping(expected, parseGrouping(grouping)); } - - - /** OTHER TESTS */ + //------------------------------------------------------------------- Other tests @Test public void testOverridingOtherQueryTree() { @@ -669,16 +663,15 @@ public class SelectParserTestCase { assertEquals("default:query", query.getModel().getQueryTree().toString()); assertEquals(Query.Type.ALL, query.getModel().getType()); - query.getSelect().setWhere("{\"contains\" : [\"default\", \"select\"] }"); + query.getSelect().setWhereString("{\"contains\" : [\"default\", \"select\"] }"); assertEquals("default:select", query.getModel().getQueryTree().toString()); assertEquals(Query.Type.SELECT, query.getModel().getType()); } - @Test public void testOverridingWhereQueryTree() { - Query query = new Query(); - query.getSelect().setWhere("{\"contains\" : [\"default\", \"select\"] }"); + Query query = new Query("?query=default:query"); + query.getSelect().setWhereString("{\"contains\" : [\"default\", \"select\"] }"); assertEquals("default:select", query.getModel().getQueryTree().toString()); assertEquals(Query.Type.SELECT, query.getModel().getType()); @@ -688,10 +681,44 @@ public class SelectParserTestCase { assertEquals(Query.Type.ALL, query.getModel().getType()); } + @Test + public void testProgrammaticAssignment() { + Query query = new Query(); + query.getSelect().setGroupingString("[ { \"all\" : { \"group\" : \"time.year(a)\", \"each\" : { \"output\" : \"count()\" } } } ]"); + assertEquals(1, query.getSelect().getGrouping().size()); + assertEquals("all(group(time.year(a)) each(output(count())))", query.getSelect().getGrouping().get(0).getRootOperation().toString()); + // Setting from string resets the grouping expression + query.getSelect().setGroupingString("[ { \"all\" : { \"group\" : \"time.dayofmonth(a)\", \"each\" : { \"output\" : \"count()\" } } } ]"); + assertEquals(1, query.getSelect().getGrouping().size()); + assertEquals("all(group(time.dayofmonth(a)) each(output(count())))", query.getSelect().getGrouping().get(0).getRootOperation().toString()); + } + @Test + public void testConstructionAndClone() { + Query query = new Query(); + query.getSelect().setWhereString("{\"contains\" : [\"default\", \"select\"] }"); + query.getSelect().setGroupingString("[ { \"all\" : { \"group\" : \"time.dayofmonth(a)\", \"each\" : { \"output\" : \"count()\" } } } ]"); + GroupingRequest secondRequest = GroupingRequest.newInstance(query); + assertEquals("default:select", query.getModel().getQueryTree().toString()); + assertEquals(2, query.getSelect().getGrouping().size()); + assertEquals("all(group(time.dayofmonth(a)) each(output(count())))", query.getSelect().getGrouping().get(0).toString()); + + Query clone = query.clone(); + assertNotSame(query.getSelect(), clone.getSelect()); + assertNotSame(query.getSelect().getGrouping(), clone.getSelect().getGrouping()); + assertNotSame(query.getSelect().getGrouping().get(0), clone.getSelect().getGrouping().get(0)); + assertNotSame(query.getSelect().getGrouping().get(1), clone.getSelect().getGrouping().get(1)); + assertEquals(query.getSelect().getWhereString(), clone.getSelect().getWhereString()); + assertEquals(query.getSelect().getGroupingString(), clone.getSelect().getGroupingString()); + assertEquals(query.getSelect().getGrouping().get(0).toString(), clone.getSelect().getGrouping().get(0).toString()); + assertEquals(query.getSelect().getGrouping().get(1).toString(), clone.getSelect().getGrouping().get(1).toString()); + + + } + + //------------------------------------------------------------------- Assert methods - /** Assert-methods */ private void assertParse(String where, String expectedQueryTree) { String queryTree = parseWhere(where).toString(); assertEquals(expectedQueryTree, queryTree); @@ -721,13 +748,10 @@ public class SelectParserTestCase { assertEquals(expected, actual.toString()); } - - - - /** Parse-methods*/ + //------------------------------------------------------------------- Parse methods private QueryTree parseWhere(String where) { - Select select = new Select(where, ""); + Select select = new Select(where, "", new Query()); return parser.parse(new Parsable().setSelect(select)); } @@ -737,17 +761,8 @@ public class SelectParserTestCase { return parser.getGroupingSteps(grouping); } - private QueryTree parse(String where, String grouping) { - Select select = new Select(where, grouping); + //------------------------------------------------------------------- Other methods - return parser.parse(new Parsable().setSelect(select)); - } - - - - - - /** Other methods */ private WordItem getRootWord(String yqlQuery) { Item root = parseWhere(yqlQuery).getRoot(); assertTrue(root instanceof WordItem); @@ -775,5 +790,4 @@ public class SelectParserTestCase { } } - } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java index 7b35eae7f2c..c2920bc08eb 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java @@ -391,7 +391,7 @@ public class DeploymentTrigger { private JobState jobStateOf(Application application, JobType jobType) { if (application.deploymentJobs().builtInternally()) { - Optional<RunStatus> run = controller.jobController().last(application.id(), jobType); + Optional<Run> run = controller.jobController().last(application.id(), jobType); return run.isPresent() && ! run.get().hasEnded() ? JobState.running : JobState.idle; } return buildService.stateOf(BuildJob.of(application.id(), diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DummyStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DummyStepRunner.java deleted file mode 100644 index 17b523c60bf..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DummyStepRunner.java +++ /dev/null @@ -1,12 +0,0 @@ -package com.yahoo.vespa.hosted.controller.deployment; - -import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; - -public class DummyStepRunner implements StepRunner { - - @Override - public Step.Status run(LockedStep step, RunId id) { - return Step.Status.succeeded; - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java index e96f88e94ca..c682356ee32 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java @@ -52,6 +52,11 @@ import static com.yahoo.vespa.hosted.controller.api.integration.configserver.Con import static com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException.ErrorCode.OUT_OF_CAPACITY; import static com.yahoo.vespa.hosted.controller.api.integration.configserver.Node.State.active; import static com.yahoo.vespa.hosted.controller.api.integration.configserver.Node.State.reserved; +import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.deploymentFailed; +import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.error; +import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.installationFailed; +import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.running; +import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.testFailure; import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.failed; import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.succeeded; import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.unfinished; @@ -89,7 +94,7 @@ public class InternalStepRunner implements StepRunner { } @Override - public Status run(LockedStep step, RunId id) { + public Optional<RunStatus> run(LockedStep step, RunId id) { ByteArrayLogger logger = ByteArrayLogger.of(id.application(), id.type(), step.get()); try { switch (step.get()) { @@ -109,14 +114,18 @@ public class InternalStepRunner implements StepRunner { } catch (RuntimeException e) { logger.log(INFO, "Unexpected exception running " + id, e); - return failed; + if (JobProfile.of(id.type()).alwaysRun().contains(step.get())) { + logger.log("Will keep trying, as this is a cleanup step."); + return Optional.empty(); + } + return Optional.of(error); } finally { controller.jobController().log(id, step.get(), logger.getLog()); } } - private Status deployInitialReal(RunId id, ByteArrayLogger logger) { + private Optional<RunStatus> deployInitialReal(RunId id, ByteArrayLogger logger) { Versions versions = controller.jobController().run(id).get().versions(); logger.log("Deploying platform version " + versions.sourcePlatform().orElse(versions.targetPlatform()) + @@ -125,14 +134,14 @@ public class InternalStepRunner implements StepRunner { return deployReal(id, true, logger); } - private Status deployReal(RunId id, ByteArrayLogger logger) { + private Optional<RunStatus> deployReal(RunId id, ByteArrayLogger logger) { Versions versions = controller.jobController().run(id).get().versions(); logger.log("Deploying platform version " + versions.targetPlatform() + " and application version " + versions.targetApplication().id() + " ..."); return deployReal(id, false, logger); } - private Status deployReal(RunId id, boolean setTheStage, ByteArrayLogger logger) { + private Optional<RunStatus> deployReal(RunId id, boolean setTheStage, ByteArrayLogger logger) { return deploy(id.application(), id.type(), () -> controller.applications().deploy(id.application(), @@ -145,7 +154,7 @@ public class InternalStepRunner implements StepRunner { logger); } - private Status deployTester(RunId id, ByteArrayLogger logger) { + private Optional<RunStatus> deployTester(RunId id, ByteArrayLogger logger) { // TODO jvenstad: Consider deploying old version of tester for initial staging feeding? logger.log("Deploying the tester container ..."); return deploy(testerOf(id.application()), @@ -160,7 +169,7 @@ public class InternalStepRunner implements StepRunner { logger); } - private Status deploy(ApplicationId id, JobType type, Supplier<ActivateResult> deployment, ByteArrayLogger logger) { + private Optional<RunStatus> deploy(ApplicationId id, JobType type, Supplier<ActivateResult> deployment, ByteArrayLogger logger) { try { PrepareResponse prepareResponse = deployment.get().prepareResponse(); if ( ! prepareResponse.configChangeActions.refeedActions.stream().allMatch(action -> action.allowed)) { @@ -179,7 +188,7 @@ public class InternalStepRunner implements StepRunner { prepareResponse.log.stream() .map(entry -> entry.message) .collect(Collectors.joining("\n"))); - return failed; + return Optional.of(deploymentFailed); } if (prepareResponse.configChangeActions.restartActions.isEmpty()) @@ -195,28 +204,28 @@ public class InternalStepRunner implements StepRunner { logger.log("Restarting services on host " + hostname.id() + "."); }); logger.log("Deployment successful."); - return succeeded; + return Optional.of(running); } catch (ConfigServerException e) { if ( e.getErrorCode() == OUT_OF_CAPACITY && type.isTest() || e.getErrorCode() == ACTIVATION_CONFLICT || e.getErrorCode() == APPLICATION_LOCK_FAILURE) { logger.log("Will retry, because of '" + e.getErrorCode() + "' deploying:\n" + e.getMessage()); - return unfinished; + return Optional.empty(); } throw e; } } - private Status installInitialReal(RunId id, ByteArrayLogger logger) { + private Optional<RunStatus> installInitialReal(RunId id, ByteArrayLogger logger) { return installReal(id, true, logger); } - private Status installReal(RunId id, ByteArrayLogger logger) { + private Optional<RunStatus> installReal(RunId id, ByteArrayLogger logger) { return installReal(id, false, logger); } - private Status installReal(RunId id, boolean setTheStage, ByteArrayLogger logger) { + private Optional<RunStatus> installReal(RunId id, boolean setTheStage, ByteArrayLogger logger) { Versions versions = controller.jobController().run(id).get().versions(); Version platform = setTheStage ? versions.sourcePlatform().orElse(versions.targetPlatform()) : versions.targetPlatform(); ApplicationVersion application = setTheStage ? versions.sourceApplication().orElse(versions.targetApplication()) : versions.targetApplication(); @@ -224,33 +233,33 @@ public class InternalStepRunner implements StepRunner { if (nodesConverged(id.application(), id.type(), platform, logger) && servicesConverged(id.application(), id.type())) { logger.log("Installation succeeded!"); - return succeeded; + return Optional.of(running); } if (timedOut(id.application(), id.type(), installationTimeout)) { logger.log(INFO, "Installation failed to complete within " + installationTimeout.toMinutes() + " minutes!"); - return failed; + return Optional.of(installationFailed); } logger.log("Installation not yet complete."); - return unfinished; + return Optional.empty(); } - private Status installTester(RunId id, ByteArrayLogger logger) { + private Optional<RunStatus> installTester(RunId id, ByteArrayLogger logger) { logger.log("Checking installation of tester container ..."); if (servicesConverged(testerOf(id.application()), id.type())) { logger.log("Tester container successfully installed!"); - return succeeded; + return Optional.of(running); } if (timedOut(id.application(), id.type(), installationTimeout)) { logger.log(WARNING, "Installation of tester failed to complete within " + installationTimeout.toMinutes() + " minutes of real deployment!"); - return failed; + return Optional.of(error); } logger.log("Installation of tester not yet complete."); - return unfinished; + return Optional.empty(); } private boolean nodesConverged(ApplicationId id, JobType type, Version target, ByteArrayLogger logger) { @@ -277,7 +286,7 @@ public class InternalStepRunner implements StepRunner { .orElse(false); } - private Status startTests(RunId id, ByteArrayLogger logger) { + private Optional<RunStatus> startTests(RunId id, ByteArrayLogger logger) { logger.log("Attempting to find endpoints ..."); Map<ZoneId, List<URI>> endpoints = deploymentEndpoints(id.application()); logger.log("Found endpoints:\n" + @@ -290,11 +299,11 @@ public class InternalStepRunner implements StepRunner { if ( ! endpoints.containsKey(id.type().zone(controller.system()))) { if (timedOut(id.application(), id.type(), endpointTimeout)) { logger.log(WARNING, "Endpoints failed to show up within " + endpointTimeout.toMinutes() + " minutes!"); - return failed; + return Optional.of(error); } logger.log("Endpoints for the deployment to test are not yet ready."); - return unfinished; + return Optional.empty(); } Optional<URI> testerEndpoint = testerEndpoint(id); @@ -303,60 +312,60 @@ public class InternalStepRunner implements StepRunner { testerCloud.startTests(testerEndpoint.get(), TesterCloud.Suite.of(id.type()), testConfig(id.application(), id.type().zone(controller.system()), controller.system(), endpoints)); - return succeeded; + return Optional.of(running); } if (timedOut(id.application(), id.type(), endpointTimeout)) { logger.log(WARNING, "Endpoint for tester failed to show up within " + endpointTimeout.toMinutes() + " minutes of real deployment!"); - return failed; + return Optional.of(error); } logger.log("Endpoints of tester container not yet available."); - return unfinished; + return Optional.empty(); } - private Status endTests(RunId id, ByteArrayLogger logger) { + private Optional<RunStatus> endTests(RunId id, ByteArrayLogger logger) { URI testerEndpoint = testerEndpoint(id) .orElseThrow(() -> new NoSuchElementException("Endpoint for tester vanished again before tests were complete!")); - Status status; + RunStatus status; switch (testerCloud.getStatus(testerEndpoint)) { case NOT_STARTED: throw new IllegalStateException("Tester reports tests not started, even though they should have!"); case RUNNING: logger.log("Tests still running ..."); - return unfinished; + return Optional.empty(); case FAILURE: logger.log("Tests failed."); - status = failed; break; + status = testFailure; break; case ERROR: logger.log(INFO, "Tester failed running its tests!"); - status = failed; break; + status = error; break; case SUCCESS: logger.log("Tests completed successfully."); - status = succeeded; break; + status = running; break; default: throw new AssertionError("Unknown status!"); } logger.log(new String(testerCloud.getLogs(testerEndpoint))); // TODO jvenstad: Replace with something less hopeless! - return status; + return Optional.of(status); } - private Status deactivateReal(RunId id, ByteArrayLogger logger) { + private Optional<RunStatus> deactivateReal(RunId id, ByteArrayLogger logger) { logger.log("Deactivating deployment of " + id.application() + " in " + id.type().zone(controller.system()) + " ..."); controller.applications().deactivate(id.application(), id.type().zone(controller.system())); - return succeeded; + return Optional.of(running); } - private Status deactivateTester(RunId id, ByteArrayLogger logger) { + private Optional<RunStatus> deactivateTester(RunId id, ByteArrayLogger logger) { logger.log("Deactivating tester of " + id.application() + " in " + id.type().zone(controller.system()) + " ..."); controller.jobController().deactivateTester(id.application(), id.type()); - return succeeded; + return Optional.of(running); } - private Status report(RunId id) { + private Optional<RunStatus> report(RunId id) { controller.jobController().active(id).ifPresent(run -> controller.applications().deploymentTrigger().notifyOfCompletion(report(run))); - return succeeded; + return Optional.of(running); } /** Returns the real application with the given id. */ @@ -370,7 +379,7 @@ public class InternalStepRunner implements StepRunner { } /** Returns a generated job report for the given run. */ - private DeploymentJobs.JobReport report(RunStatus run) { + private DeploymentJobs.JobReport report(Run run) { return new DeploymentJobs.JobReport(run.id().application(), run.id().type(), 1, diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java index c966492259f..2a8aff1b0d3 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java @@ -41,7 +41,7 @@ import static com.yahoo.vespa.hosted.controller.deployment.InternalStepRunner.te * The deployment jobs run tests using regular applications, but these tester application IDs are not to be used elsewhere. * * Jobs consist of sets of {@link Step}s, defined in {@link JobProfile}s. - * Each run is represented by a {@link RunStatus}, which holds the status of each step of the run, as well as + * Each run is represented by a {@link Run}, which holds the status of each step of the run, as well as * some other meta data. * * @author jonmv @@ -71,7 +71,7 @@ public class JobController { /** Returns the details currently logged for the given run, if known. */ public Optional<RunDetails> details(RunId id) { - RunStatus run = runs(id.application(), id.type()).get(id); + Run run = runs(id.application(), id.type()).get(id); if (run == null) return Optional.empty(); @@ -107,33 +107,33 @@ public class JobController { } /** Returns an immutable map of all known runs for the given application and job type. */ - public Map<RunId, RunStatus> runs(ApplicationId id, JobType type) { - Map<RunId, RunStatus> runs = curator.readHistoricRuns(id, type); + public Map<RunId, Run> runs(ApplicationId id, JobType type) { + Map<RunId, Run> runs = curator.readHistoricRuns(id, type); last(id, type).ifPresent(run -> runs.putIfAbsent(run.id(), run)); return ImmutableMap.copyOf(runs); } /** Returns the run with the given id, if it exists. */ - public Optional<RunStatus> run(RunId id) { + public Optional<Run> run(RunId id) { return runs(id.application(), id.type()).values().stream() .filter(run -> run.id().equals(id)) .findAny(); } /** Returns the last run of the given type, for the given application, if one has been run. */ - public Optional<RunStatus> last(ApplicationId id, JobType type) { + public Optional<Run> last(ApplicationId id, JobType type) { return curator.readLastRun(id, type); } /** Returns the run with the given id, provided it is still active. */ - public Optional<RunStatus> active(RunId id) { + public Optional<Run> active(RunId id) { return last(id.application(), id.type()) .filter(run -> ! run.hasEnded()) .filter(run -> run.id().equals(id)); } /** Returns a list of all active runs. */ - public List<RunStatus> active() { + public List<Run> active() { return copyOf(applications().stream() .flatMap(id -> Stream.of(JobType.values()) .map(type -> last(id, type)) @@ -143,14 +143,14 @@ public class JobController { } /** Changes the status of the given step, for the given run, provided it is still active. */ - public void update(RunId id, Step.Status status, LockedStep step) { + public void update(RunId id, RunStatus status, LockedStep step) { locked(id, run -> run.with(status, step)); } /** Changes the status of the given run to inactive, and stores it as a historic run. */ public void finish(RunId id) { locked(id, run -> { // Store the modified run after it has been written to the collection, in case the latter fails. - RunStatus finishedRun = run.finished(controller.clock().instant()); + Run finishedRun = run.finished(controller.clock().instant()); locked(id.application(), id.type(), runs -> runs.put(run.id(), finishedRun)); return finishedRun; }); @@ -198,12 +198,12 @@ public class JobController { throw new IllegalArgumentException(id + " is not built here!"); locked(id, type, __ -> { - Optional<RunStatus> last = last(id, type); + Optional<Run> last = last(id, type); if (last.flatMap(run -> active(run.id())).isPresent()) throw new IllegalStateException("Can not start " + type + " for " + id + "; it is already running!"); RunId newId = new RunId(id, type, last.map(run -> run.id().number()).orElse(0L) + 1); - curator.writeLastRun(RunStatus.initial(newId, versions, controller.clock().instant())); + curator.writeLastRun(Run.initial(newId, versions, controller.clock().instant())); }); }); } @@ -272,18 +272,18 @@ public class JobController { } /** Locks and modifies the list of historic runs for the given application and job type. */ - private void locked(ApplicationId id, JobType type, Consumer<Map<RunId, RunStatus>> modifications) { + private void locked(ApplicationId id, JobType type, Consumer<Map<RunId, Run>> modifications) { try (Lock __ = curator.lock(id, type)) { - Map<RunId, RunStatus> runs = curator.readHistoricRuns(id, type); + Map<RunId, Run> runs = curator.readHistoricRuns(id, type); modifications.accept(runs); curator.writeHistoricRuns(id, type, runs.values()); } } /** Locks and modifies the run with the given id, provided it is still active. */ - private void locked(RunId id, UnaryOperator<RunStatus> modifications) { + private void locked(RunId id, UnaryOperator<Run> modifications) { try (Lock __ = curator.lock(id.application(), id.type())) { - RunStatus run = active(id).orElseThrow(() -> new IllegalArgumentException(id + " is not an active run!")); + Run run = active(id).orElseThrow(() -> new IllegalArgumentException(id + " is not an active run!")); run = modifications.apply(run); curator.writeLastRun(run); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Run.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Run.java new file mode 100644 index 00000000000..0d7977c6a9d --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Run.java @@ -0,0 +1,175 @@ +package com.yahoo.vespa.hosted.controller.deployment; + +import com.google.common.collect.ImmutableList; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; + +import java.time.Instant; +import java.util.Collections; +import java.util.EnumMap; +import java.util.EnumSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.aborted; +import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.running; +import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.success; +import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.succeeded; +import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.unfinished; +import static java.util.Objects.requireNonNull; + +/** + * Immutable class containing status information for a deployment job run by a {@link JobController}. + * + * @author jonmv + */ +public class Run { + + private final RunId id; + private final Map<Step, Step.Status> steps; + private final Versions versions; + private final Instant start; + private final Optional<Instant> end; + private final RunStatus status; + + // For deserialisation only -- do not use! + public Run(RunId id, Map<Step, Step.Status> steps, Versions versions, + Instant start, Optional<Instant> end, RunStatus status) { + this.id = id; + this.steps = Collections.unmodifiableMap(new EnumMap<>(steps)); + this.versions = versions; + this.start = start; + this.end = end; + this.status = status; + } + + public static Run initial(RunId id, Versions versions, Instant now) { + EnumMap<Step, Step.Status> steps = new EnumMap<>(Step.class); + JobProfile.of(id.type()).steps().forEach(step -> steps.put(step, unfinished)); + return new Run(id, steps, requireNonNull(versions), requireNonNull(now), Optional.empty(), running); + } + + /** Returns a new Run with the new status, and with the status of the given, completed step set accordingly. */ + public Run with(RunStatus status, LockedStep step) { + if (hasEnded()) + throw new AssertionError("This run ended at " + end.get() + " -- it can't be further modified!"); + + if (steps.get(step.get()) != unfinished) + throw new AssertionError("Step '" + step.get() + "' can't be set to '" + status + "'" + + " -- it already completed with status '" + steps.get(step.get()) + "'!"); + + EnumMap<Step, Step.Status> steps = new EnumMap<>(this.steps); + steps.put(step.get(), Step.Status.of(status)); + return new Run(id, steps, versions, start, end, this.status == running ? status : this.status); // Keep first terminal status. + } + + public Run finished(Instant now) { + if (hasEnded()) + throw new AssertionError("This run ended at " + end.get() + " -- it can't be ended again!"); + + return new Run(id, new EnumMap<>(steps), versions, start, Optional.of(now), status == running ? success : status); + } + + public Run aborted() { + if (hasEnded()) + throw new AssertionError("This run ended at " + end.get() + " -- it can't be aborted now!"); + + return new Run(id, new EnumMap<>(steps), versions, start, end, aborted); + } + + /** Returns the id of this run. */ + public RunId id() { + return id; + } + + /** Returns an unmodifiable view of the status of all steps in this run. */ + public Map<Step, Step.Status> steps() { + return steps; + } + + public RunStatus status() { + return status; + } + + /** Returns the instant at which this run began. */ + public Instant start() { + return start; + } + + /** Returns the instant at which this run ended, if it has. */ + public Optional<Instant> end() { + return end; + } + + /** Returns whether the run has failed, and should switch to its run-always steps. */ + public boolean hasFailed() { + return status != running && status != success; + } + + /** Returns whether the run has ended, i.e., has become inactive, and can no longer be updated. */ + public boolean hasEnded() { + return end.isPresent(); + } + + /** Returns the target, and possibly source, versions for this run. */ + public Versions versions() { + return versions; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if ( ! (o instanceof Run)) return false; + + Run run = (Run) o; + + return id.equals(run.id); + } + + @Override + public int hashCode() { + return id.hashCode(); + } + + @Override + public String toString() { + return "RunStatus{" + + "id=" + id + + ", versions=" + versions + + ", start=" + start + + ", end=" + end + + ", status=" + status + + ", steps=" + steps + + '}'; + } + + /** Returns the list of steps to run for this job right now, depending on whether the job has failed. */ + public List<Step> readySteps() { + return hasFailed() ? forcedSteps() : normalSteps(); + } + + /** Returns the list of unfinished steps whose prerequisites have all succeeded. */ + private List<Step> normalSteps() { + return ImmutableList.copyOf(steps.entrySet().stream() + .filter(entry -> entry.getValue() == unfinished + && entry.getKey().prerequisites().stream() + .allMatch(step -> steps.get(step) == null + || steps.get(step) == succeeded)) + .map(Map.Entry::getKey) + .iterator()); + } + + /** Returns the list of not-yet-succeeded run-always steps whose run-always prerequisites have all succeeded. */ + private List<Step> forcedSteps() { + return ImmutableList.copyOf(steps.entrySet().stream() + .filter(entry -> entry.getValue() != succeeded + && JobProfile.of(id.type()).alwaysRun().contains(entry.getKey()) + && entry.getKey().prerequisites().stream() + .filter(JobProfile.of(id.type()).alwaysRun()::contains) + .allMatch(step -> steps.get(step) == null + || steps.get(step) == succeeded)) + .map(Map.Entry::getKey) + .iterator()); + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunResult.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunResult.java deleted file mode 100644 index c4aac48503f..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunResult.java +++ /dev/null @@ -1,31 +0,0 @@ -package com.yahoo.vespa.hosted.controller.deployment; - -/** - * Outcomes of jobs run by a {@link JobController}. - * - * @author jonmv - */ -public enum RunResult { - - /** Deployment of the real application was rejected due to missing capacity. */ - outOfCapacity, - - /** Deployment of the real application was rejected. */ - deploymentFailed, - - /** Installation of the real application timed out. */ - installationFailed, - - /** Real application was deployed, but the tester application was not. */ - testError, - - /** Real application was deployed, but the tests failed. */ - testFailure, - - /** Deployment and tests completed with great success! */ - success, - - /** Job completed abnormally, due to user intervention or unexpected system error. */ - aborted - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunStatus.java index 504d8cfc88c..f62e2ab42ce 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunStatus.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunStatus.java @@ -1,182 +1,34 @@ package com.yahoo.vespa.hosted.controller.deployment; -import com.google.common.collect.ImmutableList; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; - -import java.time.Instant; -import java.util.Collections; -import java.util.EnumMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; - -import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.failed; -import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.succeeded; -import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.unfinished; -import static java.util.Objects.requireNonNull; - /** - * Immutable class containing status information for a deployment job run by a {@link JobController}. + * Status of jobs run by a {@link JobController}. * * @author jonmv */ -public class RunStatus { - - private final RunId id; - private final Map<Step, Step.Status> steps; - private final Versions versions; - private final Instant start; - private final Optional<Instant> end; - private final boolean aborted; - - // For deserialisation only -- do not use! - public RunStatus(RunId id, Map<Step, Step.Status> steps, Versions versions, - Instant start, Optional<Instant> end, boolean aborted) { - this.id = id; - this.steps = Collections.unmodifiableMap(new EnumMap<>(steps)); - this.versions = versions; - this.start = start; - this.end = end; - this.aborted = aborted; - } - - public static RunStatus initial(RunId id, Versions versions, Instant now) { - EnumMap<Step, Step.Status> steps = new EnumMap<>(Step.class); - JobProfile.of(id.type()).steps().forEach(step -> steps.put(step, unfinished)); - return new RunStatus(id, steps, requireNonNull(versions), requireNonNull(now), Optional.empty(), false); - } - - public RunStatus with(Step.Status status, LockedStep step) { - if (hasEnded()) - throw new AssertionError("This step ended at " + end.get() + " -- it can't be further modified!"); - - EnumMap<Step, Step.Status> steps = new EnumMap<>(this.steps); - steps.put(step.get(), requireNonNull(status)); - return new RunStatus(id, steps, versions, start, end, aborted); - } - - public RunStatus finished(Instant now) { - if (hasEnded()) - throw new AssertionError("This step ended at " + end.get() + " -- it can't be ended again!"); - - return new RunStatus(id, new EnumMap<>(steps), versions, start, Optional.of(now), aborted); - } - - public RunStatus aborted() { - if (hasEnded()) - throw new AssertionError("This step ended at " + end.get() + " -- it can't be aborted now!"); - - return new RunStatus(id, new EnumMap<>(steps), versions, start, end, true); - } - - /** Returns the id of this run. */ - public RunId id() { - return id; - } - - /** Returns an unmodifiable view of the status of all steps in this run. - * TODO maybe reflect in the signature that the map is a EnumMap or at least behaves as a sorted map? - * */ - public Map<Step, Step.Status> steps() { - return steps; - } - - /** Returns the final result of this run, if it has ended. */ - public Optional<RunResult> result() { - - // No result of not finished yet - if ( ! hasEnded()) return Optional.empty(); - - // If any steps has failed - then we need to figure out what - for now return fixed error result - if (hasFailed()) return Optional.of(RunResult.testError); - - return Optional.of(RunResult.success); - } - - /** Returns the instant at which this run began. */ - public Instant start() { - return start; - } - - /** Returns the instant at which this run ended, if it has. */ - public Optional<Instant> end() { - return end; - } - - /** Returns whether the run has failed, and should switch to its run-always steps. */ - public boolean hasFailed() { - return aborted || steps.values().contains(failed); - } - - /** Returns whether the run has been forcefully aborted. */ - public boolean isAborted() { - return aborted; - } - - /** Returns whether the run has ended, i.e., has become inactive, and can no longer be updated. */ - public boolean hasEnded() { - return end.isPresent(); - } - - /** Returns the target, and possibly source, versions for this run. */ - public Versions versions() { - return versions; - } +public enum RunStatus { - @Override - public boolean equals(Object o) { - if (this == o) return true; - if ( ! (o instanceof RunStatus)) return false; + /** Run is still proceeding normally, i.e., without failures. */ + running, - RunStatus status = (RunStatus) o; + /** Deployment was rejected due to missing capacity. */ + outOfCapacity, - return id.equals(status.id); - } + /** Deployment of the real application was rejected. */ + deploymentFailed, - @Override - public int hashCode() { - return id.hashCode(); - } + /** Installation of the real application timed out. */ + installationFailed, - @Override - public String toString() { - return "RunStatus{" + - "id=" + id + - ", versions=" + versions + - ", start=" + start + - ", end=" + end + - ", aborted=" + aborted + - ", steps=" + steps + - '}'; - } + /** The verification tests failed. */ + testFailure, - /** Returns the list of steps to run for this job right now, depending on whether the job has failed. */ - public List<Step> readySteps() { - return hasFailed() ? forcedSteps() : normalSteps(); - } + /** An unexpected error occurred. */ + error, - /** Returns the list of unfinished steps whose prerequisites have all succeeded. */ - private List<Step> normalSteps() { - return ImmutableList.copyOf(steps.entrySet().stream() - .filter(entry -> entry.getValue() == unfinished - && entry.getKey().prerequisites().stream() - .allMatch(step -> steps.get(step) == null - || steps.get(step) == succeeded)) - .map(Map.Entry::getKey) - .iterator()); - } + /** Everything completed with great success! */ + success, - /** Returns the list of not-yet-succeeded run-always steps whose run-always prerequisites have all succeeded. */ - private List<Step> forcedSteps() { - return ImmutableList.copyOf(steps.entrySet().stream() - .filter(entry -> entry.getValue() != succeeded - && JobProfile.of(id.type()).alwaysRun().contains(entry.getKey()) - && entry.getKey().prerequisites().stream() - .filter(JobProfile.of(id.type()).alwaysRun()::contains) - .allMatch(step -> steps.get(step) == null - || steps.get(step) == succeeded)) - .map(Map.Entry::getKey) - .iterator()); - } + /** Run has been abandoned, due to user intervention or timeout. */ + aborted } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Step.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Step.java index f734dde9440..6ad57851846 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Step.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Step.java @@ -66,6 +66,7 @@ public enum Step { public List<Step> prerequisites() { return prerequisites; } + public enum Status { /** Step still has unsatisfied finish criteria -- it may not even have started. */ @@ -75,7 +76,16 @@ public enum Step { failed, /** Step succeeded and subsequent steps may now start. */ - succeeded + succeeded; + + public static Step.Status of(RunStatus status) { + switch (status) { + case success : + case aborted : throw new AssertionError("Unexpected run status '" + status + "'!"); + case running : return succeeded; + default : return failed; + } + } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/StepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/StepRunner.java index cf024064cc4..de1a84e6d8e 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/StepRunner.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/StepRunner.java @@ -1,11 +1,8 @@ package com.yahoo.vespa.hosted.controller.deployment; -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; -import com.yahoo.vespa.hosted.controller.deployment.LockedStep; -import com.yahoo.vespa.hosted.controller.deployment.RunStatus; -import com.yahoo.vespa.hosted.controller.deployment.Step; + +import java.util.Optional; /** * Advances a given job run by running the appropriate {@link Step}s, based on their current status. @@ -18,8 +15,8 @@ import com.yahoo.vespa.hosted.controller.deployment.Step; */ public interface StepRunner { - /** Attempts to run the given locked step in the given run, and returns its new status. */ - Step.Status run(LockedStep step, RunId id); + /** Attempts to run the given step in the given run, and returns the new status of the run, if the step completed. */ + Optional<RunStatus> run(LockedStep step, RunId id); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunner.java index dd89644b580..ba147ae623d 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunner.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunner.java @@ -4,6 +4,7 @@ import com.yahoo.log.LogLevel; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.deployment.JobController; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; +import com.yahoo.vespa.hosted.controller.deployment.Run; import com.yahoo.vespa.hosted.controller.deployment.RunStatus; import com.yahoo.vespa.hosted.controller.deployment.Step; import com.yahoo.vespa.hosted.controller.deployment.StepRunner; @@ -19,7 +20,7 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.logging.Logger; /** - * Advances the set of {@link RunStatus}es for a {@link JobController}. + * Advances the set of {@link Run}s for a {@link JobController}. * * @author jonmv */ @@ -63,17 +64,18 @@ public class JobRunner extends Maintainer { } /** Advances each of the ready steps for the given run, or marks it as finished, and stashes it. */ - void advance(RunStatus run) { + private void advance(Run run) { List<Step> steps = run.readySteps(); steps.forEach(step -> executors.execute(() -> advance(run.id(), step))); if (steps.isEmpty()) jobs.finish(run.id()); - else if (run.start().isBefore(controller().clock().instant().minus(jobTimeout))) + else if ( run.status() != RunStatus.aborted + && run.start().isBefore(controller().clock().instant().minus(jobTimeout))) jobs.abort(run.id()); } /** Attempts to advance the status of the given step, for the given run. */ - void advance(RunId id, Step step) { + private void advance(RunId id, Step step) { try { AtomicBoolean changed = new AtomicBoolean(false); jobs.locked(id.application(), id.type(), step, lockedStep -> { @@ -81,11 +83,10 @@ public class JobRunner extends Maintainer { if ( ! run.readySteps().contains(step)) return; // Someone may have updated the run status, making this step obsolete, so we bail out. - Step.Status status = runner.run(lockedStep, run.id()); - if (run.steps().get(step) != status) { + runner.run(lockedStep, run.id()).ifPresent(status -> { jobs.update(run.id(), status, lockedStep); changed.set(true); - } + }); }); }); if (changed.get()) diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java index 49e8d7498ab..2c7def575fd 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java @@ -16,7 +16,7 @@ import com.yahoo.vespa.curator.Lock; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; -import com.yahoo.vespa.hosted.controller.deployment.RunStatus; +import com.yahoo.vespa.hosted.controller.deployment.Run; import com.yahoo.vespa.hosted.controller.deployment.Step; import com.yahoo.vespa.hosted.controller.tenant.AthenzTenant; import com.yahoo.vespa.hosted.controller.tenant.Tenant; @@ -307,19 +307,19 @@ public class CuratorDb { // -------------- Job Runs ------------------------------------------------ - public void writeLastRun(RunStatus run) { + public void writeLastRun(Run run) { curator.set(lastRunPath(run.id().application(), run.id().type()), asJson(runSerializer.toSlime(run))); } - public void writeHistoricRuns(ApplicationId id, JobType type, Iterable<RunStatus> runs) { + public void writeHistoricRuns(ApplicationId id, JobType type, Iterable<Run> runs) { curator.set(jobPath(id, type), asJson(runSerializer.toSlime(runs))); } - public Optional<RunStatus> readLastRun(ApplicationId id, JobType type) { + public Optional<Run> readLastRun(ApplicationId id, JobType type) { return readSlime(lastRunPath(id, type)).map(runSerializer::runFromSlime); } - public Map<RunId, RunStatus> readHistoricRuns(ApplicationId id, JobType type) { + public Map<RunId, Run> readHistoricRuns(ApplicationId id, JobType type) { // TODO jvenstad: Add, somewhere, a retention filter based on age or count. return readSlime(jobPath(id, type)).map(runSerializer::runsFromSlime).orElse(new LinkedHashMap<>()); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java index 153f4e327a0..12d629f334d 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java @@ -11,6 +11,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; import com.yahoo.vespa.hosted.controller.application.ApplicationVersion; import com.yahoo.vespa.hosted.controller.application.SourceRevision; +import com.yahoo.vespa.hosted.controller.deployment.Run; import com.yahoo.vespa.hosted.controller.deployment.RunStatus; import com.yahoo.vespa.hosted.controller.deployment.Step; import com.yahoo.vespa.hosted.controller.deployment.Step.Status; @@ -22,6 +23,14 @@ import java.util.LinkedHashMap; import java.util.Map; import java.util.Optional; +import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.aborted; +import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.deploymentFailed; +import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.installationFailed; +import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.outOfCapacity; +import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.running; +import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.success; +import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.error; +import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.testFailure; import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.failed; import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.succeeded; import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.unfinished; @@ -50,7 +59,7 @@ public class RunSerializer { private static final String numberField = "number"; private static final String startField = "start"; private static final String endField = "end"; - private static final String abortedField = "aborted"; + private static final String statusField = "status"; private static final String versionsField = "versions"; private static final String platformVersionField = "platform"; private static final String repositoryField = "repository"; @@ -59,35 +68,35 @@ public class RunSerializer { private static final String buildField = "build"; private static final String sourceField = "source"; - RunStatus runFromSlime(Slime slime) { + Run runFromSlime(Slime slime) { return runFromSlime(slime.get()); } - Map<RunId, RunStatus> runsFromSlime(Slime slime) { - Map<RunId, RunStatus> runs = new LinkedHashMap<>(); + Map<RunId, Run> runsFromSlime(Slime slime) { + Map<RunId, Run> runs = new LinkedHashMap<>(); Inspector runArray = slime.get(); runArray.traverse((ArrayTraverser) (__, runObject) -> { - RunStatus run = runFromSlime(runObject); + Run run = runFromSlime(runObject); runs.put(run.id(), run); }); return runs; } - private RunStatus runFromSlime(Inspector runObject) { + private Run runFromSlime(Inspector runObject) { EnumMap<Step, Status> steps = new EnumMap<>(Step.class); runObject.field(stepsField).traverse((ObjectTraverser) (step, status) -> { - steps.put(stepOf(step), statusOf(status.asString())); + steps.put(stepOf(step), stepStatusOf(status.asString())); }); - return new RunStatus(new RunId(ApplicationId.fromSerializedForm(runObject.field(applicationField).asString()), - JobType.fromJobName(runObject.field(jobTypeField).asString()), - runObject.field(numberField).asLong()), - steps, - versionsFromSlime(runObject.field(versionsField)), - Instant.ofEpochMilli(runObject.field(startField).asLong()), - Optional.of(runObject.field(endField)) - .filter(Inspector::valid) - .map(end -> Instant.ofEpochMilli(end.asLong())), - runObject.field(abortedField).asBool()); + return new Run(new RunId(ApplicationId.fromSerializedForm(runObject.field(applicationField).asString()), + JobType.fromJobName(runObject.field(jobTypeField).asString()), + runObject.field(numberField).asLong()), + steps, + versionsFromSlime(runObject.field(versionsField)), + Instant.ofEpochMilli(runObject.field(startField).asLong()), + Optional.of(runObject.field(endField)) + .filter(Inspector::valid) + .map(end -> Instant.ofEpochMilli(end.asLong())), + runStatusOf(runObject.field(statusField).asString())); } private Versions versionsFromSlime(Inspector versionsObject) { @@ -109,26 +118,26 @@ public class RunSerializer { return new Versions(targetPlatformVersion, targetApplicationVersion, sourcePlatformVersion, sourceApplicationVersion); } - Slime toSlime(Iterable<RunStatus> runs) { + Slime toSlime(Iterable<Run> runs) { Slime slime = new Slime(); Cursor runArray = slime.setArray(); runs.forEach(run -> toSlime(run, runArray.addObject())); return slime; } - Slime toSlime(RunStatus run) { + Slime toSlime(Run run) { Slime slime = new Slime(); toSlime(run, slime.setObject()); return slime; } - private void toSlime(RunStatus run, Cursor runObject) { + private void toSlime(Run run, Cursor runObject) { runObject.setString(applicationField, run.id().application().serializedForm()); runObject.setString(jobTypeField, run.id().type().jobName()); runObject.setLong(numberField, run.id().number()); runObject.setLong(startField, run.start().toEpochMilli()); run.end().ifPresent(end -> runObject.setLong(endField, end.toEpochMilli())); - if (run.isAborted()) runObject.setBool(abortedField, true); + runObject.setString(statusField, valueOf(run.status())); Cursor stepsObject = runObject.setObject(stepsField); run.steps().forEach((step, status) -> stepsObject.setString(valueOf(step), valueOf(status))); @@ -138,7 +147,7 @@ public class RunSerializer { run.versions().sourcePlatform().ifPresent(sourcePlatformVersion -> { toSlime(sourcePlatformVersion, run.versions().sourceApplication() - .orElseThrow(() -> new IllegalArgumentException("Source versions must be both present or absent.")), + .orElseThrow(() -> new IllegalArgumentException("Source versions must be both present or absent.")), versionsObject.setObject(sourceField)); }); } @@ -146,12 +155,12 @@ public class RunSerializer { private void toSlime(Version platformVersion, ApplicationVersion applicationVersion, Cursor versionsObject) { versionsObject.setString(platformVersionField, platformVersion.toString()); SourceRevision targetSourceRevision = applicationVersion.source() - .orElseThrow(() -> new IllegalArgumentException("Source revision must be present in target application version.")); + .orElseThrow(() -> new IllegalArgumentException("Source revision must be present in target application version.")); versionsObject.setString(repositoryField, targetSourceRevision.repository()); versionsObject.setString(branchField, targetSourceRevision.branch()); versionsObject.setString(commitField, targetSourceRevision.commit()); versionsObject.setLong(buildField, applicationVersion.buildNumber() - .orElseThrow(() -> new IllegalArgumentException("Build number must be present in target application version."))); + .orElseThrow(() -> new IllegalArgumentException("Build number must be present in target application version."))); } static String valueOf(Step step) { @@ -168,7 +177,7 @@ public class RunSerializer { case endTests : return "endTests"; case report : return "report"; - default : throw new AssertionError("No value defined for '" + step + "'!"); + default: throw new AssertionError("No value defined for '" + step + "'!"); } } @@ -186,7 +195,7 @@ public class RunSerializer { case "endTests" : return endTests; case "report" : return report; - default : throw new IllegalArgumentException("No step defined by '" + step + "'!"); + default: throw new IllegalArgumentException("No step defined by '" + step + "'!"); } } @@ -196,17 +205,47 @@ public class RunSerializer { case failed : return "failed"; case succeeded : return "succeeded"; - default : throw new AssertionError("No value defined for '" + status + "'!"); + default: throw new AssertionError("No value defined for '" + status + "'!"); } } - static Status statusOf(String status) { + static Status stepStatusOf(String status) { switch (status) { case "unfinished" : return unfinished; case "failed" : return failed; case "succeeded" : return succeeded; - default : throw new IllegalArgumentException("No status defined by '" + status + "'!"); + default: throw new IllegalArgumentException("No status defined by '" + status + "'!"); + } + } + + static String valueOf(RunStatus status) { + switch (status) { + case running : return "running"; + case outOfCapacity : return "outOfCapacity"; + case deploymentFailed : return "deploymentFailed"; + case installationFailed : return "installationFailed"; + case testFailure : return "testFailure"; + case error : return "error"; + case success : return "success"; + case aborted : return "aborted"; + + default: throw new AssertionError("No value defined for '" + status + "'!"); + } + } + + static RunStatus runStatusOf(String status) { + switch (status) { + case "running" : return running; + case "outOfCapacity" : return outOfCapacity; + case "deploymentFailed" : return deploymentFailed; + case "installationFailed" : return installationFailed; + case "testFailure" : return testFailure; + case "error" : return error; + case "success" : return success; + case "aborted" : return aborted; + + default: throw new IllegalArgumentException("No run status defined by '" + status + "'!"); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index 34d2257c5f4..26df4efa3c1 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -65,7 +65,7 @@ import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics; import com.yahoo.vespa.hosted.controller.application.JobStatus; import com.yahoo.vespa.hosted.controller.application.SourceRevision; import com.yahoo.vespa.hosted.controller.deployment.DeploymentSteps; -import com.yahoo.vespa.hosted.controller.deployment.RunStatus; +import com.yahoo.vespa.hosted.controller.deployment.Run; import com.yahoo.vespa.hosted.controller.restapi.ErrorResponse; import com.yahoo.vespa.hosted.controller.restapi.MessageResponse; import com.yahoo.restapi.Path; @@ -175,7 +175,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job")) return JobControllerApiHandlerHelper.jobTypeResponse(jobTypes(path), latestRuns(path), request.getUri()); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job/{jobtype}")) - return JobControllerApiHandlerHelper.runStatusResponse(controller.jobController().runs(appIdFromPath(path), jobTypeFromPath(path)), request.getUri()); + return JobControllerApiHandlerHelper.runResponse(controller.jobController().runs(appIdFromPath(path), jobTypeFromPath(path)), request.getUri()); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job/{jobtype}/run/{number}")) return JobControllerApiHandlerHelper.runDetailsResponse(controller.jobController(), runIdFromPath(path)); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}")) return deployment(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); @@ -1235,8 +1235,8 @@ public class ApplicationApiHandler extends LoggingRequestHandler { return deploymentSteps.jobs(); } - private Map<JobType, RunStatus> latestRuns(Path path) { - Map<JobType, RunStatus> jobMap = new HashMap<>(); + private Map<JobType, Run> latestRuns(Path path) { + Map<JobType, Run> jobMap = new HashMap<>(); ApplicationId appId = appIdFromPath(path); controller.jobController().jobs(appId) .forEach(jobType -> jobMap.put(jobType, controller.jobController() diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java index 281e59ef19f..6b8b14cf9b0 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java @@ -11,7 +11,7 @@ import com.yahoo.vespa.hosted.controller.application.ApplicationVersion; import com.yahoo.vespa.hosted.controller.application.SourceRevision; import com.yahoo.vespa.hosted.controller.deployment.JobController; import com.yahoo.vespa.hosted.controller.deployment.RunDetails; -import com.yahoo.vespa.hosted.controller.deployment.RunStatus; +import com.yahoo.vespa.hosted.controller.deployment.Run; import com.yahoo.vespa.hosted.controller.deployment.Step; import com.yahoo.vespa.hosted.controller.restapi.SlimeJsonResponse; @@ -31,17 +31,17 @@ class JobControllerApiHandlerHelper { /** * @return Response with all job types that have recorded runs for the application _and_ the status for the last run of that type */ - static HttpResponse jobTypeResponse(List<JobType> sortedJobs, Map<JobType, RunStatus> lastStatus, URI baseUriForJobs) { + static HttpResponse jobTypeResponse(List<JobType> sortedJobs, Map<JobType, Run> lastRun, URI baseUriForJobs) { Slime slime = new Slime(); Cursor responseObject = slime.setObject(); Cursor jobArray = responseObject.setArray("jobs"); sortedJobs.forEach(jobType -> - jobTypeToSlime(jobArray.addObject(), jobType, Optional.ofNullable(lastStatus.get(jobType)), baseUriForJobs)); + jobTypeToSlime(jobArray.addObject(), jobType, Optional.ofNullable(lastRun.get(jobType)), baseUriForJobs)); return new SlimeJsonResponse(slime); } - private static void jobTypeToSlime(Cursor cursor, JobType jobType, Optional<RunStatus> runStatus, URI baseUriForJobs) { + private static void jobTypeToSlime(Cursor cursor, JobType jobType, Optional<Run> lastRun, URI baseUriForJobs) { Cursor jobObject = cursor.setObject(jobType.jobName()); // Url that are specific to the jobtype @@ -49,38 +49,38 @@ class JobControllerApiHandlerHelper { URI baseUriForJobType = baseUriForJobs.resolve(jobTypePath); jobObject.setString("url", baseUriForJobType.toString()); - // Add the last run status for the jobtype if present - runStatus.ifPresent(status -> { + // Add the last run for the jobtype if present + lastRun.ifPresent(run -> { Cursor lastObject = jobObject.setObject("last"); - runStatusToSlime(lastObject, status, baseUriForJobType); + runToSlime(lastObject, run, baseUriForJobType); }); } /** - * @return Response with the runstatuses for a specific jobtype + * @return Response with the runs for a specific jobtype */ - static HttpResponse runStatusResponse(Map<RunId, RunStatus> runStatuses, URI baseUriForJobType) { + static HttpResponse runResponse(Map<RunId, Run> runs, URI baseUriForJobType) { Slime slime = new Slime(); Cursor cursor = slime.setObject(); - runStatuses.forEach((runid, runstatus) -> runStatusToSlime(cursor.setObject(Long.toString(runid.number())), runstatus, baseUriForJobType)); + runs.forEach((runid, run) -> runToSlime(cursor.setObject(Long.toString(runid.number())), run, baseUriForJobType)); return new SlimeJsonResponse(slime); } - private static void runStatusToSlime(Cursor cursor, RunStatus runStatus, URI baseUriForJobType) { - runStatus.result().ifPresent(result -> cursor.setString("result", result.name())); - runStatus.end().ifPresent(instant -> cursor.setString("end", instant.toString())); + private static void runToSlime(Cursor cursor, Run run, URI baseUriForJobType) { + cursor.setString("status", run.status().name()); + run.end().ifPresent(instant -> cursor.setString("end", instant.toString())); Cursor stepsArray = cursor.setArray("steps"); - runStatus.steps().forEach((step, status) -> { + run.steps().forEach((step, status) -> { Cursor stepObject = stepsArray.addObject(); stepObject.setString(step.name(), status.name()); }); - cursor.setString("start", runStatus.start().toString()); - cursor.setLong("id", runStatus.id().number()); - String logsPath = baseUriForJobType.getPath() + "/run/" + runStatus.id().number(); + cursor.setString("start", run.start().toString()); + cursor.setLong("id", run.id().number()); + String logsPath = baseUriForJobType.getPath() + "/run/" + run.id().number(); cursor.setString("logs", baseUriForJobType.resolve(logsPath).toString()); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/ControllerAuthorizationFilter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/ControllerAuthorizationFilter.java index 803138e8b3b..7256c283ab7 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/ControllerAuthorizationFilter.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/ControllerAuthorizationFilter.java @@ -135,6 +135,7 @@ public class ControllerAuthorizationFilter extends CorsRequestFilterBase { private static boolean isTenantPipelineOperation(Path path, Method method) { if (isTenantAdminOperation(path, method)) return false; return path.matches("/application/v4/tenant/{tenant}/application/{application}/jobreport") || + path.matches("/application/v4/tenant/{tenant}/application/{application}/submit") || path.matches("/application/v4/tenant/{tenant}/application/{application}/promote") || path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/prod/{*}") || path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/test/{*}") || diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java index d1c9b9f9b7e..d9198bc216c 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java @@ -45,6 +45,7 @@ import java.util.stream.Stream; import static com.yahoo.log.LogLevel.DEBUG; import static com.yahoo.vespa.hosted.controller.deployment.InternalStepRunner.testerOf; +import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.aborted; import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.failed; import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.succeeded; import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.unfinished; @@ -170,12 +171,12 @@ public class InternalStepRunnerTest { /** Runs the whole of the given job, successfully. */ private void runJob(JobType type) { tester.readyJobTrigger().maintain(); - RunStatus run = jobs.active().stream() - .filter(r -> r.id().type() == type) - .findAny() - .orElseThrow(() -> new AssertionError(type + " is not among the active: " + jobs.active())); + Run run = jobs.active().stream() + .filter(r -> r.id().type() == type) + .findAny() + .orElseThrow(() -> new AssertionError(type + " is not among the active: " + jobs.active())); assertFalse(run.hasFailed()); - assertFalse(run.isAborted()); + assertFalse(run.status() == aborted); ZoneId zone = type.zone(tester.controller().system()); DeploymentId deployment = new DeploymentId(appId, zone); @@ -377,10 +378,10 @@ public class InternalStepRunnerTest { tester.readyJobTrigger().maintain(); } - RunStatus run = jobs.active().stream() - .filter(r -> r.id().type() == type) - .findAny() - .orElseThrow(() -> new AssertionError(type + " is not among the active: " + jobs.active())); + Run run = jobs.active().stream() + .filter(r -> r.id().type() == type) + .findAny() + .orElseThrow(() -> new AssertionError(type + " is not among the active: " + jobs.active())); return run.id(); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java index 1ec07025812..a77f0789314 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java @@ -8,6 +8,7 @@ import com.yahoo.vespa.hosted.controller.application.ApplicationVersion; import com.yahoo.vespa.hosted.controller.application.SourceRevision; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; import com.yahoo.vespa.hosted.controller.deployment.JobController; +import com.yahoo.vespa.hosted.controller.deployment.Run; import com.yahoo.vespa.hosted.controller.deployment.RunStatus; import com.yahoo.vespa.hosted.controller.deployment.Step; import com.yahoo.vespa.hosted.controller.deployment.Step.Status; @@ -36,6 +37,10 @@ import java.util.stream.Collectors; import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.stagingTest; import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.systemTest; +import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.aborted; +import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.error; +import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.running; +import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.testFailure; import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.failed; import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.succeeded; import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.unfinished; @@ -71,7 +76,7 @@ public class JobRunnerTest { DeploymentTester tester = new DeploymentTester(); JobController jobs = tester.controller().jobController(); // Fail the installation of the initial version of the real application in staging tests, and succeed everything else. - StepRunner stepRunner = (step, id) -> id.type() == stagingTest && step.get() == startTests? failed : succeeded; + StepRunner stepRunner = (step, id) -> id.type() == stagingTest && step.get() == startTests? Optional.of(error) : Optional.of(running); CountDownLatch latch = new CountDownLatch(19); // Number of steps that will run, below: all but endTests in staging and all 9 in system. JobRunner runner = new JobRunner(tester.controller(), Duration.ofDays(1), new JobControl(tester.controller().curator()), Executors.newFixedThreadPool(32), notifying(stepRunner, latch)); @@ -105,13 +110,13 @@ public class JobRunnerTest { public void stepLogic() { DeploymentTester tester = new DeploymentTester(); JobController jobs = tester.controller().jobController(); - Map<Step, Status> outcomes = new EnumMap<>(Step.class); + Map<Step, RunStatus> outcomes = new EnumMap<>(Step.class); JobRunner runner = new JobRunner(tester.controller(), Duration.ofDays(1), new JobControl(tester.controller().curator()), inThreadExecutor(), mappedRunner(outcomes)); ApplicationId id = tester.createApplication("real", "tenant", 1, 1L).id(); jobs.submit(id, versions.targetApplication().source().get(), new byte[0], new byte[0]); - Supplier<RunStatus> run = () -> jobs.last(id, systemTest).get(); + Supplier<Run> run = () -> jobs.last(id, systemTest).get(); jobs.start(id, systemTest, versions); RunId first = run.get().id(); @@ -121,32 +126,28 @@ public class JobRunnerTest { assertEquals(steps, run.get().steps()); assertEquals(Arrays.asList(deployReal, deployTester), run.get().readySteps()); - outcomes.put(deployReal, succeeded); + outcomes.put(deployReal, running); runner.maintain(); assertEquals(Arrays.asList(installReal, deployTester), run.get().readySteps()); - outcomes.put(installReal, succeeded); + outcomes.put(installReal, running); runner.maintain(); assertEquals(Arrays.asList(deployTester), run.get().readySteps()); - outcomes.put(deployTester, succeeded); + outcomes.put(deployTester, running); runner.maintain(); assertEquals(Arrays.asList(installTester), run.get().readySteps()); - outcomes.put(installTester, succeeded); + outcomes.put(installTester, running); runner.maintain(); assertEquals(Arrays.asList(startTests), run.get().readySteps()); - outcomes.put(startTests, succeeded); + outcomes.put(startTests, running); runner.maintain(); assertEquals(Arrays.asList(endTests), run.get().readySteps()); - outcomes.put(endTests, succeeded); - runner.maintain(); - assertEquals(Arrays.asList(deactivateReal, deactivateTester), run.get().readySteps()); - - // Failure deactivating real fails the run, but run-always steps continue. - outcomes.put(deactivateReal, failed); + // Failure ending tests fails the run, but run-always steps continue. + outcomes.put(endTests, testFailure); runner.maintain(); assertTrue(run.get().hasFailed()); assertEquals(Arrays.asList(deactivateReal, deactivateTester), run.get().readySteps()); @@ -156,24 +157,24 @@ public class JobRunnerTest { runner.maintain(); assertEquals(Arrays.asList(deactivateReal, deactivateTester), run.get().readySteps()); - outcomes.put(deactivateReal, succeeded); - outcomes.put(deactivateTester, succeeded); - outcomes.put(report, succeeded); + outcomes.put(deactivateReal, running); + outcomes.put(deactivateTester, running); + outcomes.put(report, running); runner.maintain(); assertTrue(run.get().hasFailed()); assertTrue(run.get().hasEnded()); - assertTrue(run.get().isAborted()); + assertTrue(run.get().status() == aborted); // A new run is attempted. jobs.start(id, systemTest, versions); assertEquals(first.number() + 1, run.get().id().number()); // Run fails on tester deployment -- remaining run-always steps succeed, and the run finishes. - outcomes.put(deployTester, failed); + outcomes.put(deployTester, error); runner.maintain(); assertTrue(run.get().hasEnded()); assertTrue(run.get().hasFailed()); - assertFalse(run.get().isAborted()); + assertFalse(run.get().status() == aborted); assertEquals(failed, run.get().steps().get(deployTester)); assertEquals(unfinished, run.get().steps().get(installTester)); assertEquals(succeeded, run.get().steps().get(report)); @@ -230,7 +231,7 @@ public class JobRunnerTest { public void timeout() { DeploymentTester tester = new DeploymentTester(); JobController jobs = tester.controller().jobController(); - Map<Step, Status> outcomes = new EnumMap<>(Step.class); + Map<Step, RunStatus> outcomes = new EnumMap<>(Step.class); JobRunner runner = new JobRunner(tester.controller(), Duration.ofDays(1), new JobControl(tester.controller().curator()), inThreadExecutor(), mappedRunner(outcomes)); @@ -240,7 +241,7 @@ public class JobRunnerTest { jobs.start(id, systemTest, versions); tester.clock().advance(JobRunner.jobTimeout.plus(Duration.ofSeconds(1))); runner.run(); - assertTrue(jobs.last(id, systemTest).get().isAborted()); + assertTrue(jobs.last(id, systemTest).get().status() == aborted); } public static ExecutorService inThreadExecutor() { @@ -257,7 +258,7 @@ public class JobRunnerTest { private static StepRunner notifying(StepRunner runner, CountDownLatch latch) { return (step, id) -> { - Status status = runner.run(step, id); + Optional<RunStatus> status = runner.run(step, id); synchronized (latch) { assertTrue(latch.getCount() > 0); latch.countDown(); @@ -266,8 +267,8 @@ public class JobRunnerTest { }; } - private static StepRunner mappedRunner(Map<Step, Status> outcomes) { - return (step, id) -> outcomes.getOrDefault(step.get(), Status.unfinished); + private static StepRunner mappedRunner(Map<Step, RunStatus> outcomes) { + return (step, id) -> Optional.ofNullable(outcomes.get(step.get())); } private static StepRunner waitingRunner(CyclicBarrier barrier) { @@ -282,7 +283,7 @@ public class JobRunnerTest { catch (InterruptedException | BrokenBarrierException e) { throw new AssertionError(e); } - return succeeded; + return Optional.of(running); }; } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializerTest.java index 9419634392e..1085c2a1b70 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializerTest.java @@ -5,6 +5,7 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.vespa.config.SlimeUtils; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; +import com.yahoo.vespa.hosted.controller.deployment.Run; import com.yahoo.vespa.hosted.controller.deployment.RunStatus; import com.yahoo.vespa.hosted.controller.deployment.Step; import org.junit.Test; @@ -16,6 +17,8 @@ import java.nio.file.Paths; import java.time.Instant; import java.util.Collections; +import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.aborted; +import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.running; import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.failed; import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.succeeded; import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.unfinished; @@ -49,17 +52,20 @@ public class RunSerializerTest { assertEquals(step, RunSerializer.stepOf(RunSerializer.valueOf(step))); for (Step.Status status : Step.Status.values()) - assertEquals(status, RunSerializer.statusOf(RunSerializer.valueOf(status))); + assertEquals(status, RunSerializer.stepStatusOf(RunSerializer.valueOf(status))); + + for (RunStatus status : RunStatus.values()) + assertEquals(status, RunSerializer.runStatusOf(RunSerializer.valueOf(status))); // The purpose of this serialised data is to ensure a new format does not break everything, so keep it up to date! - RunStatus run = serializer.runsFromSlime(SlimeUtils.jsonToSlime(Files.readAllBytes(runFile))).get(id); + Run run = serializer.runsFromSlime(SlimeUtils.jsonToSlime(Files.readAllBytes(runFile))).get(id); for (Step step : Step.values()) assertTrue(run.steps().containsKey(step)); assertEquals(id, run.id()); assertEquals(start, run.start()); assertFalse(run.hasEnded()); - assertFalse(run.isAborted()); + assertEquals(running, run.status()); assertEquals(ImmutableMap.<Step, Step.Status>builder() .put(deployInitialReal, unfinished) .put(installInitialReal, failed) @@ -76,14 +82,14 @@ public class RunSerializerTest { run.steps()); run = run.aborted().finished(Instant.now()); - assertTrue(run.isAborted()); + assertEquals(aborted, run.status()); assertTrue(run.hasEnded()); - RunStatus phoenix = serializer.runsFromSlime(serializer.toSlime(Collections.singleton(run))).get(id); + Run phoenix = serializer.runsFromSlime(serializer.toSlime(Collections.singleton(run))).get(id); assertEquals(run.id(), phoenix.id()); assertEquals(run.start(), phoenix.start()); assertEquals(run.end(), phoenix.end()); - assertEquals(run.isAborted(), phoenix.isAborted()); + assertEquals(run.status(), phoenix.status()); assertEquals(run.steps(), phoenix.steps()); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/run-status.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/run-status.json index 9b3de20acc4..31e57902d22 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/run-status.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/run-status.json @@ -4,6 +4,7 @@ "type": "production-us-east-3", "number": 112358, "start": 1196676930000, + "status": "running", "steps": { "deployInitialReal": "unfinished", "installInitialReal": "failed", diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java index e4576c20730..bef2d7d7750 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java @@ -11,6 +11,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.stubs.MockLogStore; import com.yahoo.vespa.hosted.controller.application.ApplicationVersion; import com.yahoo.vespa.hosted.controller.application.SourceRevision; import com.yahoo.vespa.hosted.controller.deployment.JobController; +import com.yahoo.vespa.hosted.controller.deployment.Run; import com.yahoo.vespa.hosted.controller.deployment.RunStatus; import com.yahoo.vespa.hosted.controller.deployment.Step; import com.yahoo.vespa.hosted.controller.deployment.Versions; @@ -51,13 +52,13 @@ public class JobControllerApiHandlerHelperTest { @Test public void jobTypeResponse() { - Map<JobType, RunStatus> jobMap = new HashMap<>(); + Map<JobType, Run> jobMap = new HashMap<>(); List<JobType> jobList = new ArrayList<>(); - jobMap.put(JobType.systemTest, createStatus(JobType.systemTest, 1, 30, lastStep, Step.Status.succeeded)); + jobMap.put(JobType.systemTest, createRun(JobType.systemTest, 1, 30, lastStep, Optional.of(RunStatus.running))); jobList.add(JobType.systemTest); - jobMap.put(JobType.productionApNortheast1, createStatus(JobType.productionApNortheast1, 1, 60, lastStep, Step.Status.succeeded)); + jobMap.put(JobType.productionApNortheast1, createRun(JobType.productionApNortheast1, 1, 60, lastStep, Optional.of(RunStatus.running))); jobList.add(JobType.productionApNortheast1); - jobMap.put(JobType.productionUsWest1, createStatus(JobType.productionUsWest1, 1, 60, Step.startTests, Step.Status.failed)); + jobMap.put(JobType.productionUsWest1, createRun(JobType.productionUsWest1, 1, 60, Step.startTests, Optional.of(RunStatus.error))); jobList.add(JobType.productionUsWest1); URI jobUrl = URI.create("https://domain.tld/application/v4/tenant/sometenant/application/someapp/instance/usuallydefault/job"); @@ -67,22 +68,22 @@ public class JobControllerApiHandlerHelperTest { } @Test - public void runStatusResponse() { - Map<RunId, RunStatus> statusMap = new HashMap<>(); - RunStatus status; + public void runResponse() { + Map<RunId, Run> runs = new HashMap<>(); + Run run; - status = createStatus(JobType.systemTest, 3, 30, lastStep, Step.Status.succeeded); - statusMap.put(status.id(), status); + run = createRun(JobType.systemTest, 3, 30, lastStep, Optional.of(RunStatus.running)); + runs.put(run.id(), run); - status = createStatus(JobType.systemTest, 2, 56, Step.installReal, Step.Status.failed); - statusMap.put(status.id(), status); + run = createRun(JobType.systemTest, 2, 56, Step.installReal, Optional.of(RunStatus.error)); + runs.put(run.id(), run); - status = createStatus(JobType.systemTest, 1, 44, lastStep, Step.Status.succeeded); - statusMap.put(status.id(), status); + run = createRun(JobType.systemTest, 1, 44, lastStep, Optional.of(RunStatus.running)); + runs.put(run.id(), run); URI jobTypeUrl = URI.create("https://domain.tld/application/v4/tenant/sometenant/application/someapp/instance/usuallydefault/job/systemtest"); - HttpResponse response = JobControllerApiHandlerHelper.runStatusResponse(statusMap, jobTypeUrl); + HttpResponse response = JobControllerApiHandlerHelper.runResponse(runs, jobTypeUrl); assertFile(response, "job/run-status-response.json"); } @@ -95,7 +96,7 @@ public class JobControllerApiHandlerHelperTest { tester.curator().writeHistoricRuns( runId.application(), runId.type(), - Collections.singleton(createStatus(JobType.systemTest, 42, 44, lastStep, Step.Status.succeeded))); + Collections.singleton(createRun(JobType.systemTest, 42, 44, lastStep, Optional.of(RunStatus.running)))); logStore.append(runId, Step.deployTester.name(), "INFO\t1234567890\tSUCCESS".getBytes()); logStore.append(runId, Step.installTester.name(), "INFO\t1234598760\tSUCCESS".getBytes()); @@ -120,26 +121,29 @@ public class JobControllerApiHandlerHelperTest { } - private RunStatus createStatus(JobType type, long runid, long duration, Step lastStep, Step.Status lastStepStatus) { + private Run createRun(JobType type, long runid, long duration, Step lastStep, Optional<RunStatus> lastStepStatus) { RunId runId = new RunId(appId, type, runid); Map<Step, Step.Status> stepStatusMap = new HashMap<>(); for (Step step : Step.values()) { if (step.ordinal() < lastStep.ordinal()) { stepStatusMap.put(step, Step.Status.succeeded); - } else if (step.equals(lastStep)) { - stepStatusMap.put(step, lastStepStatus); + } else if (step.equals(lastStep) && lastStepStatus.isPresent()) { + stepStatusMap.put(step, Step.Status.of(lastStepStatus.get())); } else { stepStatusMap.put(step, Step.Status.unfinished); } } Optional<Instant> end = Optional.empty(); - if (lastStepStatus == Step.Status.failed || stepStatusMap.get(lastStep) == Step.Status.succeeded) { + if (lastStepStatus.isPresent() && lastStep == JobControllerApiHandlerHelperTest.lastStep) { end = Optional.of(start.plusSeconds(duration)); } - return new RunStatus(runId, stepStatusMap, versions, start, end, false); + RunStatus status = end.isPresent() && lastStepStatus.equals(Optional.of(RunStatus.running)) + ? RunStatus.success + : lastStepStatus.orElse(RunStatus.running); + return new Run(runId, stepStatusMap, versions, start, end, status); } private void compare(HttpResponse response, String expected) { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/ControllerAuthorizationFilterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/ControllerAuthorizationFilterTest.java index af11923b7f2..9951e4499f7 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/ControllerAuthorizationFilterTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/ControllerAuthorizationFilterTest.java @@ -121,6 +121,10 @@ public class ControllerAuthorizationFilterTest { testApiAccess(POST, "/application/v4/tenant/mytenant/application/myapp/promote", allowed, forbidden, filter); + + testApiAccess(POST, "/application/v4/tenant/mytenant/application/myapp/submit", + allowed, forbidden, filter); + } private static void testApiAccess(Method method, diff --git a/controller-server/src/test/resources/job/job-type-response.json b/controller-server/src/test/resources/job/job-type-response.json index 2c51f0d9275..7e7953486fa 100644 --- a/controller-server/src/test/resources/job/job-type-response.json +++ b/controller-server/src/test/resources/job/job-type-response.json @@ -3,7 +3,7 @@ { "system-test":{ "last":{ - "result":"success", + "status":"success", "start":"2018-06-27T10:12:35Z", "end":"2018-06-27T10:13:05Z", "id":1, @@ -50,7 +50,7 @@ { "production-ap-northeast-1":{ "last":{ - "result":"success", + "status":"success", "start":"2018-06-27T10:12:35Z", "end":"2018-06-27T10:13:35Z", "id":1, @@ -97,9 +97,8 @@ { "production-us-west-1":{ "last":{ - "result":"testError", + "status":"error", "start":"2018-06-27T10:12:35Z", - "end":"2018-06-27T10:13:35Z", "id":1, "steps":[ { diff --git a/controller-server/src/test/resources/job/run-status-response.json b/controller-server/src/test/resources/job/run-status-response.json index c3041440fd2..43d32efed8f 100644 --- a/controller-server/src/test/resources/job/run-status-response.json +++ b/controller-server/src/test/resources/job/run-status-response.json @@ -1,6 +1,6 @@ { "1":{ - "result":"success", + "status":"success", "start":"2018-06-27T10:12:35Z", "end":"2018-06-27T10:13:19Z", "id":1, @@ -42,9 +42,8 @@ "logs":"https://domain.tld/application/v4/tenant/sometenant/application/someapp/instance/usuallydefault/job/systemtest/run/1" }, "2":{ - "result":"testError", + "status":"error", "start":"2018-06-27T10:12:35Z", - "end":"2018-06-27T10:13:31Z", "id":2, "steps":[ { @@ -84,7 +83,7 @@ "logs":"https://domain.tld/application/v4/tenant/sometenant/application/someapp/instance/usuallydefault/job/systemtest/run/2" }, "3":{ - "result":"success", + "status":"success", "start":"2018-06-27T10:12:35Z", "end":"2018-06-27T10:13:05Z", "id":3, |