diff options
39 files changed, 1681 insertions, 138 deletions
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java index a69c524e863..5855dedc415 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java @@ -932,8 +932,8 @@ public class RankProfile implements Serializable, Cloneable { public static class Macro implements Serializable, Cloneable { private final String name; - private String textualExpression=null; - private RankingExpression expression=null; + private String textualExpression = null; + private RankingExpression expression = null; private List<String> formalParams = new ArrayList<>(); /** True if this should be inlined into calling expressions. Useful for very cheap macros. */ @@ -998,6 +998,7 @@ public class RankProfile implements Serializable, Cloneable { } public static final class DiversitySettings { + private String attribute = null; private int minGroups = 0; private double cutoffFactor = 10; diff --git a/config/src/main/java/com/yahoo/config/subscription/ConfigURI.java b/config/src/main/java/com/yahoo/config/subscription/ConfigURI.java index 537aff096a9..850dbcc1bf4 100644 --- a/config/src/main/java/com/yahoo/config/subscription/ConfigURI.java +++ b/config/src/main/java/com/yahoo/config/subscription/ConfigURI.java @@ -10,9 +10,9 @@ import com.yahoo.config.subscription.impl.JRTConfigRequester; * object to simplify parameter passing. * * @author lulf - * @since 5.1 */ public class ConfigURI { + private String configId; private ConfigSource source; @@ -54,4 +54,5 @@ public class ConfigURI { public static ConfigURI createFromIdAndSource(String configId, ConfigSource source) { return new ConfigURI(configId, source); } + } diff --git a/config/src/main/java/com/yahoo/config/subscription/FileSource.java b/config/src/main/java/com/yahoo/config/subscription/FileSource.java index d69e2429dbb..27129853eae 100644 --- a/config/src/main/java/com/yahoo/config/subscription/FileSource.java +++ b/config/src/main/java/com/yahoo/config/subscription/FileSource.java @@ -5,15 +5,15 @@ import java.io.File; /** * Source specifying config from one local file - * @author vegardh - * @since 5.1 * + * @author vegardh */ public class FileSource implements ConfigSource { + private final File file; public FileSource(File file) { - if (!file.isFile()) throw new IllegalArgumentException("Not an ordinary file: "+file); + if ( ! file.isFile()) throw new IllegalArgumentException("Not an ordinary file: "+file); this.file = file; } diff --git a/config/src/main/java/com/yahoo/vespa/config/ConfigFileFormat.java b/config/src/main/java/com/yahoo/vespa/config/ConfigFileFormat.java index da51df143d2..751f6578575 100644 --- a/config/src/main/java/com/yahoo/vespa/config/ConfigFileFormat.java +++ b/config/src/main/java/com/yahoo/vespa/config/ConfigFileFormat.java @@ -13,9 +13,9 @@ import java.util.Stack; /** * @author lulf - * @since 5.1 */ public class ConfigFileFormat implements SlimeFormat, ObjectTraverser { + private final InnerCNode root; private DataOutputStream out = null; private Stack<Node> nodeStack; diff --git a/container-core/src/main/java/com/yahoo/container/config/StatisticsRequestHandler.java b/container-core/src/main/java/com/yahoo/container/config/StatisticsRequestHandler.java index 1e5c22f4023..e0b6392b64a 100644 --- a/container-core/src/main/java/com/yahoo/container/config/StatisticsRequestHandler.java +++ b/container-core/src/main/java/com/yahoo/container/config/StatisticsRequestHandler.java @@ -20,8 +20,8 @@ import java.util.concurrent.Executor; * Handler of statistics http requests. Temporary hack as a step towards a more * general network interface. * - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> - * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + * @author Steinar Knutsen + * @author Einar M R Rosenvinge */ public class StatisticsRequestHandler extends ThreadedHttpRequestHandler { diff --git a/container-core/src/main/java/com/yahoo/container/core/config/HandlersConfigurerDi.java b/container-core/src/main/java/com/yahoo/container/core/config/HandlersConfigurerDi.java index e2c6da6fab8..55d7de90f33 100644 --- a/container-core/src/main/java/com/yahoo/container/core/config/HandlersConfigurerDi.java +++ b/container-core/src/main/java/com/yahoo/container/core/config/HandlersConfigurerDi.java @@ -86,11 +86,7 @@ public class HandlersConfigurerDi { osgiWrapper = new OsgiWrapper(osgiFramework, vespaContainer.getBundleLoader()); container = new Container(subscriberFactory, configId, deconstructor, osgiWrapper); - try { - getNewComponentGraph(discInjector, false); - } catch (InterruptedException e) { - throw new RuntimeException("Interrupted while setting up handlers for the first time."); - } + getNewComponentGraph(discInjector, false); } private static class OsgiWrapper extends OsgiImpl implements com.yahoo.container.di.Osgi { @@ -139,10 +135,10 @@ public class HandlersConfigurerDi { /** * Wait for new config to arrive and produce the new graph */ - public void getNewComponentGraph(Injector discInjector, boolean restartOnRedeploy) throws InterruptedException { - currentGraph = container.getNewComponentGraph(currentGraph, createFallbackInjector(vespaContainer, discInjector), restartOnRedeploy); - - assert (currentGraph.getInstance(RegistriesHack.class) != null); // TODO: Remove, seems quite pointless? + public void getNewComponentGraph(Injector discInjector, boolean restartOnRedeploy) { + currentGraph = container.getNewComponentGraph(currentGraph, + createFallbackInjector(vespaContainer, discInjector), + restartOnRedeploy); } @SuppressWarnings("deprecation") diff --git a/container-core/src/main/java/com/yahoo/container/core/config/testutil/HandlersConfigurerTestWrapper.java b/container-core/src/main/java/com/yahoo/container/core/config/testutil/HandlersConfigurerTestWrapper.java index 033a5819585..42780f75a6c 100644 --- a/container-core/src/main/java/com/yahoo/container/core/config/testutil/HandlersConfigurerTestWrapper.java +++ b/container-core/src/main/java/com/yahoo/container/core/config/testutil/HandlersConfigurerTestWrapper.java @@ -111,11 +111,7 @@ public class HandlersConfigurerTestWrapper { public void reloadConfig() { configurer.reloadConfig(++lastGeneration); - try { - configurer.getNewComponentGraph(Guice.createInjector(), false); - } catch (InterruptedException e) { - throw new RuntimeException(e); - } + configurer.getNewComponentGraph(Guice.createInjector(), false); } public void shutdown() { diff --git a/container-core/src/main/java/com/yahoo/container/handler/AccessLogRequestHandler.java b/container-core/src/main/java/com/yahoo/container/handler/AccessLogRequestHandler.java index 241ae269fc9..c50dee43eaf 100644 --- a/container-core/src/main/java/com/yahoo/container/handler/AccessLogRequestHandler.java +++ b/container-core/src/main/java/com/yahoo/container/handler/AccessLogRequestHandler.java @@ -22,6 +22,7 @@ import java.util.concurrent.Executor; * @author dybis */ public class AccessLogRequestHandler extends ThreadedHttpRequestHandler { + private final CircularArrayAccessLogKeeper circularArrayAccessLogKeeper; private final JsonFactory jsonFactory = new JsonFactory(); @@ -53,4 +54,5 @@ public class AccessLogRequestHandler extends ThreadedHttpRequestHandler { } }; } + } diff --git a/container-core/src/main/java/com/yahoo/container/handler/test/MockService.java b/container-core/src/main/java/com/yahoo/container/handler/test/MockService.java index 99d28b9bcf1..7bd18c519eb 100644 --- a/container-core/src/main/java/com/yahoo/container/handler/test/MockService.java +++ b/container-core/src/main/java/com/yahoo/container/handler/test/MockService.java @@ -39,12 +39,10 @@ import java.util.logging.Logger; * for descriptions of the format. * * @author lulf - * @since 5.1.21 */ @Beta public class MockService extends LoggingRequestHandler { - private final static Logger log = Logger.getLogger(MockService.class.getName()); private MockServiceHandler handler; /** diff --git a/container-di/src/main/java/com/yahoo/container/di/ConfigRetriever.java b/container-di/src/main/java/com/yahoo/container/di/ConfigRetriever.java index fe315c0eba5..337997f8ac2 100644 --- a/container-di/src/main/java/com/yahoo/container/di/ConfigRetriever.java +++ b/container-di/src/main/java/com/yahoo/container/di/ConfigRetriever.java @@ -47,13 +47,14 @@ public final class ConfigRetriever { /** * Loop forever until we get config */ - public ConfigSnapshot getConfigs(Set<ConfigKey<? extends ConfigInstance>> componentConfigKeys, long leastGeneration, - boolean restartOnRedeploy) { + public ConfigSnapshot getConfigs(Set<ConfigKey<? extends ConfigInstance>> componentConfigKeys, + long leastGeneration, + boolean restartOnRedeploy) { while (true) { - if (!Sets.intersection(componentConfigKeys, bootstrapKeys).isEmpty()) { - throw new IllegalArgumentException( - "Component config keys [" + componentConfigKeys + "] overlaps with bootstrap config keys [" + bootstrapKeys + "]"); - } + if (!Sets.intersection(componentConfigKeys, bootstrapKeys).isEmpty()) + throw new IllegalArgumentException("Component config keys [" + componentConfigKeys + + "] overlaps with bootstrap config keys [" + bootstrapKeys + "]"); + log.log(DEBUG, "getConfigs: " + componentConfigKeys); Set<ConfigKey<? extends ConfigInstance>> allKeys = new HashSet<>(componentConfigKeys); allKeys.addAll(bootstrapKeys); @@ -75,8 +76,9 @@ public final class ConfigRetriever { /** * Try to get config just once */ - public Optional<ConfigSnapshot> getConfigsOnce(Set<ConfigKey<? extends ConfigInstance>> componentConfigKeys, long leastGeneration, - boolean restartOnRedeploy) { + Optional<ConfigSnapshot> getConfigsOnce(Set<ConfigKey<? extends ConfigInstance>> componentConfigKeys, + long leastGeneration, + boolean restartOnRedeploy) { if (!Sets.intersection(componentConfigKeys, bootstrapKeys).isEmpty()) { throw new IllegalArgumentException( "Component config keys [" + componentConfigKeys + "] overlaps with bootstrap config keys [" + bootstrapKeys + "]"); @@ -114,7 +116,7 @@ public final class ConfigRetriever { } else { // This should not be a normal case, and hence a warning to allow investigation. log.warning("Did not get same generation for bootstrap (" + newestBootstrapGeneration + ") and components configs (" - + newestComponentGeneration + ")."); + + newestComponentGeneration + ")."); return Optional.empty(); } } diff --git a/container-di/src/main/java/com/yahoo/container/di/Container.java b/container-di/src/main/java/com/yahoo/container/di/Container.java index 7c58120d858..fb427bcf8ae 100644 --- a/container-di/src/main/java/com/yahoo/container/di/Container.java +++ b/container-di/src/main/java/com/yahoo/container/di/Container.java @@ -73,7 +73,6 @@ public class Container { } public ComponentGraph getNewComponentGraph(ComponentGraph oldGraph, Injector fallbackInjector, boolean restartOnRedeploy) { - try { ComponentGraph newGraph = getConfigAndCreateGraph(oldGraph, fallbackInjector, restartOnRedeploy); newGraph.reuseNodes(oldGraph); @@ -87,11 +86,11 @@ public class Container { } } - public ComponentGraph getNewComponentGraph(ComponentGraph oldGraph) { + ComponentGraph getNewComponentGraph(ComponentGraph oldGraph) { return getNewComponentGraph(oldGraph, Guice.createInjector(), false); } - public ComponentGraph getNewComponentGraph() { + ComponentGraph getNewComponentGraph() { return getNewComponentGraph(new ComponentGraph(), Guice.createInjector(), false); } @@ -125,10 +124,8 @@ public class Container { } } - public ComponentGraph getConfigAndCreateGraph(ComponentGraph graph, Injector fallbackInjector, boolean restartOnRedeploy) { - + private ComponentGraph getConfigAndCreateGraph(ComponentGraph graph, Injector fallbackInjector, boolean restartOnRedeploy) { ConfigSnapshot snapshot; - while (true) { snapshot = configurer.getConfigs(graph.configKeys(), leastGeneration, restartOnRedeploy); diff --git a/container-disc/src/main/java/com/yahoo/container/jdisc/ConfiguredApplication.java b/container-disc/src/main/java/com/yahoo/container/jdisc/ConfiguredApplication.java index 181d0018278..69a525a2c54 100644 --- a/container-disc/src/main/java/com/yahoo/container/jdisc/ConfiguredApplication.java +++ b/container-disc/src/main/java/com/yahoo/container/jdisc/ConfiguredApplication.java @@ -203,7 +203,7 @@ public final class ConfiguredApplication implements Application { // Block until new config arrives, and it should be applied configurer.getNewComponentGraph(builder.guiceModules().activate(), qrConfig.restartOnDeploy()); intitializeAndActivateContainer(builder); - } catch (ConfigInterruptedException | InterruptedException e) { + } catch (ConfigInterruptedException e) { break; } catch (Exception | LinkageError e) { // LinkageError: OSGi problems log.log(Level.SEVERE, @@ -256,13 +256,12 @@ public final class ConfiguredApplication implements Application { } private void configureComponents(Injector discInjector) { - configurer = new HandlersConfigurerDi( - subscriberFactory, - Container.get(), - configId, - new Deconstructor(true), - discInjector, - osgiFramework); + configurer = new HandlersConfigurerDi(subscriberFactory, + Container.get(), + configId, + new Deconstructor(true), + discInjector, + osgiFramework); } private void setupGuiceBindings(GuiceRepository modules) { diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastHit.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastHit.java index 3a2e922f9e0..f6894439ddc 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastHit.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastHit.java @@ -131,7 +131,7 @@ public class FastHit extends Hit { /** Returns the index of the node this hit originated at */ public int getDistributionKey() { return distributionKey; } - /** Returns the index of the node this hit originated at */ + /** Sets the index of the node this hit originated at */ public void setDistributionKey(int distributionKey) { this.distributionKey = distributionKey; } /** diff --git a/container-search/src/main/java/com/yahoo/search/Query.java b/container-search/src/main/java/com/yahoo/search/Query.java index dd3cc853742..cbc8b15eff9 100644 --- a/container-search/src/main/java/com/yahoo/search/Query.java +++ b/container-search/src/main/java/com/yahoo/search/Query.java @@ -511,11 +511,15 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { * Get the appropriate timeout for the query. * * @return timeout in milliseconds - **/ + */ public long getTimeLeft() { return getTimeout() - getDurationTime(); } + /** + * @deprecated do not use + */ + @Deprecated // TODO: Remove on Vespa 7 public boolean requestHasProperty(String name) { return httpRequest.hasProperty(name); } diff --git a/container-search/src/main/java/com/yahoo/search/query/Model.java b/container-search/src/main/java/com/yahoo/search/query/Model.java index dc7a61344cb..167bb312f61 100644 --- a/container-search/src/main/java/com/yahoo/search/query/Model.java +++ b/container-search/src/main/java/com/yahoo/search/query/Model.java @@ -78,11 +78,11 @@ public class Model implements Cloneable { private String defaultIndex = null; private Query.Type type = Query.Type.ALL; private Query parent; - private Set<String> sources=new LinkedHashSet<>(); - private Set<String> restrict=new LinkedHashSet<>(); + private Set<String> sources = new LinkedHashSet<>(); + private Set<String> restrict = new LinkedHashSet<>(); private String searchPath; private String documentDbName = null; - private Execution execution=new Execution(new Execution.Context(null, null, null, null, null)); + private Execution execution = new Execution(new Execution.Context(null, null, null, null, null)); public Model(Query query) { setParent(query); @@ -101,7 +101,7 @@ public class Model implements Cloneable { */ @Deprecated public void traceLanguage() { - if (getParent().getTraceLevel()<2) return; + if (getParent().getTraceLevel() < 2) return; if (language != null) { getParent().trace("Language " + getLanguage() + " specified directly as a parameter", false, 2); } diff --git a/document/src/main/java/com/yahoo/document/datatypes/Raw.java b/document/src/main/java/com/yahoo/document/datatypes/Raw.java index 2a5383705df..23ed0cee23e 100644 --- a/document/src/main/java/com/yahoo/document/datatypes/Raw.java +++ b/document/src/main/java/com/yahoo/document/datatypes/Raw.java @@ -17,15 +17,16 @@ import java.util.Arrays; /** * FieldValue which encapsulates a Raw value * - * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + * @author Einar M R Rosenvinge */ public final class Raw extends FieldValue { + private static class Factory extends PrimitiveDataType.Factory { public FieldValue create() { return new Raw(); } } - public static PrimitiveDataType.Factory getFactory() { return new Factory(); } + public static final int classId = registerClass(Ids.document + 16, Raw.class); private ByteBuffer value; @@ -42,6 +43,8 @@ public final class Raw extends FieldValue { value.position(0); } + public static PrimitiveDataType.Factory getFactory() { return new Factory(); } + public ByteBuffer getByteBuffer() { return value; } @@ -136,11 +139,11 @@ public final class Raw extends FieldValue { } /* (non-Javadoc) - * @see com.yahoo.document.datatypes.FieldValue#deserialize(com.yahoo.document.Field, com.yahoo.document.serialization.FieldReader) - */ - + * @see com.yahoo.document.datatypes.FieldValue#deserialize(com.yahoo.document.Field, com.yahoo.document.serialization.FieldReader) + */ @Override public void deserialize(Field field, FieldReader reader) { reader.read(field, this); } + } diff --git a/linguistics/src/main/java/com/yahoo/language/Linguistics.java b/linguistics/src/main/java/com/yahoo/language/Linguistics.java index 5e28213d524..035de415aa7 100644 --- a/linguistics/src/main/java/com/yahoo/language/Linguistics.java +++ b/linguistics/src/main/java/com/yahoo/language/Linguistics.java @@ -41,7 +41,12 @@ public interface Linguistics { CHARACTER_CLASSES } - /** The same as new com.yahoo.language.simple.SimpleLinguistics(). Prefer using that directly. */ + /** + * The same as new com.yahoo.language.simple.SimpleLinguistics(). Prefer using that directly. + * + * @deprecated use new com.yahoo.language.simple.SimpleLinguistics() + */ + @Deprecated // TODO: Remove this field on Vespa 7 Linguistics SIMPLE = new SimpleLinguistics(); /** diff --git a/model-inference/pom.xml b/model-inference/pom.xml new file mode 100644 index 00000000000..ad258d1edf4 --- /dev/null +++ b/model-inference/pom.xml @@ -0,0 +1,78 @@ +<?xml version="1.0"?> +<!-- Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<project xmlns="http://maven.apache.org/POM/4.0.0" + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 + http://maven.apache.org/xsd/maven-4.0.0.xsd"> + <modelVersion>4.0.0</modelVersion> + <parent> + <groupId>com.yahoo.vespa</groupId> + <artifactId>parent</artifactId> + <version>6-SNAPSHOT</version> + <relativePath>../parent/pom.xml</relativePath> + </parent> + <artifactId>model-inference</artifactId> + <version>6-SNAPSHOT</version> + <packaging>container-plugin</packaging> + <dependencies> + <dependency> + <groupId>junit</groupId> + <artifactId>junit</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>component</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>configdefinitions</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>vespajlib</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>searchlib</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>com.google.guava</groupId> + <artifactId>guava</artifactId> + <scope>provided</scope> + </dependency> + </dependencies> + <build> + <plugins> + <plugin> + <groupId>com.yahoo.vespa</groupId> + <artifactId>bundle-plugin</artifactId> + <extensions>true</extensions> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-jar-plugin</artifactId> + <configuration> + <archive> + <manifestEntries> + <Bundle-SymbolicName>${project.artifactId}</Bundle-SymbolicName> + <Vespa-Version>${project.version}</Vespa-Version> + </manifestEntries> + </archive> + </configuration> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-compiler-plugin</artifactId> + </plugin> + </plugins> + </build> +</project> diff --git a/model-inference/src/main/java/ai/vespa/models/evaluation/FunctionEvaluator.java b/model-inference/src/main/java/ai/vespa/models/evaluation/FunctionEvaluator.java new file mode 100644 index 00000000000..c4ecc1b25c9 --- /dev/null +++ b/model-inference/src/main/java/ai/vespa/models/evaluation/FunctionEvaluator.java @@ -0,0 +1,54 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package ai.vespa.models.evaluation; + +import com.yahoo.searchlib.rankingexpression.evaluation.TensorValue; +import com.yahoo.tensor.Tensor; +import com.yahoo.tensor.TensorType; + +/** + * An evaluator which can be used to evaluate a single function once. + * + * @author bratseth + */ +// This wraps all access to the context and the ranking expression to avoid incorrect usage +public class FunctionEvaluator { + + private final LazyArrayContext context; + private boolean evaluated = false; + + FunctionEvaluator(LazyArrayContext context) { + this.context = context; + } + + /** + * Binds the given variable referred in this expression to the given value. + * + * @param name the variable to bind + * @param value the value this becomes bound to + * @return this for chaining + */ + public FunctionEvaluator bind(String name, Tensor value) { + if (evaluated) + throw new IllegalStateException("You cannot bind a value in a used evaluator"); + context.put(name, new TensorValue(value)); + return this; + } + + /** + * Binds the given variable referred in this expression to the given value. + * This is equivalent to <code>bind(name, Tensor.Builder.of(TensorType.empty).cell(value).build())</code> + * + * @param name the variable to bind + * @param value the value this becomes bound to + * @return this for chaining + */ + public FunctionEvaluator bind(String name, double value) { + return bind(name, Tensor.Builder.of(TensorType.empty).cell(value).build()); + } + + public Tensor evaluate() { + evaluated = true; + return context.expression().evaluate(context).asTensor(); + } + +} diff --git a/model-inference/src/main/java/ai/vespa/models/evaluation/LazyArrayContext.java b/model-inference/src/main/java/ai/vespa/models/evaluation/LazyArrayContext.java new file mode 100644 index 00000000000..729d8af01dc --- /dev/null +++ b/model-inference/src/main/java/ai/vespa/models/evaluation/LazyArrayContext.java @@ -0,0 +1,213 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package ai.vespa.models.evaluation; + +import com.google.common.collect.ImmutableMap; +import com.yahoo.searchlib.rankingexpression.ExpressionFunction; +import com.yahoo.searchlib.rankingexpression.RankingExpression; +import com.yahoo.searchlib.rankingexpression.Reference; +import com.yahoo.searchlib.rankingexpression.evaluation.AbstractArrayContext; +import com.yahoo.searchlib.rankingexpression.evaluation.Context; +import com.yahoo.searchlib.rankingexpression.evaluation.ContextIndex; +import com.yahoo.searchlib.rankingexpression.evaluation.DoubleValue; +import com.yahoo.searchlib.rankingexpression.evaluation.Value; +import com.yahoo.searchlib.rankingexpression.rule.CompositeNode; +import com.yahoo.searchlib.rankingexpression.rule.ExpressionNode; +import com.yahoo.searchlib.rankingexpression.rule.ReferenceNode; +import com.yahoo.tensor.TensorType; + +import java.util.Arrays; +import java.util.LinkedHashSet; +import java.util.Map; +import java.util.Set; + +/** + * An array context supporting functions invocations implemented as lazy values. + * + * @author bratseth + */ +final class LazyArrayContext extends Context implements ContextIndex { + + private final RankingExpression expression; + private final IndexedBindings indexedBindings; + + private LazyArrayContext(RankingExpression expression, IndexedBindings indexedBindings) { + this.expression = expression; + this.indexedBindings = indexedBindings.copy(this); + } + + /** + * Create a fast lookup, lazy context for an expression. + * + * @param expression the expression to create a context for + */ + LazyArrayContext(RankingExpression expression, Map<String, ExpressionFunction> functions) { + this.expression = expression; + this.indexedBindings = new IndexedBindings(expression, functions, this); + } + + /** + * Puts a value by name. + * The value will be frozen if it isn't already. + * + * @throws IllegalArgumentException if the name is not present in the ranking expression this was created with, and + * ignoredUnknownValues is false + */ + @Override + public void put(String name, Value value) { + put(requireIndexOf(name), value); + } + + /** Same as put(index,DoubleValue.frozen(value)) */ + public final void put(int index, double value) { + put(index, DoubleValue.frozen(value)); + } + + /** + * Puts a value by index. + * The value will be frozen if it isn't already. + */ + public void put(int index, Value value) { + indexedBindings.set(index, value.freeze()); + } + + @Override + public TensorType getType(Reference reference) { + // TODO: Add type information so we do not need to evaluate to get this + return get(requireIndexOf(reference.toString())).type(); + } + + /** Perform a slow lookup by name */ + @Override + public Value get(String name) { + return get(requireIndexOf(name)); + } + + /** Perform a fast lookup by index */ + @Override + public Value get(int index) { + return indexedBindings.get(index); + } + + @Override + public double getDouble(int index) { + double value = get(index).asDouble(); + if (value == Double.NaN) + throw new UnsupportedOperationException("Value at " + index + " has no double representation"); + return value; + } + + @Override + public int getIndex(String name) { + return requireIndexOf(name); + } + + @Override + public int size() { + return indexedBindings.names().size(); + } + + @Override + public Set<String> names() { return indexedBindings.names(); } + + private Integer requireIndexOf(String name) { + Integer index = indexedBindings.indexOf(name); + if (index == null) + throw new IllegalArgumentException("Value '" + name + "' can not be bound in " + this); + return index; + } + + @Override + public String toString() { return "context of '" + expression.getName() + "'"; } + + RankingExpression expression() { return expression; } + + /** + * Creates a copy of this context suitable for evaluating against the same ranking expression + * in a different thread or for re-binding free variables. + */ + LazyArrayContext copy() { + return new LazyArrayContext(expression, indexedBindings); + } + + private static class IndexedBindings { + + /** The mapping from variable name to index */ + private final ImmutableMap<String, Integer> nameToIndex; + + /** The current values set, pre-converted to doubles */ + private final Value[] values; + + private IndexedBindings(ImmutableMap<String, Integer> nameToIndex, Value[] values) { + this.nameToIndex = nameToIndex; + this.values = values; + } + + IndexedBindings(RankingExpression expression, Map<String, ExpressionFunction> functions, LazyArrayContext owner) { + Set<String> bindTargets = new LinkedHashSet<>(); + extractBindTargets(expression.getRoot(), functions, bindTargets); + + values = new Value[bindTargets.size()]; + Arrays.fill(values, DoubleValue.zero); + + int i = 0; + ImmutableMap.Builder<String, Integer> nameToIndexBuilder = new ImmutableMap.Builder<>(); + for (String variable : bindTargets) + nameToIndexBuilder.put(variable,i++); + nameToIndex = nameToIndexBuilder.build(); + + for (Map.Entry<String, ExpressionFunction> function : functions.entrySet()) { + Integer index = nameToIndex.get(function.getKey()); + if (index != null) // Referenced in this, so bind it + values[index] = new LazyValue(function.getValue(), owner); + } + } + + private void extractBindTargets(ExpressionNode node, Map<String, ExpressionFunction> functions, Set<String> bindTargets) { + if (isFunctionReference(node)) { + String reference = node.toString(); + bindTargets.add(reference); + + extractBindTargets(functions.get(reference).getBody().getRoot(), functions, bindTargets); + } + else if (isConstant(node)) { + // Ignore + } + else if (node instanceof ReferenceNode) { + bindTargets.add(node.toString()); + } + else if (node instanceof CompositeNode) { + CompositeNode cNode = (CompositeNode)node; + for (ExpressionNode child : cNode.children()) + extractBindTargets(child, functions, bindTargets); + } + } + + private boolean isFunctionReference(ExpressionNode node) { + if ( ! (node instanceof ReferenceNode)) return false; + + ReferenceNode reference = (ReferenceNode)node; + return reference.getName().equals("rankingExpression") && reference.getArguments().size() == 1; + } + + private boolean isConstant(ExpressionNode node) { + if ( ! (node instanceof ReferenceNode)) return false; + + ReferenceNode reference = (ReferenceNode)node; + return reference.getName().equals("value") && reference.getArguments().size() == 1; + } + + Value get(int index) { return values[index]; } + void set(int index, Value value) { values[index] = value; } + Set<String> names() { return nameToIndex.keySet(); } + Integer indexOf(String name) { return nameToIndex.get(name); } + + IndexedBindings copy(Context context) { + Value[] valueCopy = new Value[values.length]; + for (int i = 0; i < values.length; i++) + valueCopy[i] = values[i] instanceof LazyValue ? ((LazyValue)values[i]).copyFor(context) : values[i]; + return new IndexedBindings(nameToIndex, valueCopy); + } + + } + +} diff --git a/model-inference/src/main/java/ai/vespa/models/evaluation/LazyValue.java b/model-inference/src/main/java/ai/vespa/models/evaluation/LazyValue.java new file mode 100644 index 00000000000..7e21e426962 --- /dev/null +++ b/model-inference/src/main/java/ai/vespa/models/evaluation/LazyValue.java @@ -0,0 +1,150 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package ai.vespa.models.evaluation; + +import com.yahoo.searchlib.rankingexpression.ExpressionFunction; +import com.yahoo.searchlib.rankingexpression.evaluation.Context; +import com.yahoo.searchlib.rankingexpression.evaluation.Value; +import com.yahoo.searchlib.rankingexpression.rule.Function; +import com.yahoo.searchlib.rankingexpression.rule.TruthOperator; +import com.yahoo.tensor.Tensor; +import com.yahoo.tensor.TensorType; + +/** + * A Value which is computed from an expression when first requested. + * This is not multithread safe. + * + * @author bratseth + */ +class LazyValue extends Value { + + /** The function computing the value of this */ + private final ExpressionFunction function; + + /** The context used to compute the function of this */ + private final Context context; + + private Value computedValue = null; + + public LazyValue(ExpressionFunction function, Context context) { + this.function = function; + this.context = context; + } + + private Value computedValue() { + if (computedValue == null) + computedValue = function.getBody().evaluate(context); + return computedValue; + } + + @Override + public TensorType type() { + return computedValue().type(); // TODO: Keep type information in this/ExpressionFunction to avoid computing here + } + + @Override + public double asDouble() { + return computedValue().asDouble(); + } + + @Override + public Tensor asTensor() { + return computedValue().asTensor(); + } + + @Override + public boolean hasDouble() { + return type().rank() == 0; + } + + @Override + public boolean asBoolean() { + return computedValue().asBoolean(); + } + + @Override + public Value negate() { + return computedValue().negate(); + } + + @Override + public Value add(Value value) { + return computedValue().add(value); + } + + @Override + public Value subtract(Value value) { + return computedValue().subtract(value); + } + + @Override + public Value multiply(Value value) { + return computedValue().multiply(value); + } + + @Override + public Value divide(Value value) { + return computedValue().divide(value); + } + + @Override + public Value modulo(Value value) { + return computedValue().modulo(value); + } + + @Override + public Value and(Value value) { + return computedValue().and(value); + } + + @Override + public Value or(Value value) { + return computedValue().or(value); + } + + @Override + public Value not() { + return computedValue().not(); + } + + @Override + public Value power(Value value) { + return computedValue().power(value); + } + + @Override + public Value compare(TruthOperator operator, Value value) { + return computedValue().compare(operator, value); + } + + @Override + public Value function(Function function, Value value) { + return computedValue().function(function, value); + } + + @Override + public Value asMutable() { + return computedValue().asMutable(); + } + + @Override + public String toString() { + return "value of " + function; + } + + @Override + public boolean equals(Object other) { + if (other == this) return true; + if (!(other instanceof Value)) return false; + return computedValue().equals(other); + } + + @Override + public int hashCode() { + return computedValue().hashCode(); + } + + LazyValue copyFor(Context context) { + return new LazyValue(this.function, context); + } + +} diff --git a/model-inference/src/main/java/ai/vespa/models/evaluation/Model.java b/model-inference/src/main/java/ai/vespa/models/evaluation/Model.java new file mode 100644 index 00000000000..9a639d0803f --- /dev/null +++ b/model-inference/src/main/java/ai/vespa/models/evaluation/Model.java @@ -0,0 +1,106 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package ai.vespa.models.evaluation; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.yahoo.searchlib.rankingexpression.ExpressionFunction; + +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +/** + * A named collection of functions + * + * @author bratseth + */ +public class Model { + + private final String name; + + /** Free functions */ + private final ImmutableList<ExpressionFunction> functions; + + /** Instances of each usage of the above function, where variables (if any) are replaced by their bindings */ + private final ImmutableMap<String, ExpressionFunction> referredFunctions; + + private final ImmutableMap<String, LazyArrayContext> contextPrototypes; + + public Model(String name, Collection<ExpressionFunction> functions) { + this(name, functions, Collections.emptyList()); + } + + Model(String name, Collection<ExpressionFunction> functions, Collection<ExpressionFunction> referredFunctions) { + // TODO: Optimize functions + this.name = name; + this.functions = ImmutableList.copyOf(functions); + + ImmutableMap.Builder<String, ExpressionFunction> functionsBuilder = new ImmutableMap.Builder<>(); + for (ExpressionFunction function : referredFunctions) + functionsBuilder.put(function.getName(), function); + this.referredFunctions = functionsBuilder.build(); + + ImmutableMap.Builder<String, LazyArrayContext> contextBuilder = new ImmutableMap.Builder<>(); + for (ExpressionFunction function : functions) { + try { + contextBuilder.put(function.getName(), new LazyArrayContext(function.getBody(), this.referredFunctions)); + } + catch (RuntimeException e) { + throw new IllegalArgumentException("Could not prepare an evaluation context for " + function, e); + } + } + this.contextPrototypes = contextBuilder.build(); + } + + public String name() { return name; } + + /** Returns an immutable list of the free functions of this */ + public List<ExpressionFunction> functions() { return functions; } + + /** Returns the given function, or throws a IllegalArgumentException if it does not exist */ + ExpressionFunction requireFunction(String name) { + ExpressionFunction function = function(name); + if (function == null) + throw new IllegalArgumentException("No function named '" + name + "' in " + this + ". Available functions: " + + functions.stream().map(f -> f.getName()).collect(Collectors.joining(", "))); + return function; + } + + /** Returns the given function, or throws a IllegalArgumentException if it does not exist */ + private LazyArrayContext requireContextProprotype(String name) { + LazyArrayContext context = contextPrototypes.get(name); + if (context == null) // Implies function is not present + throw new IllegalArgumentException("No function named '" + name + "' in " + this + ". Available functions: " + + functions.stream().map(f -> f.getName()).collect(Collectors.joining(", "))); + return context; + } + + /** Returns the function withe the given name, or null if none */ // TODO: Parameter overloading? + ExpressionFunction function(String name) { + for (ExpressionFunction function : functions) + if (function.getName().equals(name)) + return function; + return null; + } + + /** Returns an immutable map of the bound function instances of this, indexed by the bound instance if */ + Map<String, ExpressionFunction> boundFunctions() { return referredFunctions; } + + /** + * Returns an evaluator which can be used to evaluate the given function in a single thread once. + + * Usage: + * <code>Tensor result = model.evaluatorOf("myFunction").bind("foo", value).bind("bar", value).evaluate()</code> + * + * @throws IllegalArgumentException if the function is not present + */ + public FunctionEvaluator evaluatorOf(String function) { // TODO: Parameter overloading? + return new FunctionEvaluator(requireContextProprotype(function).copy()); + } + + @Override + public String toString() { return "model '" + name + "'"; } + +} diff --git a/model-inference/src/main/java/ai/vespa/models/evaluation/ModelsEvaluator.java b/model-inference/src/main/java/ai/vespa/models/evaluation/ModelsEvaluator.java new file mode 100644 index 00000000000..b36e06e5505 --- /dev/null +++ b/model-inference/src/main/java/ai/vespa/models/evaluation/ModelsEvaluator.java @@ -0,0 +1,46 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package ai.vespa.models.evaluation; + +import com.google.common.collect.ImmutableMap; +import com.yahoo.vespa.config.search.RankProfilesConfig; + +import java.util.Map; +import java.util.stream.Collectors; + +/** + * Evaluates machine-learned models added to Vespa applications and available as config form. + * Usage: + * <code>Tensor result = evaluator.bind("foo", value).bind("bar", value").evaluate()</code> + * + * @author bratseth + */ +public class ModelsEvaluator { + + private final ImmutableMap<String, Model> models; + + public ModelsEvaluator(RankProfilesConfig config) { + models = ImmutableMap.copyOf(new RankProfilesConfigImporter().importFrom(config)); + } + + /** Returns the models of this as an immutable map */ + public Map<String, Model> models() { return models; } + + /** + * Returns a function which can be used to evaluate the given function in the given model + * + * @throws IllegalArgumentException if the function or model is not present + */ + public FunctionEvaluator evaluatorOf(String modelName, String functionName) { + return requireModel(modelName).evaluatorOf(functionName); + } + + /** Returns the given model, or throws a IllegalArgumentException if it does not exist */ + Model requireModel(String name) { + Model model = models.get(name); + if (model == null) + throw new IllegalArgumentException("No model named '" + name + ". Available models: " + + models.keySet().stream().collect(Collectors.joining(", "))); + return model; + } + +} diff --git a/model-inference/src/main/java/ai/vespa/models/evaluation/RankProfilesConfigImporter.java b/model-inference/src/main/java/ai/vespa/models/evaluation/RankProfilesConfigImporter.java new file mode 100644 index 00000000000..bd0453f2826 --- /dev/null +++ b/model-inference/src/main/java/ai/vespa/models/evaluation/RankProfilesConfigImporter.java @@ -0,0 +1,93 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package ai.vespa.models.evaluation; + +import com.yahoo.searchlib.rankingexpression.ExpressionFunction; +import com.yahoo.searchlib.rankingexpression.RankingExpression; +import com.yahoo.searchlib.rankingexpression.parser.ParseException; +import com.yahoo.vespa.config.search.RankProfilesConfig; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * Converts RankProfilesConfig instances to RankingExpressions for evaluation + * + * @author bratseth + */ +class RankProfilesConfigImporter { + + private static final Pattern expressionPattern = + Pattern.compile("rankingExpression\\(([a-zA-Z0-9_]+)(@[a-f0-9]+\\.[a-f0-9]+)?\\)\\.rankingScript"); + + /** + * Returns a map of the models contained in this config, indexed on name. + * The map is modifiable and owned by the caller. + */ + Map<String, Model> importFrom(RankProfilesConfig config) { + try { + Map<String, Model> models = new HashMap<>(); + for (RankProfilesConfig.Rankprofile profile : config.rankprofile()) { + Model model = importProfile(profile); + models.put(model.name(), model); + } + return models; + } + catch (ParseException e) { + throw new IllegalArgumentException("Could not read rank profiles config - version mismatch?", e); + } + } + + private Model importProfile(RankProfilesConfig.Rankprofile profile) throws ParseException { + List<ExpressionFunction> functions = new ArrayList<>(); + List<ExpressionFunction> boundFunctions = new ArrayList<>(); + ExpressionFunction firstPhase = null; + ExpressionFunction secondPhase = null; + for (RankProfilesConfig.Rankprofile.Fef.Property property : profile.fef().property()) { + Matcher expressionMatcher = expressionPattern.matcher(property.name()); + if ( expressionMatcher.matches()) { + String name = expressionMatcher.group(1); + String instance = expressionMatcher.group(2); + List<String> arguments = new ArrayList<>(); // TODO: Arguments? + RankingExpression expression = new RankingExpression(name, property.value()); + + if (instance == null) // free function; make available in model under configured name + functions.add(new ExpressionFunction(name, arguments, expression)); // + + // Make all functions, bound or not available under the name they are referenced by in expressions + boundFunctions.add(new ExpressionFunction("rankingExpression(" + name + (instance != null ? instance : "") + ")", + arguments, expression)); + } + else if (property.name().equals("vespa.rank.firstphase")) { // Include in addition to macros + firstPhase = new ExpressionFunction("firstphase", new ArrayList<>(), + new RankingExpression("first-phase", property.value())); + } + else if (property.name().equals("vespa.rank.secondphase")) { // Include in addition to macros + secondPhase = new ExpressionFunction("secondphase", new ArrayList<>(), + new RankingExpression("second-phase", property.value())); + } + } + if (functionByName("firstphase", functions) == null && firstPhase != null) // may be already included, depending on body + functions.add(firstPhase); + if (functionByName("secondphase", functions) == null && secondPhase != null) // may be already included, depending on body + functions.add(secondPhase); + + try { + return new Model(profile.name(), functions, boundFunctions); + } + catch (RuntimeException e) { + throw new IllegalArgumentException("Could not load model '" + profile.name() + "'", e); + } + } + + private ExpressionFunction functionByName(String name, List<ExpressionFunction> functions) { + for (ExpressionFunction function : functions) + if (function.getName().equals(name)) + return function; + return null; + } + +} diff --git a/model-inference/src/test/java/ai/vespa/models/evaluation/ModelsEvaluatorTest.java b/model-inference/src/test/java/ai/vespa/models/evaluation/ModelsEvaluatorTest.java new file mode 100644 index 00000000000..f2031b140e7 --- /dev/null +++ b/model-inference/src/test/java/ai/vespa/models/evaluation/ModelsEvaluatorTest.java @@ -0,0 +1,51 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package ai.vespa.models.evaluation; + +import com.yahoo.config.subscription.ConfigGetter; +import com.yahoo.config.subscription.FileSource; +import com.yahoo.tensor.Tensor; +import com.yahoo.vespa.config.search.RankProfilesConfig; +import org.junit.Test; + +import java.io.File; + +import static org.junit.Assert.assertEquals; + +/** + * @author bratseth + */ +public class ModelsEvaluatorTest { + + private static final double delta = 0.00000000001; + + private ModelsEvaluator createModels() { + String configPath = "src/test/resources/config/rankexpression/rank-profiles.cfg"; + RankProfilesConfig config = new ConfigGetter<>(new FileSource(new File(configPath)), RankProfilesConfig.class).getConfig(""); + return new ModelsEvaluator(config); + } + + @Test + public void testTensorEvaluation() { + ModelsEvaluator models = createModels(); + FunctionEvaluator function = models.evaluatorOf("macros", "fourtimessum"); + function.bind("var1", Tensor.from("{{x:0}:3,{x:1}:5}")); + function.bind("var2", Tensor.from("{{x:0}:7,{x:1}:11}")); + assertEquals(Tensor.from("{{x:0}:40.0,{x:1}:64.0}"), function.evaluate()); + } + + @Test + public void testEvaluationDependingOnMacroTakingArguments() { + ModelsEvaluator models = createModels(); + FunctionEvaluator function = models.evaluatorOf("macros", "secondphase"); + function.bind("match", 3); + function.bind("rankBoost", 5); + assertEquals(32.0, function.evaluate().asDouble(), delta); + } + + // TODO: Test argument-less function + // TODO: Test that binding nonexisting variable doesn't work + // TODO: Test that rebinding dopesn't work + // TODO: Test with nested macros + // TODO: Test TF/ONNX model + +} diff --git a/model-inference/src/test/java/ai/vespa/models/evaluation/RankProfilesImporterTest.java b/model-inference/src/test/java/ai/vespa/models/evaluation/RankProfilesImporterTest.java new file mode 100644 index 00000000000..c520a389b15 --- /dev/null +++ b/model-inference/src/test/java/ai/vespa/models/evaluation/RankProfilesImporterTest.java @@ -0,0 +1,60 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package ai.vespa.models.evaluation; + +import com.yahoo.config.subscription.ConfigGetter; +import com.yahoo.config.subscription.FileSource; +import com.yahoo.searchlib.rankingexpression.ExpressionFunction; +import com.yahoo.vespa.config.search.RankProfilesConfig; +import org.junit.Test; + +import java.io.File; +import java.util.Map; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; + +/** + * Tests instantiating models from rank-profiles configs. + * + * @author bratseth + */ +public class RankProfilesImporterTest { + + @Test + public void testImporting() { + String configPath = "src/test/resources/config/rankexpression/rank-profiles.cfg"; + RankProfilesConfig config = new ConfigGetter<>(new FileSource(new File(configPath)), RankProfilesConfig.class).getConfig(""); + Map<String, Model> models = new RankProfilesConfigImporter().importFrom(config); + assertEquals(18, models.size()); + + Model macros = models.get("macros"); + assertNotNull(macros); + assertEquals("macros", macros.name()); + assertEquals(4, macros.functions().size()); + assertFunction("fourtimessum", "4 * (var1 + var2)", macros); + assertFunction("firstphase", "match + fieldMatch(title) + rankingExpression(myfeature)", macros); + assertFunction("secondphase", "rankingExpression(fourtimessum@5cf279212355b980.67f1e87166cfef86)", macros); + assertFunction("myfeature", + "70 * fieldMatch(title).completeness * pow(0 - fieldMatch(title).earliness,2) + " + + "30 * pow(0 - fieldMatch(description).earliness,2)", + macros); + assertEquals(4, macros.boundFunctions().size()); + assertBoundFunction("rankingExpression(fourtimessum@5cf279212355b980.67f1e87166cfef86)", + "4 * (match + rankBoost)", macros); + } + + private void assertFunction(String name, String expression, Model model) { + ExpressionFunction function = model.function(name); + assertNotNull(function); + assertEquals(name, function.getName()); + assertEquals(expression, function.getBody().getRoot().toString()); + } + + private void assertBoundFunction(String name, String expression, Model model) { + ExpressionFunction function = model.boundFunctions().get(name); + assertNotNull("Function '" + name + "' is present", function); + assertEquals(name, function.getName()); + assertEquals(expression, function.getBody().getRoot().toString()); + } + +} diff --git a/model-inference/src/test/resources/config/rankexpression/rank-profiles.cfg b/model-inference/src/test/resources/config/rankexpression/rank-profiles.cfg new file mode 100644 index 00000000000..f5652c31d2a --- /dev/null +++ b/model-inference/src/test/resources/config/rankexpression/rank-profiles.cfg @@ -0,0 +1,296 @@ +rankprofile[0].name "default" +rankprofile[0].fef.property[0].name "foo" +rankprofile[0].fef.property[0].value "bar, baz" +rankprofile[0].fef.property[1].name "foo" +rankprofile[0].fef.property[1].value "foobar" +rankprofile[0].fef.property[2].name "qux" +rankprofile[0].fef.property[2].value "quux" +rankprofile[0].fef.property[3].name "foo.bar" +rankprofile[0].fef.property[3].value "foo.bar" +rankprofile[0].fef.property[4].name "foo.bar.baz" +rankprofile[0].fef.property[4].value "123" +rankprofile[0].fef.property[5].name "foo(bar).baz.2" +rankprofile[0].fef.property[5].value "123.4" +rankprofile[0].fef.property[6].name "foo(bar).baz.qux" +rankprofile[0].fef.property[6].value "foo(bar)" +rankprofile[0].fef.property[7].name "nud" +rankprofile[0].fef.property[7].value "ity" +rankprofile[0].fef.property[8].name "vespa.rank.firstphase" +rankprofile[0].fef.property[8].value "classicRank" +rankprofile[0].fef.property[9].name "vespa.rank.secondphase" +rankprofile[0].fef.property[9].value "rankingExpression(secondphase)" +rankprofile[0].fef.property[10].name "rankingExpression(secondphase).rankingScript" +rankprofile[0].fef.property[10].value "4" +rankprofile[0].fef.property[11].name "vespa.dump.feature" +rankprofile[0].fef.property[11].value "attribute(foo1).out" +rankprofile[0].fef.property[12].name "vespa.dump.feature" +rankprofile[0].fef.property[12].value "attribute(bar1)" +rankprofile[0].fef.property[13].name "vespa.dump.feature" +rankprofile[0].fef.property[13].value "attribute(foo2).out" +rankprofile[0].fef.property[14].name "vespa.dump.feature" +rankprofile[0].fef.property[14].value "attribute(bar2).out" +rankprofile[0].fef.property[15].name "vespa.dump.feature" +rankprofile[0].fef.property[15].value "attribute(foo3).out" +rankprofile[0].fef.property[16].name "vespa.dump.feature" +rankprofile[0].fef.property[16].value "attribute(bar3).out" +rankprofile[0].fef.property[17].name "vespa.dump.feature" +rankprofile[0].fef.property[17].value "attribute(foo4).out" +rankprofile[0].fef.property[18].name "vespa.dump.feature" +rankprofile[0].fef.property[18].value "attribute(bar4).out" +rankprofile[0].fef.property[19].name "vespa.hitcollector.heapsize" +rankprofile[0].fef.property[19].value "10" +rankprofile[0].fef.property[20].name "vespa.hitcollector.arraysize" +rankprofile[0].fef.property[20].value "20" +rankprofile[0].fef.property[21].name "vespa.hitcollector.rankscoredroplimit" +rankprofile[0].fef.property[21].value "-0.5" +rankprofile[0].fef.property[22].name "vespa.dump.ignoredefaultfeatures" +rankprofile[0].fef.property[22].value "true" +rankprofile[1].name "unranked" +rankprofile[1].fef.property[0].name "vespa.rank.firstphase" +rankprofile[1].fef.property[0].value "value(0)" +rankprofile[1].fef.property[1].name "vespa.hitcollector.heapsize" +rankprofile[1].fef.property[1].value "0" +rankprofile[1].fef.property[2].name "vespa.hitcollector.arraysize" +rankprofile[1].fef.property[2].value "0" +rankprofile[1].fef.property[3].name "vespa.dump.ignoredefaultfeatures" +rankprofile[1].fef.property[3].value "true" +rankprofile[2].name "static" +rankprofile[2].fef.property[0].name "vespa.rank.firstphase" +rankprofile[2].fef.property[0].value "attribute" +rankprofile[2].fef.property[1].name "vespa.rank.secondphase" +rankprofile[2].fef.property[1].value "rankingExpression(secondphase)" +rankprofile[2].fef.property[2].name "rankingExpression(secondphase).rankingScript" +rankprofile[2].fef.property[2].value "10 + feature(arg1).out.out" +rankprofile[2].fef.property[3].name "vespa.summary.feature" +rankprofile[2].fef.property[3].value "attribute(foo1).out" +rankprofile[2].fef.property[4].name "vespa.summary.feature" +rankprofile[2].fef.property[4].value "attribute(bar1)" +rankprofile[2].fef.property[5].name "vespa.summary.feature" +rankprofile[2].fef.property[5].value "attribute(foo2).out" +rankprofile[2].fef.property[6].name "vespa.summary.feature" +rankprofile[2].fef.property[6].value "attribute(bar2).out" +rankprofile[2].fef.property[7].name "vespa.summary.feature" +rankprofile[2].fef.property[7].value "attribute(foo3).out" +rankprofile[2].fef.property[8].name "vespa.summary.feature" +rankprofile[2].fef.property[8].value "attribute(bar3).out" +rankprofile[2].fef.property[9].name "vespa.summary.feature" +rankprofile[2].fef.property[9].value "attribute(foo4).out" +rankprofile[2].fef.property[10].name "vespa.summary.feature" +rankprofile[2].fef.property[10].value "attribute(bar4).out" +rankprofile[3].name "overflow" +rankprofile[3].fef.property[0].name "vespa.rank.firstphase" +rankprofile[3].fef.property[0].value "rankingExpression(firstphase)" +rankprofile[3].fef.property[1].name "rankingExpression(firstphase).rankingScript" +rankprofile[3].fef.property[1].value "feature1(argument1,argument2,argument3,argument4).output + feature2(argument1,argument2,argument3,argument4).output + feature3(argument1,argument2,argument3,argument4).output + feature4(argument1,argument2,argument3,argument4).output + feature5(argument1,argument2,argument3,argument4).output + feature6(argument1,argument2,argument3,argument4).output + feature7(argument1,argument2,argument3,argument4).output + feature8(argument1,argument2,argument3,argument4).output + feature9(argument1,argument2,argument3,argument4).output + feature10(argument1,argument2,argument3,argument4).output + feature11(argument1,argument2,argument3,argument4).output + feature12(argument1,argument2,argument3,argument4).output + feature13(argument1,argument2,argument3,argument4).output + feature14(argument1,argument2,argument3,argument4).output + feature15(argument1,argument2,argument3,argument4).output + feature16(argument1,argument2,argument3,argument4).output + feature17(argument1,argument2,argument3,argument4).output + feature18(argument1,argument2,argument3,argument4).output + feature19(argument1,argument2,argument3,argument4).output + feature20(argument1,argument2,argument3,argument4).output + feature21(argument1,argument2,argument3,argument4).output + feature22(argument1,argument2,argument3,argument4).output + feature23(argument1,argument2,argument3,argument4).output + feature24(argument1,argument2,argument3,argument4).output + feature25(argument1,argument2,argument3,argument4).output + feature26(argument1,argument2,argument3,argument4).output + feature27(argument1,argument2,argument3,argument4).output + feature28(argument1,argument2,argument3,argument4).output + feature29(argument1,argument2,argument3,argument4).output + feature30(argument1,argument2,argument3,argument4).output + feature31(argument1,argument2,argument3,argument4).output + feature32(argument1,argument2,argument3,argument4).output + feature33(argument1,argument2,argument3,argument4).output + feature34(argument1,argument2,argument3,argument4).output + feature35(argument1,argument2,argument3,argument4).output + feature36(argument1,argument2,argument3,argument4).output + feature37(argument1,argument2,argument3,argument4).output + feature38(argument1,argument2,argument3,argument4).output + feature39(argument1,argument2,argument3,argument4).output + feature40(argument1,argument2,argument3,argument4).output + feature41(argument1,argument2,argument3,argument4).output + feature42(argument1,argument2,argument3,argument4).output + feature43(argument1,argument2,argument3,argument4).output + feature44(argument1,argument2,argument3,argument4).output + feature45(argument1,argument2,argument3,argument4).output + feature46(argument1,argument2,argument3,argument4).output + feature47(argument1,argument2,argument3,argument4).output + feature48(argument1,argument2,argument3,argument4).output + feature49(argument1,argument2,argument3,argument4).output + feature50(argument1,argument2,argument3,argument4).output + feature51(argument1,argument2,argument3,argument4).output + feature52(argument1,argument2,argument3,argument4).output + feature53(argument1,argument2,argument3,argument4).output + feature54(argument1,argument2,argument3,argument4).output + feature55(argument1,argument2,argument3,argument4).output + feature56(argument1,argument2,argument3,argument4).output + feature57(argument1,argument2,argument3,argument4).output + feature58(argument1,argument2,argument3,argument4).output + feature59(argument1,argument2,argument3,argument4).output + feature60(argument1,argument2,argument3,argument4).output + feature61(argument1,argument2,argument3,argument4).output + feature62(argument1,argument2,argument3,argument4).output + feature63(argument1,argument2,argument3,argument4).output + feature64(argument1,argument2,argument3,argument4).output + feature65(argument1,argument2,argument3,argument4).output + feature66(argument1,argument2,argument3,argument4).output + feature67(argument1,argument2,argument3,argument4).output + feature68(argument1,argument2,argument3,argument4).output + feature69(argument1,argument2,argument3,argument4).output + feature70(argument1,argument2,argument3,argument4).output + feature71(argument1,argument2,argument3,argument4).output + feature72(argument1,argument2,argument3,argument4).output + feature73(argument1,argument2,argument3,argument4).output + feature74(argument1,argument2,argument3,argument4).output + feature75(argument1,argument2,argument3,argument4).output + feature76(argument1,argument2,argument3,argument4).output + feature77(argument1,argument2,argument3,argument4).output + feature78(argument1,argument2,argument3,argument4).output + feature79(argument1,argument2,argument3,argument4).output + feature80(argument1,argument2,argument3,argument4).output + feature81(argument1,argument2,argument3,argument4).output + feature82(argument1,argument2,argument3,argument4).output + feature83(argument1,argument2,argument3,argument4).output + feature84(argument1,argument2,argument3,argument4).output + feature85(argument1,argument2,argument3,argument4).output + feature86(argument1,argument2,argument3,argument4).output + feature87(argument1,argument2,argument3,argument4).output + feature88(argument1,argument2,argument3,argument4).output + feature89(argument1,argument2,argument3,argument4).output + feature90(argument1,argument2,argument3,argument4).output + feature91(argument1,argument2,argument3,argument4).output + feature92(argument1,argument2,argument3,argument4).output + feature93(argument1,argument2,argument3,argument4).output + feature94(argument1,argument2,argument3,argument4).output + feature95(argument1,argument2,argument3,argument4).output + feature96(argument1,argument2,argument3,argument4).output + feature97(argument1,argument2,argument3,argument4).output + feature98(argument1,argument2,argument3,argument4).output + feature99(argument1,argument2,argument3,argument4).output + feature100(argument1,argument2,argument3,argument4).output + feature101(argument1,argument2,argument3,argument4).output + feature102(argument1,argument2,argument3,argument4).output + feature103(argument1,argument2,argument3,argument4).output + feature104(argument1,argument2,argument3,argument4).output + feature105(argument1,argument2,argument3,argument4).output + feature106(argument1,argument2,argument3,argument4).output + feature107(argument1,argument2,argument3,argument4).output + feature108(argument1,argument2,argument3,argument4).output + feature109(argument1,argument2,argument3,argument4).output + feature110(argument1,argument2,argument3,argument4).output + feature111(argument1,argument2,argument3,argument4).output + feature112(argument1,argument2,argument3,argument4).output + feature113(argument1,argument2,argument3,argument4).output + feature114(argument1,argument2,argument3,argument4).output + feature115(argument1,argument2,argument3,argument4).output + feature116(argument1,argument2,argument3,argument4).output + feature117(argument1,argument2,argument3,argument4).output + feature118(argument1,argument2,argument3,argument4).output + feature119(argument1,argument2,argument3,argument4).output + feature120(argument1,argument2,argument3,argument4).output + feature121(argument1,argument2,argument3,argument4).output + feature122(argument1,argument2,argument3,argument4).output + feature123(argument1,argument2,argument3,argument4).output + feature124(argument1,argument2,argument3,argument4).output + feature125(argument1,argument2,argument3,argument4).output + feature126(argument1,argument2,argument3,argument4).output + feature127(argument1,argument2,argument3,argument4).output + feature128(argument1,argument2,argument3,argument4).output + feature129(argument1,argument2,argument3,argument4).output + feature130(argument1,argument2,argument3,argument4).output + feature131(argument1,argument2,argument3,argument4).output + feature132(argument1,argument2,argument3,argument4).output + feature133(argument1,argument2,argument3,argument4).output + feature134(argument1,argument2,argument3,argument4).output + feature135(argument1,argument2,argument3,argument4).output + feature136(argument1,argument2,argument3,argument4).output + feature137(argument1,argument2,argument3,argument4).output + feature138(argument1,argument2,argument3,argument4).output + feature139(argument1,argument2,argument3,argument4).output + feature140(argument1,argument2,argument3,argument4).output + feature141(argument1,argument2,argument3,argument4).output + feature142(argument1,argument2,argument3,argument4).output + feature143(argument1,argument2,argument3,argument4).output + feature144(argument1,argument2,argument3,argument4).output + feature145(argument1,argument2,argument3,argument4).output + feature146(argument1,argument2,argument3,argument4).output + feature147(argument1,argument2,argument3,argument4).output + feature148(argument1,argument2,argument3,argument4).output + feature149(argument1,argument2,argument3,argument4).output + feature150(argument1,argument2,argument3,argument4).output + feature151(argument1,argument2,argument3,argument4).output + feature152(argument1,argument2,argument3,argument4).output + feature153(argument1,argument2,argument3,argument4).output + feature154(argument1,argument2,argument3,argument4).output + feature155(argument1,argument2,argument3,argument4).output + feature156(argument1,argument2,argument3,argument4).output + feature157(argument1,argument2,argument3,argument4).output + feature158(argument1,argument2,argument3,argument4).output + feature159(argument1,argument2,argument3,argument4).output + feature160(argument1,argument2,argument3,argument4).output + feature161(argument1,argument2,argument3,argument4).output + feature162(argument1,argument2,argument3,argument4).output + feature163(argument1,argument2,argument3,argument4).output + feature164(argument1,argument2,argument3,argument4).output + feature165(argument1,argument2,argument3,argument4).output + feature166(argument1,argument2,argument3,argument4).output + feature167(argument1,argument2,argument3,argument4).output + feature168(argument1,argument2,argument3,argument4).output + feature169(argument1,argument2,argument3,argument4).output + feature170(argument1,argument2,argument3,argument4).output + feature171(argument1,argument2,argument3,argument4).output + feature172(argument1,argument2,argument3,argument4).output + feature173(argument1,argument2,argument3,argument4).output + feature174(argument1,argument2,argument3,argument4).output + feature175(argument1,argument2,argument3,argument4).output + feature176(argument1,argument2,argument3,argument4).output + feature177(argument1,argument2,argument3,argument4).output + feature178(argument1,argument2,argument3,argument4).output + feature179(argument1,argument2,argument3,argument4).output + feature180(argument1,argument2,argument3,argument4).output + feature181(argument1,argument2,argument3,argument4).output + feature182(argument1,argument2,argument3,argument4).output + feature183(argument1,argument2,argument3,argument4).output + feature184(argument1,argument2,argument3,argument4).output + feature185(argument1,argument2,argument3,argument4).output + feature186(argument1,argument2,argument3,argument4).output + feature187(argument1,argument2,argument3,argument4).output + feature188(argument1,argument2,argument3,argument4).output + feature189(argument1,argument2,argument3,argument4).output + feature190(argument1,argument2,argument3,argument4).output + feature191(argument1,argument2,argument3,argument4).output + feature192(argument1,argument2,argument3,argument4).output + feature193(argument1,argument2,argument3,argument4).output + feature194(argument1,argument2,argument3,argument4).output + feature195(argument1,argument2,argument3,argument4).output + feature196(argument1,argument2,argument3,argument4).output + feature197(argument1,argument2,argument3,argument4).output + feature198(argument1,argument2,argument3,argument4).output + feature199(argument1,argument2,argument3,argument4).output + feature200(argument1,argument2,argument3,argument4).output + feature201(argument1,argument2,argument3,argument4).output + feature202(argument1,argument2,argument3,argument4).output + feature203(argument1,argument2,argument3,argument4).output + feature204(argument1,argument2,argument3,argument4).output + feature205(argument1,argument2,argument3,argument4).output + feature206(argument1,argument2,argument3,argument4).output + feature207(argument1,argument2,argument3,argument4).output + feature208(argument1,argument2,argument3,argument4).output + feature209(argument1,argument2,argument3,argument4).output + feature210(argument1,argument2,argument3,argument4).output + feature211(argument1,argument2,argument3,argument4).output + feature212(argument1,argument2,argument3,argument4).output + feature213(argument1,argument2,argument3,argument4).output + feature214(argument1,argument2,argument3,argument4).output + feature215(argument1,argument2,argument3,argument4).output + feature216(argument1,argument2,argument3,argument4).output + feature217(argument1,argument2,argument3,argument4).output + feature218(argument1,argument2,argument3,argument4).output + feature219(argument1,argument2,argument3,argument4).output + feature220(argument1,argument2,argument3,argument4).output + feature221(argument1,argument2,argument3,argument4).output + feature222(argument1,argument2,argument3,argument4).output + feature223(argument1,argument2,argument3,argument4).output + feature224(argument1,argument2,argument3,argument4).output + feature225(argument1,argument2,argument3,argument4).output + feature226(argument1,argument2,argument3,argument4).output + feature227(argument1,argument2,argument3,argument4).output + feature228(argument1,argument2,argument3,argument4).output + feature229(argument1,argument2,argument3,argument4).output + feature230(argument1,argument2,argument3,argument4).output + feature231(argument1,argument2,argument3,argument4).output + feature232(argument1,argument2,argument3,argument4).output + feature233(argument1,argument2,argument3,argument4).output + feature234(argument1,argument2,argument3,argument4).output + feature235(argument1,argument2,argument3,argument4).output + feature236(argument1,argument2,argument3,argument4).output + feature237(argument1,argument2,argument3,argument4).output + feature238(argument1,argument2,argument3,argument4).output + feature239(argument1,argument2,argument3,argument4).output + feature240(argument1,argument2,argument3,argument4).output + feature241(argument1,argument2,argument3,argument4).output + feature242(argument1,argument2,argument3,argument4).output + feature243(argument1,argument2,argument3,argument4).output + feature244(argument1,argument2,argument3,argument4).output + feature245(argument1,argument2,argument3,argument4).output + feature246(argument1,argument2,argument3,argument4).output + feature247(argument1,argument2,argument3,argument4).output + feature248(argument1,argument2,argument3,argument4).output + feature249(argument1,argument2,argument3,argument4).output + feature250(argument1,argument2,argument3,argument4).output + feature251(argument1,argument2,argument3,argument4).output + feature252(argument1,argument2,argument3,argument4).output + feature253(argument1,argument2,argument3,argument4).output + feature254(argument1,argument2,argument3,argument4).output + feature255(argument1,argument2,argument3,argument4).output + feature256(argument1,argument2,argument3,argument4).output + feature257(argument1,argument2,argument3,argument4).output + feature258(argument1,argument2,argument3,argument4).output + feature259(argument1,argument2,argument3,argument4).output + feature260(argument1,argument2,argument3,argument4).output + feature261(argument1,argument2,argument3,argument4).output + feature262(argument1,argument2,argument3,argument4).output + feature263(argument1,argument2,argument3,argument4).output + feature264(argument1,argument2,argument3,argument4).output + feature265(argument1,argument2,argument3,argument4).output + feature266(argument1,argument2,argument3,argument4).output + feature267(argument1,argument2,argument3,argument4).output + feature268(argument1,argument2,argument3,argument4).output + feature269(argument1,argument2,argument3,argument4).output + feature270(argument1,argument2,argument3,argument4).output + feature271(argument1,argument2,argument3,argument4).output + feature272(argument1,argument2,argument3,argument4).output + feature273(argument1,argument2,argument3,argument4).output + feature274(argument1,argument2,argument3,argument4).output + feature275(argument1,argument2,argument3,argument4).output + feature276(argument1,argument2,argument3,argument4).output + feature277(argument1,argument2,argument3,argument4).output + feature278(argument1,argument2,argument3,argument4).output + feature279(argument1,argument2,argument3,argument4).output + feature280(argument1,argument2,argument3,argument4).output + feature281(argument1,argument2,argument3,argument4).output + feature282(argument1,argument2,argument3,argument4).output + feature283(argument1,argument2,argument3,argument4).output + feature284(argument1,argument2,argument3,argument4).output + feature285(argument1,argument2,argument3,argument4).output + feature286(argument1,argument2,argument3,argument4).output + feature287(argument1,argument2,argument3,argument4).output + feature288(argument1,argument2,argument3,argument4).output + feature289(argument1,argument2,argument3,argument4).output + feature290(argument1,argument2,argument3,argument4).output + feature291(argument1,argument2,argument3,argument4).output + feature292(argument1,argument2,argument3,argument4).output + feature293(argument1,argument2,argument3,argument4).output + feature294(argument1,argument2,argument3,argument4).output + feature295(argument1,argument2,argument3,argument4).output + feature296(argument1,argument2,argument3,argument4).output + feature297(argument1,argument2,argument3,argument4).output + feature298(argument1,argument2,argument3,argument4).output + feature299(argument1,argument2,argument3,argument4).output + feature300(argument1,argument2,argument3,argument4).output" +rankprofile[3].fef.property[2].name "vespa.rank.secondphase" +rankprofile[3].fef.property[2].value "rankingExpression(secondphase)" +rankprofile[3].fef.property[3].name "rankingExpression(secondphase).rankingScript" +rankprofile[3].fef.property[3].value "exp(0) + mysum(attribute(foo),\"attribute( bar )\",\"attribute( \\\"baz\\\" )\")" +rankprofile[3].fef.property[4].name "vespa.hitcollector.heapsize" +rankprofile[3].fef.property[4].value "101" +rankprofile[3].fef.property[5].name "vespa.hitcollector.arraysize" +rankprofile[3].fef.property[5].value "201" +rankprofile[3].fef.property[6].name "vespa.hitcollector.rankscoredroplimit" +rankprofile[3].fef.property[6].value "501.5" +rankprofile[4].name "duplicates" +rankprofile[4].fef.property[0].name "fieldMatch(a).proximityLimit" +rankprofile[4].fef.property[0].value "4" +rankprofile[4].fef.property[1].name "fieldMatch(a).proximityTable" +rankprofile[4].fef.property[1].value "0.2" +rankprofile[4].fef.property[2].name "fieldMatch(a).proximityTable" +rankprofile[4].fef.property[2].value "0.4" +rankprofile[4].fef.property[3].name "fieldMatch(a).proximityTable" +rankprofile[4].fef.property[3].value "0.6" +rankprofile[4].fef.property[4].name "fieldMatch(a).proximityTable" +rankprofile[4].fef.property[4].value "0.8" +rankprofile[4].fef.property[5].name "fieldMatch(a).proximityTable" +rankprofile[4].fef.property[5].value "1" +rankprofile[4].fef.property[6].name "fieldMatch(a).proximityTable" +rankprofile[4].fef.property[6].value "0.8" +rankprofile[4].fef.property[7].name "fieldMatch(a).proximityTable" +rankprofile[4].fef.property[7].value "0.6" +rankprofile[4].fef.property[8].name "fieldMatch(a).proximityTable" +rankprofile[4].fef.property[8].value "0.4" +rankprofile[4].fef.property[9].name "fieldMatch(a).proximityTable" +rankprofile[4].fef.property[9].value "0.2" +rankprofile[5].name "whitespace1" +rankprofile[5].fef.property[0].name "vespa.rank.firstphase" +rankprofile[5].fef.property[0].value "rankingExpression(firstphase)" +rankprofile[5].fef.property[1].name "rankingExpression(firstphase).rankingScript" +rankprofile[5].fef.property[1].value "1" +rankprofile[6].name "whitespace2" +rankprofile[6].fef.property[0].name "vespa.rank.firstphase" +rankprofile[6].fef.property[0].value "rankingExpression(firstphase)" +rankprofile[6].fef.property[1].name "rankingExpression(firstphase).rankingScript" +rankprofile[6].fef.property[1].value "1" +rankprofile[7].name "macros" +rankprofile[7].fef.property[0].name "rankingExpression(fourtimessum).rankingScript" +rankprofile[7].fef.property[0].value "4 * (var1 + var2)" +rankprofile[7].fef.property[1].name "rankingExpression(myfeature).rankingScript" +rankprofile[7].fef.property[1].value "70 * fieldMatch(title).completeness * pow(0 - fieldMatch(title).earliness,2) + 30 * pow(0 - fieldMatch(description).earliness,2)" +rankprofile[7].fef.property[2].name "rankingExpression(fourtimessum@5cf279212355b980.67f1e87166cfef86).rankingScript" +rankprofile[7].fef.property[2].value "4 * (match + rankBoost)" +rankprofile[7].fef.property[3].name "vespa.rank.firstphase" +rankprofile[7].fef.property[3].value "rankingExpression(firstphase)" +rankprofile[7].fef.property[4].name "rankingExpression(firstphase).rankingScript" +rankprofile[7].fef.property[4].value "match + fieldMatch(title) + rankingExpression(myfeature)" +rankprofile[7].fef.property[5].name "vespa.rank.secondphase" +rankprofile[7].fef.property[5].value "rankingExpression(fourtimessum@5cf279212355b980.67f1e87166cfef86)" +rankprofile[7].fef.property[6].name "vespa.summary.feature" +rankprofile[7].fef.property[6].value "fieldMatch(title)" +rankprofile[8].name "macros2" +rankprofile[8].fef.property[0].name "foo" +rankprofile[8].fef.property[0].value "some, list" +rankprofile[8].fef.property[1].name "rankingExpression(fourtimessum).rankingScript" +rankprofile[8].fef.property[1].value "4 * (var1 + var2)" +rankprofile[8].fef.property[2].name "rankingExpression(myfeature).rankingScript" +rankprofile[8].fef.property[2].value "70 * fieldMatch(title).completeness * pow(0 - fieldMatch(title).earliness,2) + 30 * pow(0 - fieldMatch(description).earliness,2)" +rankprofile[8].fef.property[3].name "rankingExpression(mysummaryfeature).rankingScript" +rankprofile[8].fef.property[3].value "70 * fieldMatch(title).completeness" +rankprofile[8].fef.property[4].name "rankingExpression(mysummaryfeature2).rankingScript" +rankprofile[8].fef.property[4].value "71 * fieldMatch(title).completeness" +rankprofile[8].fef.property[5].name "rankingExpression(fourtimessum@2b1138e8965e7ff5.67f1e87166cfef86).rankingScript" +rankprofile[8].fef.property[5].value "4 * (match + match)" +rankprofile[8].fef.property[6].name "vespa.rank.firstphase" +rankprofile[8].fef.property[6].value "classicRank" +rankprofile[8].fef.property[7].name "vespa.rank.secondphase" +rankprofile[8].fef.property[7].value "rankingExpression(secondphase)" +rankprofile[8].fef.property[8].name "rankingExpression(secondphase).rankingScript" +rankprofile[8].fef.property[8].value "rankingExpression(fourtimessum@2b1138e8965e7ff5.67f1e87166cfef86) + rankingExpression(mysummaryfeature) + rankingExpression(myfeature)" +rankprofile[8].fef.property[9].name "vespa.summary.feature" +rankprofile[8].fef.property[9].value "rankingExpression(mysummaryfeature2)" +rankprofile[8].fef.property[10].name "vespa.summary.feature" +rankprofile[8].fef.property[10].value "rankingExpression(mysummaryfeature)" +rankprofile[9].name "macros3" +rankprofile[9].fef.property[0].name "rankingExpression(onlyusedinsummaryfeature).rankingScript" +rankprofile[9].fef.property[0].value "5" +rankprofile[9].fef.property[1].name "vespa.summary.feature" +rankprofile[9].fef.property[1].value "rankingExpression(matches(title,rankingExpression(onlyusedinsummaryfeature)))" +rankprofile[10].name "macros3-inherited" +rankprofile[10].fef.property[0].name "rankingExpression(onlyusedinsummaryfeature).rankingScript" +rankprofile[10].fef.property[0].value "5" +rankprofile[10].fef.property[1].name "vespa.summary.feature" +rankprofile[10].fef.property[1].value "rankingExpression(matches(title,rankingExpression(onlyusedinsummaryfeature)))" +rankprofile[11].name "macros-inherited" +rankprofile[11].fef.property[0].name "foo" +rankprofile[11].fef.property[0].value "some, list" +rankprofile[11].fef.property[1].name "rankingExpression(fourtimessum).rankingScript" +rankprofile[11].fef.property[1].value "4 * (var1 + var2)" +rankprofile[11].fef.property[2].name "rankingExpression(myfeature).rankingScript" +rankprofile[11].fef.property[2].value "70 * fieldMatch(title).completeness * pow(0 - fieldMatch(title).earliness,2) + 30 * pow(0 - fieldMatch(description).earliness,2)" +rankprofile[11].fef.property[3].name "rankingExpression(mysummaryfeature).rankingScript" +rankprofile[11].fef.property[3].value "80 * fieldMatch(title).completeness" +rankprofile[11].fef.property[4].name "rankingExpression(mysummaryfeature2).rankingScript" +rankprofile[11].fef.property[4].value "71 * fieldMatch(title).completeness" +rankprofile[11].fef.property[5].name "rankingExpression(fourtimessum@2b1138e8965e7ff5.67f1e87166cfef86).rankingScript" +rankprofile[11].fef.property[5].value "4 * (match + match)" +rankprofile[11].fef.property[6].name "vespa.rank.firstphase" +rankprofile[11].fef.property[6].value "rankingExpression(firstphase)" +rankprofile[11].fef.property[7].name "rankingExpression(firstphase).rankingScript" +rankprofile[11].fef.property[7].value "20000 * rankingExpression(myfeature) + rankingExpression(mysummaryfeature)" +rankprofile[11].fef.property[8].name "vespa.rank.secondphase" +rankprofile[11].fef.property[8].value "rankingExpression(secondphase)" +rankprofile[11].fef.property[9].name "rankingExpression(secondphase).rankingScript" +rankprofile[11].fef.property[9].value "rankingExpression(fourtimessum@2b1138e8965e7ff5.67f1e87166cfef86) + rankingExpression(mysummaryfeature) + rankingExpression(myfeature)" +rankprofile[11].fef.property[10].name "vespa.summary.feature" +rankprofile[11].fef.property[10].value "rankingExpression(mysummaryfeature2)" +rankprofile[11].fef.property[11].name "vespa.summary.feature" +rankprofile[11].fef.property[11].value "rankingExpression(mysummaryfeature)" +rankprofile[12].name "macros-inherited2" +rankprofile[12].fef.property[0].name "foo" +rankprofile[12].fef.property[0].value "some, list" +rankprofile[12].fef.property[1].name "rankingExpression(fourtimessum).rankingScript" +rankprofile[12].fef.property[1].value "4 * (var1 + var2)" +rankprofile[12].fef.property[2].name "rankingExpression(myfeature).rankingScript" +rankprofile[12].fef.property[2].value "70 * fieldMatch(title).completeness * pow(0 - fieldMatch(title).earliness,2) + 30 * pow(0 - fieldMatch(description).earliness,2)" +rankprofile[12].fef.property[3].name "rankingExpression(mysummaryfeature).rankingScript" +rankprofile[12].fef.property[3].value "80 * fieldMatch(title).completeness" +rankprofile[12].fef.property[4].name "rankingExpression(mysummaryfeature2).rankingScript" +rankprofile[12].fef.property[4].value "71 * fieldMatch(title).completeness" +rankprofile[12].fef.property[5].name "rankingExpression(fourtimessum@2b1138e8965e7ff5.67f1e87166cfef86).rankingScript" +rankprofile[12].fef.property[5].value "4 * (match + match)" +rankprofile[12].fef.property[6].name "vespa.rank.firstphase" +rankprofile[12].fef.property[6].value "rankingExpression(firstphase)" +rankprofile[12].fef.property[7].name "rankingExpression(firstphase).rankingScript" +rankprofile[12].fef.property[7].value "30000 * rankingExpression(mysummaryfeature) + rankingExpression(myfeature)" +rankprofile[12].fef.property[8].name "vespa.rank.secondphase" +rankprofile[12].fef.property[8].value "rankingExpression(secondphase)" +rankprofile[12].fef.property[9].name "rankingExpression(secondphase).rankingScript" +rankprofile[12].fef.property[9].value "rankingExpression(fourtimessum@2b1138e8965e7ff5.67f1e87166cfef86) + rankingExpression(mysummaryfeature) + rankingExpression(myfeature)" +rankprofile[12].fef.property[10].name "vespa.summary.feature" +rankprofile[12].fef.property[10].value "rankingExpression(mysummaryfeature2)" +rankprofile[12].fef.property[11].name "vespa.summary.feature" +rankprofile[12].fef.property[11].value "rankingExpression(mysummaryfeature)" +rankprofile[13].name "macros-inherited3" +rankprofile[13].fef.property[0].name "foo" +rankprofile[13].fef.property[0].value "some, list" +rankprofile[13].fef.property[1].name "rankingExpression(fourtimessum).rankingScript" +rankprofile[13].fef.property[1].value "4 * (var1 + var2)" +rankprofile[13].fef.property[2].name "rankingExpression(myfeature).rankingScript" +rankprofile[13].fef.property[2].value "700 * fieldMatch(title).completeness" +rankprofile[13].fef.property[3].name "rankingExpression(mysummaryfeature).rankingScript" +rankprofile[13].fef.property[3].value "80 * fieldMatch(title).completeness" +rankprofile[13].fef.property[4].name "rankingExpression(mysummaryfeature2).rankingScript" +rankprofile[13].fef.property[4].value "71 * fieldMatch(title).completeness" +rankprofile[13].fef.property[5].name "vespa.rank.firstphase" +rankprofile[13].fef.property[5].value "rankingExpression(firstphase)" +rankprofile[13].fef.property[6].name "rankingExpression(firstphase).rankingScript" +rankprofile[13].fef.property[6].value "30000 * rankingExpression(mysummaryfeature) + rankingExpression(myfeature)" +rankprofile[13].fef.property[7].name "vespa.rank.secondphase" +rankprofile[13].fef.property[7].value "rankingExpression(secondphase)" +rankprofile[13].fef.property[8].name "rankingExpression(secondphase).rankingScript" +rankprofile[13].fef.property[8].value "40000 * rankingExpression(mysummaryfeature) + rankingExpression(myfeature)" +rankprofile[13].fef.property[9].name "vespa.summary.feature" +rankprofile[13].fef.property[9].value "rankingExpression(mysummaryfeature2)" +rankprofile[13].fef.property[10].name "vespa.summary.feature" +rankprofile[13].fef.property[10].value "rankingExpression(mysummaryfeature)" +rankprofile[14].name "macros-refering-macros" +rankprofile[14].fef.property[0].name "rankingExpression(m1).rankingScript" +rankprofile[14].fef.property[0].value "700 * fieldMatch(title).completeness" +rankprofile[14].fef.property[1].name "rankingExpression(m2).rankingScript" +rankprofile[14].fef.property[1].value "rankingExpression(m1) * 67" +rankprofile[14].fef.property[2].name "rankingExpression(m4).rankingScript" +rankprofile[14].fef.property[2].value "703 * fieldMatch(fromfile).completeness" +rankprofile[14].fef.property[3].name "vespa.rank.secondphase" +rankprofile[14].fef.property[3].value "rankingExpression(secondphase)" +rankprofile[14].fef.property[4].name "rankingExpression(secondphase).rankingScript" +rankprofile[14].fef.property[4].value "40000 * rankingExpression(m2)" +rankprofile[15].name "macros-refering-macros-inherited" +rankprofile[15].fef.property[0].name "rankingExpression(m1).rankingScript" +rankprofile[15].fef.property[0].value "700 * fieldMatch(title).completeness" +rankprofile[15].fef.property[1].name "rankingExpression(m2).rankingScript" +rankprofile[15].fef.property[1].value "rankingExpression(m1) * 67" +rankprofile[15].fef.property[2].name "rankingExpression(m4).rankingScript" +rankprofile[15].fef.property[2].value "701 * fieldMatch(title).completeness" +rankprofile[15].fef.property[3].name "rankingExpression(m3).rankingScript" +rankprofile[15].fef.property[3].value "if (isNan(attribute(nrtgmp)) == 1, 0.0, rankingExpression(m2))" +rankprofile[15].fef.property[4].name "vespa.rank.secondphase" +rankprofile[15].fef.property[4].value "rankingExpression(secondphase)" +rankprofile[15].fef.property[5].name "rankingExpression(secondphase).rankingScript" +rankprofile[15].fef.property[5].value "3000 * rankingExpression(m2)" +rankprofile[16].name "macros-refering-macros-inherited2" +rankprofile[16].fef.property[0].name "rankingExpression(m1).rankingScript" +rankprofile[16].fef.property[0].value "700 * fieldMatch(title).completeness" +rankprofile[16].fef.property[1].name "rankingExpression(m2).rankingScript" +rankprofile[16].fef.property[1].value "rankingExpression(m1) * 67" +rankprofile[16].fef.property[2].name "rankingExpression(m4).rankingScript" +rankprofile[16].fef.property[2].value "703 * fieldMatch(fromfile).completeness" +rankprofile[16].fef.property[3].name "vespa.rank.secondphase" +rankprofile[16].fef.property[3].value "rankingExpression(secondphase)" +rankprofile[16].fef.property[4].name "rankingExpression(secondphase).rankingScript" +rankprofile[16].fef.property[4].value "3002 * rankingExpression(m2)" +rankprofile[17].name "macros-refering-macros-inherited-two-levels" +rankprofile[17].fef.property[0].name "rankingExpression(m1).rankingScript" +rankprofile[17].fef.property[0].value "700 * fieldMatch(title).completeness" +rankprofile[17].fef.property[1].name "rankingExpression(m2).rankingScript" +rankprofile[17].fef.property[1].value "rankingExpression(m1) * 67" +rankprofile[17].fef.property[2].name "rankingExpression(m4).rankingScript" +rankprofile[17].fef.property[2].value "701 * fieldMatch(title).completeness" +rankprofile[17].fef.property[3].name "rankingExpression(m3).rankingScript" +rankprofile[17].fef.property[3].value "if (isNan(attribute(nrtgmp)) == 1, 0.0, rankingExpression(m2))" +rankprofile[17].fef.property[4].name "rankingExpression(m5).rankingScript" +rankprofile[17].fef.property[4].value "if (isNan(attribute(glmpfw)) == 1, rankingExpression(m1), rankingExpression(m4))" +rankprofile[17].fef.property[5].name "vespa.rank.secondphase" +rankprofile[17].fef.property[5].value "rankingExpression(secondphase)" +rankprofile[17].fef.property[6].name "rankingExpression(secondphase).rankingScript" +rankprofile[17].fef.property[6].value "3000 * rankingExpression(m2)"
\ No newline at end of file diff --git a/model-inference/src/test/resources/config/rankexpression/rankexpression.sd b/model-inference/src/test/resources/config/rankexpression/rankexpression.sd new file mode 100644 index 00000000000..d3e0057cfe1 --- /dev/null +++ b/model-inference/src/test/resources/config/rankexpression/rankexpression.sd @@ -0,0 +1,327 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +search rankexpression { + + document rankexpression { + + field artist type string { + indexing: summary | index + } + + field title type string { + indexing: summary | index + } + + field surl type string { + indexing: summary + } + + field year type int { + indexing: summary | attribute + } + + field foo1 type int { + indexing: attribute + } + + field foo2 type int { + indexing: attribute + } + + field foo3 type int { + indexing: attribute + } + + field foo4 type int { + indexing: attribute + } + + field bar1 type int { + indexing: attribute + } + + field bar2 type int { + indexing: attribute + } + + field bar3 type int { + indexing: attribute + } + + field bar4 type int { + indexing: attribute + } + + } + + rank-profile default { + first-phase { + expression: classicRank + keep-rank-count: 20 + rank-score-drop-limit: -0.5 + } + second-phase { + expression: if(3>2,4,2) + rerank-count: 10 + } + rank-features: attribute(foo1).out attribute(bar1) + rank-features { attribute(foo2).out attribute(bar2).out } + rank-features { + attribute(foo3).out attribute(bar3).out } + rank-features { + attribute(foo4).out + attribute(bar4).out + } + ignore-default-rank-features + + rank-properties { + foo: "bar, baz" + qux: "quux" + foo: "foobar" + foo.bar: "foo.bar" + foo.bar.baz: 123 + foo ( bar ) . baz.2 : 123.4 + foo(bar).baz.qux: "foo(bar)" + "nud":"ity" + } + + } + + rank-profile static { + first-phase { + expression { attribute } + } + second-phase { + expression { + file:rankexpression + } + } + summary-features: attribute(foo1).out attribute(bar1) + summary-features { attribute(foo2).out attribute(bar2).out } + summary-features { + attribute(foo3).out attribute(bar3).out } + summary-features { + attribute(foo4).out + attribute(bar4).out + } + } + + rank-profile overflow { + first-phase { + expression: file:overflow.expression + keep-rank-count: 201 + rank-score-drop-limit: 501.5 + } + second-phase { + expression { + exp(0) + + mysum(attribute(foo), + "attribute( bar )", + "attribute( \"baz\" )") + } + rerank-count: 101 + } + } + + rank-profile duplicates { + rank-properties { + fieldMatch(a).proximityLimit: 4 + fieldMatch(a).proximityTable: 0.2 + fieldMatch(a).proximityTable: 0.4 + fieldMatch(a).proximityTable: 0.6 + fieldMatch(a).proximityTable: 0.8 + fieldMatch(a).proximityTable: 1 + fieldMatch(a).proximityTable: 0.8 + fieldMatch(a).proximityTable: 0.6 + fieldMatch(a).proximityTable: 0.4 + fieldMatch(a).proximityTable: 0.2 + } + } + + rank-profile whitespace1 { + first-phase { + expression + { + + 1 + }}} + + rank-profile whitespace2 { + first-phase + { + expression { 1 } + } + } + + rank-profile macros { + first-phase { + expression: match + fieldMatch(title) + myfeature + } + second-phase { + expression: fourtimessum(match,rankBoost) + } + macro fourtimessum(var1, var2) { + expression: 4*(var1+var2) + } + macro myfeature() { + expression { + 70 * fieldMatch(title).completeness * pow(0 - fieldMatch(title).earliness, 2) + + 30 * pow(0 - fieldMatch(description).earliness, 2) + } + } + summary-features { + fieldMatch(title) + } + } + + rank-profile macros2 { + first-phase { + expression: classicRank + } + rank-properties { + foo: "some, list" + } + + second-phase { + expression: fourtimessum(match,match) + mysummaryfeature + myfeature + } + macro fourtimessum(var1, var2) { + expression: 4*(var1+var2) + } + macro myfeature() { + expression { + 70 * fieldMatch(title).completeness * pow(0 - fieldMatch(title).earliness, 2) + + 30 * pow(0 - fieldMatch(description).earliness, 2) + } + } + macro mysummaryfeature() { + expression { + 70 * fieldMatch(title).completeness + } + } + macro mysummaryfeature2() { + expression { + 71 * fieldMatch(title).completeness + } + } + summary-features { + mysummaryfeature + rankingExpression(mysummaryfeature2) # Required form earlier + } + } + + rank-profile macros3 { + macro onlyusedinsummaryfeature() { + expression: 5 + } + summary-features { + rankingExpression(matches(title,rankingExpression(onlyusedinsummaryfeature))) + } + + } + + rank-profile macros3-inherited inherits macros3 { + summary-features { + rankingExpression(matches(title,rankingExpression(onlyusedinsummaryfeature))) + } + } + + rank-profile macros-inherited inherits macros2 { + macro mysummaryfeature() { + expression { + 80 * fieldMatch(title).completeness + } + } + first-phase { + expression { + 20000 * myfeature + mysummaryfeature + } + } + } + + rank-profile macros-inherited2 inherits macros-inherited { + first-phase { + expression { + 30000 * mysummaryfeature + myfeature + } + } + } + + rank-profile macros-inherited3 inherits macros-inherited2 { + macro myfeature() { + expression { + 700 * fieldMatch(title).completeness + } + } + second-phase { + expression { + 40000 * mysummaryfeature + myfeature + } + } + } + + rank-profile macros-refering-macros { + macro m2() { + expression: m1 * 67 + } + + macro m1() { + expression { + 700 * fieldMatch(title).completeness + } + } + + macro m4() { + expression: file:macro.expression + } + + second-phase { + expression { + 40000 * m2 + } + } + + } + + rank-profile macros-refering-macros-inherited inherits macros-refering-macros { + macro m3() { + expression { + if(isNan(attribute(nrtgmp))==1, + 0.0, + (m2) + ) + } + } + macro m4() { + expression { + 701 * fieldMatch(title).completeness + } + } + second-phase { + expression { + 3000 * m2 + } + } + } + + rank-profile macros-refering-macros-inherited2 inherits macros-refering-macros { + second-phase { + expression { + 3002 * m2 + } + } + } + + rank-profile macros-refering-macros-inherited-two-levels inherits macros-refering-macros-inherited { + macro m5() { + expression { + if(isNan(attribute(glmpfw))==1, + m1, + (m4) + ) + } + } + } + +} + + @@ -94,6 +94,7 @@ <module>messagebus-disc</module> <module>messagebus</module> <module>metrics</module> + <module>model-inference</module> <module>node-repository</module> <module>node-admin</module> <module>node-maintainer</module> diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/ExpressionFunction.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/ExpressionFunction.java index 9267034e985..f2239776d48 100755 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/ExpressionFunction.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/ExpressionFunction.java @@ -110,7 +110,7 @@ public class ExpressionFunction { @Override public String toString() { - return name; + return "function '" + name + "'"; } /** diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/RankingExpression.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/RankingExpression.java index f03a8bea583..38e642c8145 100755 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/RankingExpression.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/RankingExpression.java @@ -57,10 +57,10 @@ ArrayContext contextPrototype; // Create reusable, gbdt optimized expression and context. // The expression is multithread-safe while the context created is not try { - RankingExpression expression=new RankingExpression("10*if(i>35,if(i>one,if(i>=670,4,8),if(i>8000,5,3)),if(i==478,90,91))"); - ArrayContext contextPrototype=new ArrayContext(expression); - ExpressionOptimizer optimizer=new ExpressionOptimizer(); // Increases evaluation speed of gbdt form expressions by 3-4x - OptimizationReport triviaAboutTheOptimization=optimizer.optimize(expression,contextPrototype); + RankingExpression expression = new RankingExpression("10*if(i>35,if(i>one,if(i>=670,4,8),if(i>8000,5,3)),if(i==478,90,91))"); + ArrayContext contextPrototype = new ArrayContext(expression); + ExpressionOptimizer optimizer = new ExpressionOptimizer(); // Increases evaluation speed of gbdt form expressions by 3-4x + OptimizationReport triviaAboutTheOptimization = optimizer.optimize(expression,contextPrototype); } catch (ParseException e) { throw new RuntimeException(e); @@ -69,9 +69,9 @@ catch (ParseException e) { ... // Execution (many) -context=contextPrototype.clone(); // If evaluation is multithreaded - skip this if execution is single-threaded +context = contextPrototype.clone(); // If evaluation is multithreaded - skip this if execution is single-threaded context.put("one",1d); -double result=expression.evaluate(context); +double result = expression.evaluate(context); </code></pre> * * @author Simon Thoresen Hult @@ -276,7 +276,7 @@ public class RankingExpression implements Serializable { * Returns the value of evaluating this expression over the given context. * * @param context The variable bindings to use for this evaluation. - * @return The evaluation result. + * @return the evaluation result. * @throws IllegalArgumentException if there are variables which are not bound in the given map */ public Value evaluate(Context context) { diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/AbstractArrayContext.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/AbstractArrayContext.java index ed9fa346c11..41bf827748a 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/AbstractArrayContext.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/AbstractArrayContext.java @@ -7,8 +7,6 @@ import com.yahoo.searchlib.rankingexpression.rule.CompositeNode; import com.yahoo.searchlib.rankingexpression.rule.ExpressionNode; import com.yahoo.searchlib.rankingexpression.rule.ReferenceNode; -import java.util.Collections; -import java.util.HashMap; import java.util.LinkedHashSet; import java.util.Map; import java.util.Set; @@ -20,19 +18,15 @@ import java.util.Set; * * @author bratseth */ -public abstract class AbstractArrayContext extends Context implements Cloneable { +public abstract class AbstractArrayContext extends Context implements Cloneable, ContextIndex { private final boolean ignoreUnknownValues; - /** The mapping from variable name to index */ - private final ImmutableMap<String, Integer> nameToIndex; - - /** The current values set, pre-converted to doubles */ - private double[] doubleValues; - /** The name of the ranking expression this was created for */ private final String rankingExpressionName; + private IndexedBindings indexedBindings; + /** * Create a fast lookup context for an expression. * This instance should be reused indefinitely by a single thread. @@ -53,44 +47,51 @@ public abstract class AbstractArrayContext extends Context implements Cloneable protected AbstractArrayContext(RankingExpression expression, boolean ignoreUnknownValues) { this.ignoreUnknownValues = ignoreUnknownValues; this.rankingExpressionName = expression.getName(); - Set<String> variables = new LinkedHashSet<>(); - extractVariables(expression.getRoot(),variables); + this.indexedBindings = new IndexedBindings(expression); + } - doubleValues = new double[variables.size()]; + protected final Map<String, Integer> nameToIndex() { return indexedBindings.nameToIndex(); } + protected final double[] doubleValues() { return indexedBindings.doubleValues(); } + protected final boolean ignoreUnknownValues() { return ignoreUnknownValues; } - int i = 0; - ImmutableMap.Builder<String, Integer> nameToIndexBuilder = new ImmutableMap.Builder<>(); - for (String variable : variables) - nameToIndexBuilder.put(variable,i++); - nameToIndex = nameToIndexBuilder.build(); + @Override + public Set<String> names() { + return indexedBindings.names(); } - private void extractVariables(ExpressionNode node,Set<String> variables) { - if (node instanceof ReferenceNode) { - ReferenceNode fNode=(ReferenceNode)node; - if (fNode.getArguments().expressions().size()>0) - throw new UnsupportedOperationException("Array lookup is not supported with features having arguments)"); - variables.add(fNode.toString()); - } - else if (node instanceof CompositeNode) { - CompositeNode cNode=(CompositeNode)node; - for (ExpressionNode child : cNode.children()) - extractVariables(child,variables); - } + /** + * Returns the index from a name. + * + * @throws NullPointerException is this name is not known to this context + */ + @Override + public final int getIndex(String name) { return indexedBindings.nameToIndex.get(name); } + + /** Returns the max number of variables which may be set in this */ + @Override + public int size() { return indexedBindings.size(); } + + /** Perform a fast lookup directly of the value as a double. This is faster than get(index).asDouble() */ + @Override + public double getDouble(int index) { + return indexedBindings.getDouble(index); } - protected final Map<String, Integer> nameToIndex() { return nameToIndex; } - protected final double[] doubleValues() { return doubleValues; } - protected final boolean ignoreUnknownValues() { return ignoreUnknownValues; } + @Override + public String toString() { + return "fast lookup context for ranking expression '" + rankingExpressionName + + "' [" + size() + " variables]"; + } /** * Creates a clone of this context suitable for evaluating against the same ranking expression * in a different thread (i.e, name name to index map, different value set. */ + @Override public AbstractArrayContext clone() { try { - AbstractArrayContext clone=(AbstractArrayContext)super.clone(); - clone.doubleValues=new double[nameToIndex.size()]; + AbstractArrayContext clone = (AbstractArrayContext)super.clone(); + clone.indexedBindings = indexedBindings.clone(); return clone; } catch (CloneNotSupportedException e) { @@ -98,34 +99,64 @@ public abstract class AbstractArrayContext extends Context implements Cloneable } } - public Set<String> names() { - return nameToIndex.keySet(); - } + private static class IndexedBindings implements Cloneable { - /** - * Returns the index from a name. - * - * @throws NullPointerException is this name is not known to this context - */ - public final int getIndex(String name) { - return nameToIndex.get(name); - } + /** The mapping from variable name to index */ + private final ImmutableMap<String, Integer> nameToIndex; - /** Returns the max number of variables which may be set in this */ - public int size() { - return doubleValues.length; - } + /** The current values set, pre-converted to doubles */ + private double[] doubleValues; - /** Perform a fast lookup directly of the value as a double. This is faster than get(index).asDouble() */ - @Override - public double getDouble(int index) { - return doubleValues[index]; - } + public IndexedBindings(RankingExpression expression) { + Set<String> bindTargets = new LinkedHashSet<>(); + extractBindTargets(expression.getRoot(), bindTargets); + + doubleValues = new double[bindTargets.size()]; + + int i = 0; + ImmutableMap.Builder<String, Integer> nameToIndexBuilder = new ImmutableMap.Builder<>(); + for (String variable : bindTargets) + nameToIndexBuilder.put(variable,i++); + nameToIndex = nameToIndexBuilder.build(); + } + + private void extractBindTargets(ExpressionNode node, Set<String> bindTargets) { + if (node instanceof ReferenceNode) { + if (((ReferenceNode)node).getArguments().expressions().size() > 0) + throw new UnsupportedOperationException("Can not bind " + node + + ": Array lookup is not supported with features having arguments)"); + bindTargets.add(node.toString()); + } + else if (node instanceof CompositeNode) { + CompositeNode cNode = (CompositeNode)node; + for (ExpressionNode child : cNode.children()) + extractBindTargets(child, bindTargets); + } + } + + public Map<String, Integer> nameToIndex() { return nameToIndex; } + public double[] doubleValues() { return doubleValues; } + public Set<String> names() { return nameToIndex.keySet(); } + public int getIndex(String name) { return nameToIndex.get(name); } + public int size() { return doubleValues.length; } + public double getDouble(int index) { return doubleValues[index]; } + + /** + * Creates a clone of this context suitable for evaluating against the same ranking expression + * in a different thread (i.e, name name to index map, different value set. + */ + @Override + public IndexedBindings clone() { + try { + IndexedBindings clone = (IndexedBindings)super.clone(); + clone.doubleValues = new double[nameToIndex.size()]; + return clone; + } + catch (CloneNotSupportedException e) { + throw new RuntimeException("Programming error"); + } + } - @Override - public String toString() { - return "fast lookup context for ranking expression '" + rankingExpressionName + - "' [" + doubleValues.length + " variables]"; } } diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/ArrayContext.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/ArrayContext.java index ee5952d9aea..237c3a1d0b1 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/ArrayContext.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/ArrayContext.java @@ -119,7 +119,7 @@ public class ArrayContext extends AbstractArrayContext implements Cloneable { public ArrayContext clone() { ArrayContext clone = (ArrayContext)super.clone(); clone.values = new Value[nameToIndex().size()]; - Arrays.fill(values,constantZero); + Arrays.fill(values, constantZero); return clone; } diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/ContextIndex.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/ContextIndex.java new file mode 100644 index 00000000000..ad6facbf0af --- /dev/null +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/ContextIndex.java @@ -0,0 +1,26 @@ +package com.yahoo.searchlib.rankingexpression.evaluation; + +/** + * Indexed context lookup methods. + * Any context which implements these methods supports optimizations where map lookups + * are replaced by indexed lookups. + * + * @author bratseth + */ +public interface ContextIndex { + + /** Returns the number of bound variables in this */ + int size(); + + /** + * Returns the index from a name. + * + * @throws NullPointerException is this name is not known to this context + */ + int getIndex(String name); + + Value get(int index); + + double getDouble(int index); + +} diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/ExpressionOptimizer.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/ExpressionOptimizer.java index 836aadd9f70..b82173eabd5 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/ExpressionOptimizer.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/ExpressionOptimizer.java @@ -44,7 +44,7 @@ public class ExpressionOptimizer { return null; } - public OptimizationReport optimize(RankingExpression expression, AbstractArrayContext arrayContext) { + public OptimizationReport optimize(RankingExpression expression, ContextIndex arrayContext) { OptimizationReport report = new OptimizationReport(); // Note: Order of optimizations matter gbdtOptimizer.optimize(expression, arrayContext, report); diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/Optimizer.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/Optimizer.java index fd9c02100f7..044b5b589a5 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/Optimizer.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/Optimizer.java @@ -18,6 +18,6 @@ public abstract class Optimizer { /** Returns whether this is enabled */ public boolean isEnabled() { return enabled; } - public abstract void optimize(RankingExpression expression, AbstractArrayContext context, OptimizationReport report); + public abstract void optimize(RankingExpression expression, ContextIndex context, OptimizationReport report); } diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/Value.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/Value.java index e5a9e6a5ef1..7809cdd4e1b 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/Value.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/Value.java @@ -115,4 +115,12 @@ public abstract class Value { return new DoubleValue(Double.parseDouble(value)); } + public static Value of(Tensor tensor) { + return new TensorValue(tensor); + } + + public static Value of(double scalar) { + return new DoubleValue(scalar); + } + } diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/gbdtoptimization/GBDTForestOptimizer.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/gbdtoptimization/GBDTForestOptimizer.java index 8999be4745a..bb8b91eecab 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/gbdtoptimization/GBDTForestOptimizer.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/gbdtoptimization/GBDTForestOptimizer.java @@ -2,8 +2,7 @@ package com.yahoo.searchlib.rankingexpression.evaluation.gbdtoptimization; import com.yahoo.searchlib.rankingexpression.RankingExpression; -import com.yahoo.searchlib.rankingexpression.evaluation.AbstractArrayContext; -import com.yahoo.searchlib.rankingexpression.evaluation.ArrayContext; +import com.yahoo.searchlib.rankingexpression.evaluation.ContextIndex; import com.yahoo.searchlib.rankingexpression.evaluation.OptimizationReport; import com.yahoo.searchlib.rankingexpression.evaluation.Optimizer; import com.yahoo.searchlib.rankingexpression.rule.ArithmeticNode; @@ -34,7 +33,7 @@ public class GBDTForestOptimizer extends Optimizer { * @param report the optimization report to which actions of this is logged */ @Override - public void optimize(RankingExpression expression, AbstractArrayContext context, OptimizationReport report) { + public void optimize(RankingExpression expression, ContextIndex context, OptimizationReport report) { if ( ! isEnabled()) return; this.report = report; diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/gbdtoptimization/GBDTOptimizer.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/gbdtoptimization/GBDTOptimizer.java index 74af3e576c1..787818b0f42 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/gbdtoptimization/GBDTOptimizer.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/gbdtoptimization/GBDTOptimizer.java @@ -33,7 +33,7 @@ public class GBDTOptimizer extends Optimizer { * @param report the optimization report to which actions of this is logged */ @Override - public void optimize(RankingExpression expression, AbstractArrayContext context, OptimizationReport report) { + public void optimize(RankingExpression expression, ContextIndex context, OptimizationReport report) { if (!isEnabled()) return; this.report = report; @@ -60,7 +60,7 @@ public class GBDTOptimizer extends Optimizer { * * @return the optimized expression */ - private ExpressionNode optimize(ExpressionNode node, AbstractArrayContext context) { + private ExpressionNode optimize(ExpressionNode node, ContextIndex context) { if (node instanceof ArithmeticNode) { Iterator<ExpressionNode> childIt = ((ArithmeticNode)node).children().iterator(); ExpressionNode ret = optimize(childIt.next(), context); @@ -77,7 +77,7 @@ public class GBDTOptimizer extends Optimizer { return node; } - private ExpressionNode createGBDTNode(IfNode cNode, AbstractArrayContext context) { + private ExpressionNode createGBDTNode(IfNode cNode,ContextIndex context) { List<Double> values = new ArrayList<>(); try { consumeNode(cNode, values, context); @@ -93,7 +93,7 @@ public class GBDTOptimizer extends Optimizer { /** * Recursively consume nodes into the value list Returns the number of values produced by this. */ - private int consumeNode(ExpressionNode node, List<Double> values, AbstractArrayContext context) { + private int consumeNode(ExpressionNode node, List<Double> values, ContextIndex context) { int beforeIndex = values.size(); if ( node instanceof IfNode) { IfNode ifNode = (IfNode)node; @@ -113,7 +113,7 @@ public class GBDTOptimizer extends Optimizer { } /** Consumes the if condition and return the size of the values resulting, for convenience */ - private int consumeIfCondition(ExpressionNode condition, List<Double> values, AbstractArrayContext context) { + private int consumeIfCondition(ExpressionNode condition, List<Double> values, ContextIndex context) { if (condition instanceof ComparisonNode) { ComparisonNode comparison = (ComparisonNode)condition; if (comparison.getOperator() == TruthOperator.SMALLER) @@ -138,7 +138,7 @@ public class GBDTOptimizer extends Optimizer { return values.size(); } - private double getVariableIndex(ExpressionNode node, AbstractArrayContext context) { + private double getVariableIndex(ExpressionNode node, ContextIndex context) { if (!(node instanceof ReferenceNode)) { throw new IllegalArgumentException("Contained a left-hand comparison expression " + "which was not a feature value but was: " + node); |