diff options
author | Jon Bratseth <bratseth@gmail.com> | 2024-03-31 23:16:14 +0200 |
---|---|---|
committer | GitHub <noreply@github.com> | 2024-03-31 23:16:14 +0200 |
commit | 273d965d50979f84431a5575a04c86baed3b5a80 (patch) | |
tree | 9deeeea285fb3ef92ee90cb3ccf31fbce4415bb3 | |
parent | 74b0626d7e9f9b2b007a454587ef6df78d85b61e (diff) | |
parent | a3761f8fb2e72f3185011809fe21442cbe9378c1 (diff) |
Merge pull request #30766 from vespa-engine/bratseth/indexing-statement-cachingv8.325.46
Bratseth/indexing statement caching
5 files changed, 111 insertions, 41 deletions
diff --git a/config-model/src/main/java/com/yahoo/schema/derived/IndexingScript.java b/config-model/src/main/java/com/yahoo/schema/derived/IndexingScript.java index 60d27c617f1..39879f2bed7 100644 --- a/config-model/src/main/java/com/yahoo/schema/derived/IndexingScript.java +++ b/config-model/src/main/java/com/yahoo/schema/derived/IndexingScript.java @@ -30,7 +30,7 @@ import java.util.List; import java.util.Set; /** - * An indexing language script derived from a search definition. An indexing script contains a set of indexing + * An indexing language script derived from a schema. An indexing script contains a set of indexing * statements, organized in a composite structure of indexing code snippets. * * @author bratseth @@ -62,12 +62,8 @@ public final class IndexingScript extends Derived { if (field.hasFullIndexingDocprocRights()) docFields.add(field.getName()); - if (field.usesStructOrMap() && ! GeoPos.isAnyPos(field)) { - return; // unsupported - } - - if (fieldsSettingLanguage.size() == 1 && fieldsSettingLanguage.get(0).equals(field)) - return; // Already added + if (field.usesStructOrMap() && ! GeoPos.isAnyPos(field)) return; // unsupported + if (fieldsSettingLanguage.size() == 1 && fieldsSettingLanguage.get(0).equals(field)) return; // Already added addExpression(field.getIndexingScript()); } diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ExecutionContext.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ExecutionContext.java index b0478f8001c..1935664cddc 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ExecutionContext.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ExecutionContext.java @@ -10,16 +10,18 @@ import com.yahoo.language.detect.Detection; import java.util.HashMap; import java.util.Map; +import java.util.Objects; /** * @author Simon Thoresen Hult */ -public class ExecutionContext implements FieldTypeAdapter, FieldValueAdapter, Cloneable { +public class ExecutionContext implements FieldTypeAdapter, FieldValueAdapter { private final Map<String, FieldValue> variables = new HashMap<>(); private final FieldValueAdapter adapter; private FieldValue value; private Language language; + private Map<String, Object> cache = null; public ExecutionContext() { this(null); @@ -40,7 +42,9 @@ public class ExecutionContext implements FieldTypeAdapter, FieldValueAdapter, Cl * Returns whether this is for a complete execution of all statements of a script, * or a partial execution of only the statements accessing the available data. */ - public boolean isComplete() { return adapter == null ? false : adapter.isComplete(); } + public boolean isComplete() { + return adapter != null && adapter.isComplete(); + } @Override public DataType getInputType(Expression exp, String fieldName) { @@ -89,43 +93,45 @@ public class ExecutionContext implements FieldTypeAdapter, FieldValueAdapter, Cl return this; } - public Language getLanguage() { - return language; - } + public Language getLanguage() { return language; } public ExecutionContext setLanguage(Language language) { - language.getClass(); - this.language = language; + this.language = Objects.requireNonNull(language); return this; } public Language resolveLanguage(Linguistics linguistics) { - if (language != null && language != Language.UNKNOWN) { - return language; - } - if (linguistics == null) { - return Language.ENGLISH; - } + if (language != null && language != Language.UNKNOWN) return language; + if (linguistics == null) return Language.ENGLISH; + Detection detection = linguistics.getDetector().detect(String.valueOf(value), null); - if (detection == null) { - return Language.ENGLISH; - } + if (detection == null) return Language.ENGLISH; + Language detected = detection.getLanguage(); - if (detected == Language.UNKNOWN) { - return Language.ENGLISH; - } + if (detected == Language.UNKNOWN) return Language.ENGLISH; return detected; } - public FieldValue getValue() { - return value; - } + public FieldValue getValue() { return value; } public ExecutionContext setValue(FieldValue value) { this.value = value; return this; } + public void putCachedValue(String key, Object value) { + if (cache == null) + cache = new HashMap<>(); + cache.put(key, value); + } + + /** Returns a cached value, or null if not present. */ + public Object getCachedValue(String key) { + if (cache == null) return null; + return cache.get(key); + } + + /** Clears all state in this except the cache. */ public ExecutionContext clear() { variables.clear(); value = null; diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ScriptExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ScriptExpression.java index a88e56939ee..7d180b9fd7a 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ScriptExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ScriptExpression.java @@ -51,7 +51,8 @@ public final class ScriptExpression extends ExpressionList<StatementExpression> for (StatementExpression statement : this) { if (context.isComplete() || (statement.getInputFields().isEmpty() || containsAtLeastOneInputFrom(statement.getInputFields(), context))) { - context.setValue(input).execute(statement); + context.setValue(input); + context.execute(statement); } } context.setValue(input); diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ScriptTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ScriptTestCase.java index 75f852f0331..df7f99d22d2 100644 --- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ScriptTestCase.java +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ScriptTestCase.java @@ -75,6 +75,20 @@ public class ScriptTestCase { } @Test + public void testCache() { + SimpleTestAdapter adapter = new SimpleTestAdapter(new Field("field1", DataType.STRING)); + var script = newScript(newStatement(new InputExpression("field1"), + new PutCacheExpression("myCacheKey", "myCacheValue")), + newStatement(new ClearStateExpression()), // inserted by config model + newStatement(new InputExpression("field1"), + new AssertCacheExpression("myCacheKey", "myCacheValue"))); + adapter.setValue("field1", new StringFieldValue("foo1")); + ExecutionContext context = new ExecutionContext(adapter); + script.execute(context); + assertEquals("myCacheValue", context.getCachedValue("myCacheKey")); + } + + @Test public void requireThatStatementsProcessingMissingInputsAreSkipped() { SimpleTestAdapter adapter = new SimpleTestAdapter(new Field("foo", DataType.STRING), new Field("bar", DataType.STRING)); @@ -155,4 +169,52 @@ public class ScriptTestCase { } + private static class PutCacheExpression extends Expression { + + private final String keyToSet; + private final String valueToSet; + + public PutCacheExpression(String keyToSet, String valueToSet) { + super(null); + this.keyToSet = keyToSet; + this.valueToSet = valueToSet; + } + + @Override + protected void doExecute(ExecutionContext context) { + context.putCachedValue(keyToSet, valueToSet); + } + + @Override + protected void doVerify(VerificationContext context) {} + + @Override + public DataType createdOutputType() { return null; } + + } + + private static class AssertCacheExpression extends Expression { + + private final String expectedKey; + private final String expectedValue; + + public AssertCacheExpression(String expectedKey, String expectedValue) { + super(null); + this.expectedKey = expectedKey; + this.expectedValue = expectedValue; + } + + @Override + protected void doExecute(ExecutionContext context) { + assertEquals(expectedValue, context.getCachedValue(expectedKey)); + } + + @Override + protected void doVerify(VerificationContext context) {} + + @Override + public DataType createdOutputType() { return null; } + + } + } diff --git a/vespajlib/src/test/java/com/yahoo/slime/JsonBenchmark.java b/vespajlib/src/test/java/com/yahoo/slime/JsonBenchmark.java index 4f77a933a74..ee755a44010 100644 --- a/vespajlib/src/test/java/com/yahoo/slime/JsonBenchmark.java +++ b/vespajlib/src/test/java/com/yahoo/slime/JsonBenchmark.java @@ -15,10 +15,11 @@ import java.lang.Integer; * @author baldersheim */ public class JsonBenchmark { - private static byte [] createJson(int numElements) { + + private static byte[] createJson(int numElements) { Slime slime = new Slime(); Cursor a = slime.setArray(); - for (int i=0; i < numElements; i++) { + for (int i = 0; i < numElements; i++) { Cursor e = a.addObject(); e.setString("key", "i"); e.setLong("weight", i); @@ -32,12 +33,13 @@ public class JsonBenchmark { } return bs.toByteArray(); } - private static long benchmarkJacksonStreaming(byte [] json, int numIterations) { + + private static long benchmarkJacksonStreaming(byte[] json, int numIterations) { long count = 0; JsonFactory jsonFactory = new JsonFactory(); try { - for (int i=0; i < numIterations; i++) { + for (int i = 0; i < numIterations; i++) { try (JsonParser jsonParser = jsonFactory.createParser(json)) { JsonToken array = jsonParser.nextToken(); for (JsonToken token = jsonParser.nextToken(); !JsonToken.END_ARRAY.equals(token); token = jsonParser.nextToken()) { @@ -53,12 +55,13 @@ public class JsonBenchmark { } return count; } - private static long benchmarkJacksonTree(byte [] json, int numIterations) { + + private static long benchmarkJacksonTree(byte[] json, int numIterations) { long count = 0; // use the ObjectMapper to read the json string and create a tree var mapper = Jackson.mapper(); try { - for (int i=0; i < numIterations; i++) { + for (int i = 0; i < numIterations; i++) { JsonNode node = mapper.readTree(json); for(JsonNode item : node) { count += item.get("weight").asLong(); @@ -69,9 +72,10 @@ public class JsonBenchmark { } return count; } - private static long benchmarkSlime(byte [] json, int numIterations) { + + private static long benchmarkSlime(byte[] json, int numIterations) { long count = 0; - for (int i=0; i < numIterations; i++) { + for (int i = 0; i < numIterations; i++) { JsonDecoder decoder = new JsonDecoder(); Slime slime = decoder.decode(new Slime(), json); @@ -83,7 +87,7 @@ public class JsonBenchmark { } return count; } - private static void warmup(byte [] json) { + private static void warmup(byte[] json) { System.out.println(System.currentTimeMillis() + " Warming up"); benchmarkSlime(json, 5000); System.out.println(System.currentTimeMillis() + " Done Warming up"); @@ -95,9 +99,9 @@ public class JsonBenchmark { * slime 1000 40000 = 17.5 seconds * @param argv type, num elements in weigted set, num iterations */ - static public void main(String [] argv) { + static public void main(String[] argv) { String type = argv[0]; - byte [] json = createJson(Integer.parseInt(argv[1])); + byte[] json = createJson(Integer.parseInt(argv[1])); warmup(json); int count = Integer.parseInt(argv[2]); System.out.println(System.currentTimeMillis() + " Start"); @@ -112,4 +116,5 @@ public class JsonBenchmark { } System.out.println(System.currentTimeMillis() + " End with " + numValues + " values in " + (System.currentTimeMillis() - start) + " milliseconds."); } + } |