From 54c6c15bcc460db3066c78ccf2e4167c11442f38 Mon Sep 17 00:00:00 2001 From: Henning Baldersheim Date: Fri, 12 Jan 2024 20:38:47 +0100 Subject: Revert "Drop tokenize expressions from ilscript for streaming mode." --- .../src/main/java/com/yahoo/schema/Schema.java | 1 + .../yahoo/schema/derived/DerivedConfiguration.java | 2 +- .../com/yahoo/schema/derived/IndexingScript.java | 33 ++++------------- .../yahoo/schema/AttributeSettingsTestCase.java | 2 +- .../schema/derived/IndexingScriptTestCase.java | 41 ---------------------- .../yahoo/schema/derived/VsmFieldsTestCase.java | 2 +- .../schema/processing/AssertIndexingScript.java | 2 +- .../expressions/StatementExpression.java | 3 +- 8 files changed, 13 insertions(+), 73 deletions(-) delete mode 100644 config-model/src/test/java/com/yahoo/schema/derived/IndexingScriptTestCase.java diff --git a/config-model/src/main/java/com/yahoo/schema/Schema.java b/config-model/src/main/java/com/yahoo/schema/Schema.java index 3402ba31be9..279b5729ea1 100644 --- a/config-model/src/main/java/com/yahoo/schema/Schema.java +++ b/config-model/src/main/java/com/yahoo/schema/Schema.java @@ -18,6 +18,7 @@ import com.yahoo.schema.document.SDDocumentType; import com.yahoo.schema.document.SDField; import com.yahoo.schema.document.Stemming; import com.yahoo.schema.document.TemporaryImportedFields; +import com.yahoo.schema.document.annotation.SDAnnotationType; import com.yahoo.searchlib.rankingexpression.Reference; import com.yahoo.vespa.documentmodel.DocumentSummary; import com.yahoo.vespa.documentmodel.SummaryField; diff --git a/config-model/src/main/java/com/yahoo/schema/derived/DerivedConfiguration.java b/config-model/src/main/java/com/yahoo/schema/derived/DerivedConfiguration.java index 575b7264628..7f874d04f13 100644 --- a/config-model/src/main/java/com/yahoo/schema/derived/DerivedConfiguration.java +++ b/config-model/src/main/java/com/yahoo/schema/derived/DerivedConfiguration.java @@ -80,7 +80,7 @@ public class DerivedConfiguration implements AttributesConfig.Producer { summaries = new Summaries(schema, deployState.getDeployLogger(), deployState.getProperties().featureFlags()); juniperrc = new Juniperrc(schema); rankProfileList = new RankProfileList(schema, schema.rankExpressionFiles(), attributeFields, deployState); - indexingScript = new IndexingScript(schema, isStreaming); + indexingScript = new IndexingScript(schema); indexInfo = new IndexInfo(schema, isStreaming); schemaInfo = new SchemaInfo(schema, deployState.rankProfileRegistry(), summaries); indexSchema = new IndexSchema(schema); diff --git a/config-model/src/main/java/com/yahoo/schema/derived/IndexingScript.java b/config-model/src/main/java/com/yahoo/schema/derived/IndexingScript.java index 6f0420f5203..73f3507ab00 100644 --- a/config-model/src/main/java/com/yahoo/schema/derived/IndexingScript.java +++ b/config-model/src/main/java/com/yahoo/schema/derived/IndexingScript.java @@ -6,7 +6,6 @@ import com.yahoo.schema.document.GeoPos; import com.yahoo.schema.document.ImmutableSDField; import com.yahoo.vespa.configdefinition.IlscriptsConfig; import com.yahoo.vespa.configdefinition.IlscriptsConfig.Ilscript.Builder; -import com.yahoo.vespa.indexinglanguage.ExpressionConverter; import com.yahoo.vespa.indexinglanguage.ExpressionVisitor; import com.yahoo.vespa.indexinglanguage.expressions.ClearStateExpression; import com.yahoo.vespa.indexinglanguage.expressions.Expression; @@ -17,7 +16,6 @@ import com.yahoo.vespa.indexinglanguage.expressions.PassthroughExpression; import com.yahoo.vespa.indexinglanguage.expressions.ScriptExpression; import com.yahoo.vespa.indexinglanguage.expressions.SetLanguageExpression; import com.yahoo.vespa.indexinglanguage.expressions.StatementExpression; -import com.yahoo.vespa.indexinglanguage.expressions.TokenizeExpression; import com.yahoo.vespa.indexinglanguage.expressions.ZCurveExpression; import java.util.ArrayList; @@ -38,10 +36,8 @@ public final class IndexingScript extends Derived implements IlscriptsConfig.Pro private final List docFields = new ArrayList<>(); private final List expressions = new ArrayList<>(); private List fieldsSettingLanguage; - private final boolean isStreaming; - public IndexingScript(Schema schema, boolean isStreaming) { - this.isStreaming = isStreaming; + public IndexingScript(Schema schema) { derive(schema); } @@ -100,38 +96,21 @@ public final class IndexingScript extends Derived implements IlscriptsConfig.Pro configBuilder.ilscript(ilscriptBuilder); } - private static class DropTokenize extends ExpressionConverter { - @Override - protected boolean shouldConvert(Expression exp) { - return exp instanceof TokenizeExpression; - } - - @Override - protected Expression doConvert(Expression exp) { - return null; - } - } - private void addContentInOrder(IlscriptsConfig.Ilscript.Builder ilscriptBuilder) { ArrayList later = new ArrayList<>(); Set touchedFields = new HashSet<>(); for (Expression expression : expressions) { - if (isStreaming) { - expression = expression.convertChildren(new DropTokenize()); - } - if (modifiesSelf(expression) && ! setsLanguage(expression)) { + if (modifiesSelf(expression) && ! setsLanguage(expression)) later.add(expression); - } else { + else ilscriptBuilder.content(expression.toString()); - } FieldScanVisitor fieldFetcher = new FieldScanVisitor(); fieldFetcher.visit(expression); touchedFields.addAll(fieldFetcher.touchedFields()); } - for (Expression exp : later) { + for (Expression exp : later) ilscriptBuilder.content(exp.toString()); - } generateSyntheticStatementsForUntouchedFields(ilscriptBuilder, touchedFields); } @@ -192,8 +171,8 @@ public final class IndexingScript extends Derived implements IlscriptsConfig.Pro } private static class FieldScanVisitor extends ExpressionVisitor { - List touchedFields = new ArrayList<>(); - List candidates = new ArrayList<>(); + List touchedFields = new ArrayList(); + List candidates = new ArrayList(); @Override protected void doVisit(Expression exp) { diff --git a/config-model/src/test/java/com/yahoo/schema/AttributeSettingsTestCase.java b/config-model/src/test/java/com/yahoo/schema/AttributeSettingsTestCase.java index 6624a7d3b32..ceac2b94997 100644 --- a/config-model/src/test/java/com/yahoo/schema/AttributeSettingsTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/AttributeSettingsTestCase.java @@ -280,7 +280,7 @@ public class AttributeSettingsTestCase extends AbstractSchemaTestCase { @Test void requireThatMutableIsAllowedThroughIndexing() throws ParseException { - IndexingScript script = new IndexingScript(getSearchWithMutables(), false); + IndexingScript script = new IndexingScript(getSearchWithMutables()); IlscriptsConfig.Builder builder = new IlscriptsConfig.Builder(); script.getConfig(builder); IlscriptsConfig cfg = builder.build(); diff --git a/config-model/src/test/java/com/yahoo/schema/derived/IndexingScriptTestCase.java b/config-model/src/test/java/com/yahoo/schema/derived/IndexingScriptTestCase.java deleted file mode 100644 index 81536bf99fb..00000000000 --- a/config-model/src/test/java/com/yahoo/schema/derived/IndexingScriptTestCase.java +++ /dev/null @@ -1,41 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.schema.derived; - -import com.yahoo.document.DataType; -import com.yahoo.schema.Schema; -import com.yahoo.schema.document.MatchType; -import com.yahoo.schema.document.Matching; -import com.yahoo.schema.document.SDField; -import com.yahoo.schema.document.TemporarySDField; -import com.yahoo.vespa.configdefinition.IlscriptsConfig; -import org.junit.jupiter.api.Test; - -import static org.junit.jupiter.api.Assertions.assertEquals; - -public class IndexingScriptTestCase { - - private static IlscriptsConfig ilscriptsConfig(Schema schema, boolean isStreaming) { - IndexingScript script = new IndexingScript(schema, isStreaming); - IlscriptsConfig.Builder cfgBuilder = new IlscriptsConfig.Builder(); - script.getConfig(cfgBuilder); - return cfgBuilder.build(); - } - - private void verifyIndexingScript(boolean isStreaming) { - Schema schema = VsmFieldsTestCase.createSchema(); - SDField field = new TemporarySDField(schema.getDocument(), "f", DataType.STRING); - field.parseIndexingScript("{ tokenize | index }"); - field.setMatching(new Matching(MatchType.TEXT)); - schema.getDocument().addField(field); - IlscriptsConfig cfg = ilscriptsConfig(schema, isStreaming); - assertEquals(1, cfg.ilscript().size()); - assertEquals(2, cfg.ilscript(0).content().size()); - String indexing = isStreaming ? "index" : "tokenize | index"; - assertEquals("clear_state | guard { " + indexing + "; }", cfg.ilscript(0).content(0)); - } - @Test - void testThatTokenizeIsIgnoredFromStreaming() { - verifyIndexingScript(false); - verifyIndexingScript(true); - } -} diff --git a/config-model/src/test/java/com/yahoo/schema/derived/VsmFieldsTestCase.java b/config-model/src/test/java/com/yahoo/schema/derived/VsmFieldsTestCase.java index 042feeb5f46..e469b80ab40 100644 --- a/config-model/src/test/java/com/yahoo/schema/derived/VsmFieldsTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/derived/VsmFieldsTestCase.java @@ -23,7 +23,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; */ public class VsmFieldsTestCase { - static Schema createSchema() { + private static Schema createSchema() { Schema schema = new Schema("test", MockApplicationPackage.createEmpty(), new MockFileRegistry(), new TestableDeployLogger(), new TestProperties()); var sdoc = new SDDocumentType("test"); schema.addDocument(sdoc); diff --git a/config-model/src/test/java/com/yahoo/schema/processing/AssertIndexingScript.java b/config-model/src/test/java/com/yahoo/schema/processing/AssertIndexingScript.java index b82dc197838..8aeb11aaa2e 100644 --- a/config-model/src/test/java/com/yahoo/schema/processing/AssertIndexingScript.java +++ b/config-model/src/test/java/com/yahoo/schema/processing/AssertIndexingScript.java @@ -18,7 +18,7 @@ import static org.junit.jupiter.api.Assertions.fail; public abstract class AssertIndexingScript { public static void assertIndexing(List expected, Schema schema) { - assertIndexing(expected, new IndexingScript(schema, false).expressions()); + assertIndexing(expected, new IndexingScript(schema).expressions()); } public static void assertIndexing(List expected, IndexingScript script) { diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/StatementExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/StatementExpression.java index 2db6c760380..66a45cb75e2 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/StatementExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/StatementExpression.java @@ -12,6 +12,7 @@ import com.yahoo.vespa.indexinglanguage.parser.IndexingInput; import com.yahoo.vespa.indexinglanguage.parser.ParseException; import java.util.Arrays; +import java.util.Iterator; import java.util.LinkedList; import java.util.List; import java.util.Map; @@ -48,7 +49,7 @@ public final class StatementExpression extends ExpressionList { @Override public StatementExpression convertChildren(ExpressionConverter converter) { return new StatementExpression(asList().stream() - .map(converter::convert) + .map(child -> converter.convert(child)) .filter(Objects::nonNull) .toList()); } -- cgit v1.2.3