From 54c6c15bcc460db3066c78ccf2e4167c11442f38 Mon Sep 17 00:00:00 2001 From: Henning Baldersheim Date: Fri, 12 Jan 2024 20:38:47 +0100 Subject: Revert "Drop tokenize expressions from ilscript for streaming mode." --- .../src/main/java/com/yahoo/schema/Schema.java | 1 + .../yahoo/schema/derived/DerivedConfiguration.java | 2 +- .../com/yahoo/schema/derived/IndexingScript.java | 33 ++++------------------ 3 files changed, 8 insertions(+), 28 deletions(-) (limited to 'config-model/src/main/java/com') diff --git a/config-model/src/main/java/com/yahoo/schema/Schema.java b/config-model/src/main/java/com/yahoo/schema/Schema.java index 3402ba31be9..279b5729ea1 100644 --- a/config-model/src/main/java/com/yahoo/schema/Schema.java +++ b/config-model/src/main/java/com/yahoo/schema/Schema.java @@ -18,6 +18,7 @@ import com.yahoo.schema.document.SDDocumentType; import com.yahoo.schema.document.SDField; import com.yahoo.schema.document.Stemming; import com.yahoo.schema.document.TemporaryImportedFields; +import com.yahoo.schema.document.annotation.SDAnnotationType; import com.yahoo.searchlib.rankingexpression.Reference; import com.yahoo.vespa.documentmodel.DocumentSummary; import com.yahoo.vespa.documentmodel.SummaryField; diff --git a/config-model/src/main/java/com/yahoo/schema/derived/DerivedConfiguration.java b/config-model/src/main/java/com/yahoo/schema/derived/DerivedConfiguration.java index 575b7264628..7f874d04f13 100644 --- a/config-model/src/main/java/com/yahoo/schema/derived/DerivedConfiguration.java +++ b/config-model/src/main/java/com/yahoo/schema/derived/DerivedConfiguration.java @@ -80,7 +80,7 @@ public class DerivedConfiguration implements AttributesConfig.Producer { summaries = new Summaries(schema, deployState.getDeployLogger(), deployState.getProperties().featureFlags()); juniperrc = new Juniperrc(schema); rankProfileList = new RankProfileList(schema, schema.rankExpressionFiles(), attributeFields, deployState); - indexingScript = new IndexingScript(schema, isStreaming); + indexingScript = new IndexingScript(schema); indexInfo = new IndexInfo(schema, isStreaming); schemaInfo = new SchemaInfo(schema, deployState.rankProfileRegistry(), summaries); indexSchema = new IndexSchema(schema); diff --git a/config-model/src/main/java/com/yahoo/schema/derived/IndexingScript.java b/config-model/src/main/java/com/yahoo/schema/derived/IndexingScript.java index 6f0420f5203..73f3507ab00 100644 --- a/config-model/src/main/java/com/yahoo/schema/derived/IndexingScript.java +++ b/config-model/src/main/java/com/yahoo/schema/derived/IndexingScript.java @@ -6,7 +6,6 @@ import com.yahoo.schema.document.GeoPos; import com.yahoo.schema.document.ImmutableSDField; import com.yahoo.vespa.configdefinition.IlscriptsConfig; import com.yahoo.vespa.configdefinition.IlscriptsConfig.Ilscript.Builder; -import com.yahoo.vespa.indexinglanguage.ExpressionConverter; import com.yahoo.vespa.indexinglanguage.ExpressionVisitor; import com.yahoo.vespa.indexinglanguage.expressions.ClearStateExpression; import com.yahoo.vespa.indexinglanguage.expressions.Expression; @@ -17,7 +16,6 @@ import com.yahoo.vespa.indexinglanguage.expressions.PassthroughExpression; import com.yahoo.vespa.indexinglanguage.expressions.ScriptExpression; import com.yahoo.vespa.indexinglanguage.expressions.SetLanguageExpression; import com.yahoo.vespa.indexinglanguage.expressions.StatementExpression; -import com.yahoo.vespa.indexinglanguage.expressions.TokenizeExpression; import com.yahoo.vespa.indexinglanguage.expressions.ZCurveExpression; import java.util.ArrayList; @@ -38,10 +36,8 @@ public final class IndexingScript extends Derived implements IlscriptsConfig.Pro private final List docFields = new ArrayList<>(); private final List expressions = new ArrayList<>(); private List fieldsSettingLanguage; - private final boolean isStreaming; - public IndexingScript(Schema schema, boolean isStreaming) { - this.isStreaming = isStreaming; + public IndexingScript(Schema schema) { derive(schema); } @@ -100,38 +96,21 @@ public final class IndexingScript extends Derived implements IlscriptsConfig.Pro configBuilder.ilscript(ilscriptBuilder); } - private static class DropTokenize extends ExpressionConverter { - @Override - protected boolean shouldConvert(Expression exp) { - return exp instanceof TokenizeExpression; - } - - @Override - protected Expression doConvert(Expression exp) { - return null; - } - } - private void addContentInOrder(IlscriptsConfig.Ilscript.Builder ilscriptBuilder) { ArrayList later = new ArrayList<>(); Set touchedFields = new HashSet<>(); for (Expression expression : expressions) { - if (isStreaming) { - expression = expression.convertChildren(new DropTokenize()); - } - if (modifiesSelf(expression) && ! setsLanguage(expression)) { + if (modifiesSelf(expression) && ! setsLanguage(expression)) later.add(expression); - } else { + else ilscriptBuilder.content(expression.toString()); - } FieldScanVisitor fieldFetcher = new FieldScanVisitor(); fieldFetcher.visit(expression); touchedFields.addAll(fieldFetcher.touchedFields()); } - for (Expression exp : later) { + for (Expression exp : later) ilscriptBuilder.content(exp.toString()); - } generateSyntheticStatementsForUntouchedFields(ilscriptBuilder, touchedFields); } @@ -192,8 +171,8 @@ public final class IndexingScript extends Derived implements IlscriptsConfig.Pro } private static class FieldScanVisitor extends ExpressionVisitor { - List touchedFields = new ArrayList<>(); - List candidates = new ArrayList<>(); + List touchedFields = new ArrayList(); + List candidates = new ArrayList(); @Override protected void doVisit(Expression exp) { -- cgit v1.2.3