From 0c855ec9883c8f49cca892bed80358647c7cd9c0 Mon Sep 17 00:00:00 2001 From: Jon Bratseth Date: Mon, 3 Oct 2022 06:27:03 +0200 Subject: Revert "Merge pull request #24279 from vespa-engine/jonmv/revert-GC-heaven-commits" This reverts commit 539f2871e4812a463aa108639aac66c4903f3c33, reversing changes made to aeaa3c2da60259a8ba80345591657922c90c1993. --- .../yahoo/schema/derived/DerivedConfiguration.java | 41 +++++++------ .../BooleanExpressionTransformer.java | 32 +++++++--- .../src/test/derived/neuralnet/neuralnet.sd | 4 ++ .../derived/neuralnet_noqueryprofile/neuralnet.sd | 4 ++ .../neuralnet_noqueryprofile/schema-info.cfg | 3 + .../derived/rankingexpression/rankexpression.sd | 8 +++ .../src/test/derived/rankingexpression/summary.cfg | 8 ++- .../BooleanExpressionTransformerTestCase.java | 68 +++++++++++++--------- .../RankingExpressionWithOnnxTestCase.java | 51 ++++++++++------ 9 files changed, 149 insertions(+), 70 deletions(-) diff --git a/config-model/src/main/java/com/yahoo/schema/derived/DerivedConfiguration.java b/config-model/src/main/java/com/yahoo/schema/derived/DerivedConfiguration.java index 11bd14cbe46..8b07aa48a24 100644 --- a/config-model/src/main/java/com/yahoo/schema/derived/DerivedConfiguration.java +++ b/config-model/src/main/java/com/yahoo/schema/derived/DerivedConfiguration.java @@ -66,26 +66,31 @@ public class DerivedConfiguration implements AttributesConfig.Producer { * schema is later modified. */ public DerivedConfiguration(Schema schema, DeployState deployState) { - Validator.ensureNotNull("Schema", schema); - this.schema = schema; - this.queryProfiles = deployState.getQueryProfiles().getRegistry(); - this.maxUncommittedMemory = deployState.getProperties().featureFlags().maxUnCommittedMemory(); - if ( ! schema.isDocumentsOnly()) { - streamingFields = new VsmFields(schema); - streamingSummary = new VsmSummary(schema); + try { + Validator.ensureNotNull("Schema", schema); + this.schema = schema; + this.queryProfiles = deployState.getQueryProfiles().getRegistry(); + this.maxUncommittedMemory = deployState.getProperties().featureFlags().maxUnCommittedMemory(); + if (!schema.isDocumentsOnly()) { + streamingFields = new VsmFields(schema); + streamingSummary = new VsmSummary(schema); + } + if (!schema.isDocumentsOnly()) { + attributeFields = new AttributeFields(schema); + summaries = new Summaries(schema, deployState.getDeployLogger(), deployState.getProperties().featureFlags()); + juniperrc = new Juniperrc(schema); + rankProfileList = new RankProfileList(schema, schema.rankExpressionFiles(), attributeFields, deployState); + indexingScript = new IndexingScript(schema); + indexInfo = new IndexInfo(schema); + schemaInfo = new SchemaInfo(schema, deployState.rankProfileRegistry(), summaries); + indexSchema = new IndexSchema(schema); + importedFields = new ImportedFields(schema); + } + Validation.validate(this, schema); } - if ( ! schema.isDocumentsOnly()) { - attributeFields = new AttributeFields(schema); - summaries = new Summaries(schema, deployState.getDeployLogger(), deployState.getProperties().featureFlags()); - juniperrc = new Juniperrc(schema); - rankProfileList = new RankProfileList(schema, schema.rankExpressionFiles(), attributeFields, deployState); - indexingScript = new IndexingScript(schema); - indexInfo = new IndexInfo(schema); - schemaInfo = new SchemaInfo(schema, deployState.rankProfileRegistry(), summaries); - indexSchema = new IndexSchema(schema); - importedFields = new ImportedFields(schema); + catch (IllegalArgumentException e) { + throw new IllegalArgumentException("Invalid " + schema, e); } - Validation.validate(this, schema); } /** diff --git a/config-model/src/main/java/com/yahoo/schema/expressiontransforms/BooleanExpressionTransformer.java b/config-model/src/main/java/com/yahoo/schema/expressiontransforms/BooleanExpressionTransformer.java index ad050d4ca63..49fb48225e7 100644 --- a/config-model/src/main/java/com/yahoo/schema/expressiontransforms/BooleanExpressionTransformer.java +++ b/config-model/src/main/java/com/yahoo/schema/expressiontransforms/BooleanExpressionTransformer.java @@ -36,12 +36,12 @@ public class BooleanExpressionTransformer extends ExpressionTransformer child = node.children().iterator(); // Transform in precedence order: @@ -51,24 +51,25 @@ public class BooleanExpressionTransformer extends ExpressionTransformer 1 && ! op.hasPrecedenceOver(stack.peek().op)) { - popStack(stack); + popStack(stack, context); } } stack.push(new ChildNode(op, child.next())); } while (stack.size() > 1) - popStack(stack); + popStack(stack, context); return stack.getFirst().child; } - private void popStack(Deque stack) { + private void popStack(Deque stack, TransformContext context) { ChildNode rhs = stack.pop(); ChildNode lhs = stack.peek(); + boolean primitives = isDefinitelyPrimitive(lhs.child, context) && isDefinitelyPrimitive(rhs.child, context); ExpressionNode combination; - if (rhs.op == Operator.and) + if (primitives && rhs.op == Operator.and) combination = andByIfNode(lhs.child, rhs.child); - else if (rhs.op == Operator.or) + else if (primitives && rhs.op == Operator.or) combination = orByIfNode(lhs.child, rhs.child); else { combination = resolve(lhs, rhs); @@ -77,6 +78,22 @@ public class BooleanExpressionTransformer extends ExpressionTransformer(d0[1],d1[784]) }\n"; @AfterEach public void removeGeneratedModelFiles() { @@ -41,7 +43,7 @@ public class RankingExpressionWithOnnxTestCase { void testOnnxReferenceWithConstantFeature() { RankProfileSearchFixture search = fixtureWith("constant(mytensor)", "onnx_vespa('mnist_softmax.onnx')", - "constant mytensor { file: ignored\ntype: tensor(d0[1],d1[784]) }", + vespaExpressionConstants + "constant mytensor { file: ignored\ntype: tensor(d0[1],d1[784]) }", null); search.assertFirstPhaseExpression(vespaExpression, "my_profile"); } @@ -58,7 +60,7 @@ public class RankingExpressionWithOnnxTestCase { queryProfileType); RankProfileSearchFixture search = fixtureWith("query(mytensor)", "onnx_vespa('mnist_softmax.onnx')", - null, + vespaExpressionConstants, null, "Placeholder", application); @@ -70,7 +72,7 @@ public class RankingExpressionWithOnnxTestCase { StoringApplicationPackage application = new StoringApplicationPackage(applicationDir); RankProfileSearchFixture search = fixtureWith("attribute(mytensor)", "onnx_vespa('mnist_softmax.onnx')", - null, + vespaExpressionConstants, "field mytensor type tensor(d0[1],d1[784]) { indexing: attribute }", "Placeholder", application); @@ -88,7 +90,7 @@ public class RankingExpressionWithOnnxTestCase { StoringApplicationPackage application = new StoringApplicationPackage(applicationDir, queryProfile, queryProfileType); RankProfileSearchFixture search = fixtureWith("sum(query(mytensor) * attribute(mytensor) * constant(mytensor),d2)", "onnx_vespa('mnist_softmax.onnx')", - "constant mytensor { file: ignored\ntype: tensor(d0[1],d1[784]) }", + vespaExpressionConstants + "constant mytensor { file: ignored\ntype: tensor(d0[1],d1[784]) }", "field mytensor type tensor(d0[1],d1[784]) { indexing: attribute }", "Placeholder", application); @@ -99,21 +101,24 @@ public class RankingExpressionWithOnnxTestCase { @Test void testNestedOnnxReference() { RankProfileSearchFixture search = fixtureWith("tensor(d0[1],d1[784])(0.0)", - "5 + sum(onnx_vespa('mnist_softmax.onnx'))"); + "5 + sum(onnx_vespa('mnist_softmax.onnx'))", + vespaExpressionConstants); search.assertFirstPhaseExpression("5 + reduce(" + vespaExpression + ", sum)", "my_profile"); } @Test void testOnnxReferenceWithSpecifiedOutput() { RankProfileSearchFixture search = fixtureWith("tensor(d0[1],d1[784])(0.0)", - "onnx_vespa('mnist_softmax.onnx', 'layer_add')"); + "onnx_vespa('mnist_softmax.onnx', 'layer_add')", + vespaExpressionConstants); search.assertFirstPhaseExpression(vespaExpression, "my_profile"); } @Test void testOnnxReferenceWithSpecifiedOutputAndSignature() { RankProfileSearchFixture search = fixtureWith("tensor(d0[1],d1[784])(0.0)", - "onnx_vespa('mnist_softmax.onnx', 'default.layer_add')"); + "onnx_vespa('mnist_softmax.onnx', 'default.layer_add')", + vespaExpressionConstants); search.assertFirstPhaseExpression(vespaExpression, "my_profile"); } @@ -177,7 +182,8 @@ public class RankingExpressionWithOnnxTestCase { @Test void testImportingFromStoredExpressions() throws IOException { RankProfileSearchFixture search = fixtureWith("tensor(d0[1],d1[784])(0.0)", - "onnx_vespa(\"mnist_softmax.onnx\")"); + "onnx_vespa(\"mnist_softmax.onnx\")", + vespaExpressionConstants); search.assertFirstPhaseExpression(vespaExpression, "my_profile"); // At this point the expression is stored - copy application to another location which do not have a models dir @@ -187,12 +193,14 @@ public class RankingExpressionWithOnnxTestCase { IOUtils.copyDirectory(applicationDir.append(ApplicationPackage.MODELS_GENERATED_DIR).toFile(), storedApplicationDirectory.append(ApplicationPackage.MODELS_GENERATED_DIR).toFile()); StoringApplicationPackage storedApplication = new StoringApplicationPackage(storedApplicationDirectory); + String constants = "constant mnist_softmax_layer_Variable { file: ignored\ntype: tensor(d0[2],d1[784]) }\n" + + "constant mnist_softmax_layer_Variable_1 { file: ignored\ntype: tensor(d0[2],d1[784]) }\n"; RankProfileSearchFixture searchFromStored = fixtureWith("tensor(d0[2],d1[784])(0.0)", - "onnx_vespa('mnist_softmax.onnx')", - null, - null, - "Placeholder", - storedApplication); + "onnx_vespa('mnist_softmax.onnx')", + constants, + null, + "Placeholder", + storedApplication); searchFromStored.assertFirstPhaseExpression(vespaExpression, "my_profile"); // Verify that the constants exists, but don't verify the content as we are not // simulating file distribution in this test @@ -221,7 +229,8 @@ public class RankingExpressionWithOnnxTestCase { String vespaExpressionWithoutConstant = "join(reduce(join(rename(Placeholder, (d0, d1), (d0, d2)), " + name + "_layer_Variable, f(a,b)(a * b)), sum, d2) * 1.0, constant(" + name + "_layer_Variable_1) * 1.0, f(a,b)(a + b))"; - RankProfileSearchFixture search = uncompiledFixtureWith(rankProfile, new StoringApplicationPackage(applicationDir)); + String constant = "constant mnist_softmax_layer_Variable_1 { file: ignored\ntype: tensor(d0[1],d1[10]) }\n"; + RankProfileSearchFixture search = uncompiledFixtureWith(rankProfile, new StoringApplicationPackage(applicationDir), constant); search.compileRankProfile("my_profile", applicationDir.append("models")); search.compileRankProfile("my_profile_child", applicationDir.append("models")); search.assertFirstPhaseExpression(vespaExpressionWithoutConstant, "my_profile"); @@ -237,7 +246,7 @@ public class RankingExpressionWithOnnxTestCase { IOUtils.copyDirectory(applicationDir.append(ApplicationPackage.MODELS_GENERATED_DIR).toFile(), storedApplicationDirectory.append(ApplicationPackage.MODELS_GENERATED_DIR).toFile()); StoringApplicationPackage storedApplication = new StoringApplicationPackage(storedApplicationDirectory); - RankProfileSearchFixture searchFromStored = uncompiledFixtureWith(rankProfile, storedApplication); + RankProfileSearchFixture searchFromStored = uncompiledFixtureWith(rankProfile, storedApplication, constant); searchFromStored.compileRankProfile("my_profile", applicationDir.append("models")); searchFromStored.compileRankProfile("my_profile_child", applicationDir.append("models")); searchFromStored.assertFirstPhaseExpression(vespaExpressionWithoutConstant, "my_profile"); @@ -326,7 +335,11 @@ public class RankingExpressionWithOnnxTestCase { } private RankProfileSearchFixture fixtureWith(String placeholderExpression, String firstPhaseExpression) { - return fixtureWith(placeholderExpression, firstPhaseExpression, null, null, "Placeholder", + return fixtureWith(placeholderExpression, firstPhaseExpression, null); + } + + private RankProfileSearchFixture fixtureWith(String placeholderExpression, String firstPhaseExpression, String constant) { + return fixtureWith(placeholderExpression, firstPhaseExpression, constant, null, "Placeholder", new StoringApplicationPackage(applicationDir)); } @@ -337,9 +350,13 @@ public class RankingExpressionWithOnnxTestCase { } private RankProfileSearchFixture uncompiledFixtureWith(String rankProfile, StoringApplicationPackage application) { + return uncompiledFixtureWith(rankProfile, application, null); + } + + private RankProfileSearchFixture uncompiledFixtureWith(String rankProfile, StoringApplicationPackage application, String constant) { try { return new RankProfileSearchFixture(application, application.getQueryProfiles(), - rankProfile, null, null); + rankProfile, constant, null); } catch (ParseException e) { throw new IllegalArgumentException(e); -- cgit v1.2.3