diff options
Diffstat (limited to 'config-model/src/test')
39 files changed, 822 insertions, 338 deletions
diff --git a/config-model/src/test/derived/ngram/chunk.sd b/config-model/src/test/derived/ngram/chunk.sd new file mode 100644 index 00000000000..7c2a7465327 --- /dev/null +++ b/config-model/src/test/derived/ngram/chunk.sd @@ -0,0 +1,20 @@ +schema chunk { + + document chunk { + field content type string { + indexing: summary | index + match { + gram + gram-size: 3 + } + } + } + + document-summary content-summary inherits default { + summary content_dynamic type string { + source: content + dynamic + } + } + +} diff --git a/config-model/src/test/derived/ngram/index-info.cfg b/config-model/src/test/derived/ngram/index-info.cfg new file mode 100644 index 00000000000..72b6760ceb5 --- /dev/null +++ b/config-model/src/test/derived/ngram/index-info.cfg @@ -0,0 +1,21 @@ +indexinfo[].name "chunk" +indexinfo[].command[].indexname "sddocname" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "sddocname" +indexinfo[].command[].command "word" +indexinfo[].command[].indexname "content" +indexinfo[].command[].command "lowercase" +indexinfo[].command[].indexname "content" +indexinfo[].command[].command "string" +indexinfo[].command[].indexname "content" +indexinfo[].command[].command "type string" +indexinfo[].command[].indexname "content" +indexinfo[].command[].command "ngram 3" +indexinfo[].command[].indexname "content_dynamic" +indexinfo[].command[].command "string" +indexinfo[].command[].indexname "content_dynamic" +indexinfo[].command[].command "type string" +indexinfo[].command[].indexname "content_dynamic" +indexinfo[].command[].command "ngram 3" +indexinfo[].command[].indexname "content_dynamic" +indexinfo[].command[].command "dynteaser" diff --git a/config-model/src/test/examples/indexing_attribute_changed.sd b/config-model/src/test/examples/indexing_attribute_changed.sd deleted file mode 100644 index bab878d09ab..00000000000 --- a/config-model/src/test/examples/indexing_attribute_changed.sd +++ /dev/null @@ -1,8 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -search indexing_attribute_changed { - document indexing_attribute_changed { - field foo type string { - indexing: summary | lowercase | attribute - } - } -} diff --git a/config-model/src/test/examples/indexing_attribute_other.sd b/config-model/src/test/examples/indexing_attribute_other.sd deleted file mode 100644 index e3f58f20910..00000000000 --- a/config-model/src/test/examples/indexing_attribute_other.sd +++ /dev/null @@ -1,8 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -search indexing_attribute_other { - document indexing_attribute_other { - field foo type string { - indexing: attribute bar - } - } -} diff --git a/config-model/src/test/examples/indexing_extra_field_input_extra_field.sd b/config-model/src/test/examples/indexing_extra_field_input_extra_field.sd deleted file mode 100644 index 315d6c2a677..00000000000 --- a/config-model/src/test/examples/indexing_extra_field_input_extra_field.sd +++ /dev/null @@ -1,12 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -search indexing_extra_field_input_extra_field { - document indexing_extra_field_input_extra_field { - - } - field foo type string { - - } - field bar type string { - indexing: input bar | index - } -} diff --git a/config-model/src/test/examples/indexing_extra_field_input_implicit.sd b/config-model/src/test/examples/indexing_extra_field_input_implicit.sd deleted file mode 100644 index 8aff3284ce3..00000000000 --- a/config-model/src/test/examples/indexing_extra_field_input_implicit.sd +++ /dev/null @@ -1,9 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -search indexing_extra_field_input_implicit { - document indexing_extra_field_input_implicit { - - } - field foo type string { - indexing: index - } -} diff --git a/config-model/src/test/examples/indexing_extra_field_input_null.sd b/config-model/src/test/examples/indexing_extra_field_input_null.sd deleted file mode 100644 index c4600fa680a..00000000000 --- a/config-model/src/test/examples/indexing_extra_field_input_null.sd +++ /dev/null @@ -1,9 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -search indexing_extra_field_input_null { - document indexing_extra_field_input_null { - - } - field foo type string { - indexing: input foo | index - } -} diff --git a/config-model/src/test/examples/indexing_extra_field_input_self.sd b/config-model/src/test/examples/indexing_extra_field_input_self.sd deleted file mode 100644 index 36dbae21449..00000000000 --- a/config-model/src/test/examples/indexing_extra_field_input_self.sd +++ /dev/null @@ -1,9 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -search indexing_extra_field_input_self { - document indexing_extra_field_input_self { - - } - field foo type string { - indexing: input foo | index - } -} diff --git a/config-model/src/test/examples/indexing_index_changed.sd b/config-model/src/test/examples/indexing_index_changed.sd deleted file mode 100644 index 194a9bd3177..00000000000 --- a/config-model/src/test/examples/indexing_index_changed.sd +++ /dev/null @@ -1,8 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -search indexing_index_changed { - document indexing_index_changed { - field foo type string { - indexing: attribute | lowercase | index - } - } -} diff --git a/config-model/src/test/examples/indexing_index_other.sd b/config-model/src/test/examples/indexing_index_other.sd deleted file mode 100644 index 40b8b150a5b..00000000000 --- a/config-model/src/test/examples/indexing_index_other.sd +++ /dev/null @@ -1,8 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -search indexing_index_other { - document indexing_index_other { - field foo type string { - indexing: index bar - } - } -} diff --git a/config-model/src/test/examples/indexing_modify_field_no_output.sd b/config-model/src/test/examples/indexing_modify_field_no_output.sd deleted file mode 100644 index ac2bed520e8..00000000000 --- a/config-model/src/test/examples/indexing_modify_field_no_output.sd +++ /dev/null @@ -1,8 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -search indexing_modify_field_no_output { - document indexing_modify_field_no_output { - field foo type string { - indexing: lowercase | echo - } - } -} diff --git a/config-model/src/test/examples/indexing_output_conflict.sd b/config-model/src/test/examples/indexing_output_conflict.sd deleted file mode 100644 index 9cf1cbc0823..00000000000 --- a/config-model/src/test/examples/indexing_output_conflict.sd +++ /dev/null @@ -1,11 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -search indexing_output_confict { - document indexing_output_confict { - field foo type string { - - } - } - field bar type string { - indexing: input foo | attribute | lowercase | index - } -} diff --git a/config-model/src/test/examples/indexing_output_other_field.sd b/config-model/src/test/examples/indexing_output_other_field.sd deleted file mode 100644 index 40b08bb15b2..00000000000 --- a/config-model/src/test/examples/indexing_output_other_field.sd +++ /dev/null @@ -1,11 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -search indexing_output_other_field { - document indexing_output_other_field { - field foo type string { - indexing: index bar - } - } - field bar type string { - - } -} diff --git a/config-model/src/test/examples/indexing_summary_other.sd b/config-model/src/test/examples/indexing_summary_other.sd deleted file mode 100644 index 871ab854c51..00000000000 --- a/config-model/src/test/examples/indexing_summary_other.sd +++ /dev/null @@ -1,8 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -search indexing_summary_other { - document indexing_summary_other { - field foo type string { - indexing: summary bar - } - } -} diff --git a/config-model/src/test/examples/matchphase/non_existing_attribute.sd b/config-model/src/test/examples/matchphase/non_existing_attribute.sd deleted file mode 100644 index cd3842fde8a..00000000000 --- a/config-model/src/test/examples/matchphase/non_existing_attribute.sd +++ /dev/null @@ -1,14 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -search test { - document test { - field foo type int { - indexing: summary - } - } - rank-profile default { - match-phase { - attribute: foo - max-hits: 100 - } - } -} diff --git a/config-model/src/test/examples/matchphase/non_fast_search_attribute.sd b/config-model/src/test/examples/matchphase/non_fast_search_attribute.sd deleted file mode 100644 index 5fde096cf61..00000000000 --- a/config-model/src/test/examples/matchphase/non_fast_search_attribute.sd +++ /dev/null @@ -1,14 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -search test { - document test { - field foo type int { - indexing: attribute - } - } - rank-profile default { - match-phase { - attribute: foo - max-hits: 100 - } - } -} diff --git a/config-model/src/test/examples/matchphase/wrong_collection_type_attribute.sd b/config-model/src/test/examples/matchphase/wrong_collection_type_attribute.sd deleted file mode 100644 index 8a9166c94f7..00000000000 --- a/config-model/src/test/examples/matchphase/wrong_collection_type_attribute.sd +++ /dev/null @@ -1,14 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -search test { - document test { - field foo type array<int> { - indexing: attribute - } - } - rank-profile default { - match-phase { - attribute: foo - max-hits: 100 - } - } -} diff --git a/config-model/src/test/examples/matchphase/wrong_data_type_attribute.sd b/config-model/src/test/examples/matchphase/wrong_data_type_attribute.sd deleted file mode 100644 index d4f526569ea..00000000000 --- a/config-model/src/test/examples/matchphase/wrong_data_type_attribute.sd +++ /dev/null @@ -1,14 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -search test { - document test { - field foo type string { - indexing: attribute - } - } - rank-profile default { - match-phase { - attribute: foo - max-hits: 100 - } - } -} diff --git a/config-model/src/test/java/com/yahoo/config/model/MockModelContext.java b/config-model/src/test/java/com/yahoo/config/model/MockModelContext.java index af05a144b79..3f5173a3ae9 100644 --- a/config-model/src/test/java/com/yahoo/config/model/MockModelContext.java +++ b/config-model/src/test/java/com/yahoo/config/model/MockModelContext.java @@ -10,6 +10,7 @@ import com.yahoo.config.model.api.ConfigDefinitionRepo; import com.yahoo.config.model.api.HostProvisioner; import com.yahoo.config.model.api.Model; import com.yahoo.config.model.api.ModelContext; +import com.yahoo.config.model.api.OnnxModelCost; import com.yahoo.config.model.api.Provisioned; import com.yahoo.config.model.application.provider.BaseDeployLogger; import com.yahoo.config.model.application.provider.MockFileRegistry; @@ -84,4 +85,6 @@ public class MockModelContext implements ModelContext { public ExecutorService getExecutor() { return new InThreadExecutorService(); } + + @Override public OnnxModelCost onnxModelCost() { return OnnxModelCost.disabled(); } } diff --git a/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java b/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java index 2f8a8bddf20..38f51323ee2 100644 --- a/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java +++ b/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java @@ -148,7 +148,7 @@ public class ModelProvisioningTest { assertEquals("-Xlog:gc", mydisc2.getContainers().get(1).getJvmOptions()); assertEquals("lib/blablamalloc.so", mydisc2.getContainers().get(0).getPreLoad()); assertEquals("lib/blablamalloc.so", mydisc2.getContainers().get(1).getPreLoad()); - assertEquals(Optional.of(45), mydisc2.getMemoryPercentage()); + assertEquals(45, mydisc2.getMemoryPercentage().get().percentage()); assertEquals(Optional.of("-XX:+UseParNewGC"), mydisc2.getJvmGCOptions()); QrStartConfig.Builder qrStartBuilder = new QrStartConfig.Builder(); mydisc2.getConfig(qrStartBuilder); @@ -288,10 +288,11 @@ public class ModelProvisioningTest { assertEquals(2025077080L, protonMemorySize(model.getContentClusters().get("content1")), "Memory for proton is lowered to account for the jvm heap"); assertProvisioned(0, ClusterSpec.Id.from("container1"), ClusterSpec.Type.container, model); assertProvisioned(2, ClusterSpec.Id.from("content1"), ClusterSpec.Id.from("container1"), ClusterSpec.Type.combined, model); - assertEquals(1, logger.msgs().size()); + var msgs = logger.msgs().stream().filter(m -> m.level().equals(Level.WARNING)).toList(); + assertEquals(1, msgs.size()); assertEquals("Declaring combined cluster with <nodes of=\"...\"> is deprecated without replacement, " + "and the feature will be removed in Vespa 9. Use separate container and content clusters instead", - logger.msgs().get(0).message); + msgs.get(0).message); } @Test diff --git a/config-model/src/test/java/com/yahoo/schema/AttributeSettingsTestCase.java b/config-model/src/test/java/com/yahoo/schema/AttributeSettingsTestCase.java index 3ca182e18c2..db862dd388e 100644 --- a/config-model/src/test/java/com/yahoo/schema/AttributeSettingsTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/AttributeSettingsTestCase.java @@ -21,7 +21,7 @@ import static org.junit.jupiter.api.Assertions.*; /** * Attribute settings * - * @author bratseth + * @author bratseth */ public class AttributeSettingsTestCase extends AbstractSchemaTestCase { diff --git a/config-model/src/test/java/com/yahoo/schema/derived/NGramTestCase.java b/config-model/src/test/java/com/yahoo/schema/derived/NGramTestCase.java new file mode 100644 index 00000000000..4481445858a --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/derived/NGramTestCase.java @@ -0,0 +1,15 @@ +package com.yahoo.schema.derived; + +import com.yahoo.schema.parser.ParseException; +import org.junit.jupiter.api.Test; + +import java.io.IOException; + +public class NGramTestCase extends AbstractExportingTestCase { + + @Test + void testNGram() throws IOException, ParseException { + assertCorrectDeriving("ngram"); + } + +}
\ No newline at end of file diff --git a/config-model/src/test/java/com/yahoo/schema/parser/SchemaParserTestCase.java b/config-model/src/test/java/com/yahoo/schema/parser/SchemaParserTestCase.java index e69f26a31c9..89eff4ec464 100644 --- a/config-model/src/test/java/com/yahoo/schema/parser/SchemaParserTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/parser/SchemaParserTestCase.java @@ -269,7 +269,6 @@ public class SchemaParserTestCase { checkFileParses("src/test/examples/implicitsummaryfields.sd"); checkFileParses("src/test/examples/incorrectrankingexpressionfileref.sd"); checkFileParses("src/test/examples/indexing_extra.sd"); - checkFileParses("src/test/examples/indexing_modify_field_no_output.sd"); checkFileParses("src/test/examples/indexing.sd"); checkFileParses("src/test/examples/indexrewrite.sd"); checkFileParses("src/test/examples/indexsettings.sd"); diff --git a/config-model/src/test/java/com/yahoo/schema/processing/AssertIndexingScript.java b/config-model/src/test/java/com/yahoo/schema/processing/AssertIndexingScript.java index f9c1e992347..36f20c18588 100644 --- a/config-model/src/test/java/com/yahoo/schema/processing/AssertIndexingScript.java +++ b/config-model/src/test/java/com/yahoo/schema/processing/AssertIndexingScript.java @@ -38,6 +38,6 @@ public abstract class AssertIndexingScript { String str = actualExp.toString(); assertTrue(parsedExpected.remove(str), "Unexpected: " + str); } - assertTrue(parsedExpected.isEmpty(), "Missing: " + parsedExpected.toString()); + assertTrue(parsedExpected.isEmpty(), "Missing: " + parsedExpected); } } diff --git a/config-model/src/test/java/com/yahoo/schema/processing/AssertSearchBuilder.java b/config-model/src/test/java/com/yahoo/schema/processing/AssertSearchBuilder.java deleted file mode 100644 index 12da3f0797b..00000000000 --- a/config-model/src/test/java/com/yahoo/schema/processing/AssertSearchBuilder.java +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.schema.processing; - -import com.yahoo.schema.ApplicationBuilder; -import com.yahoo.schema.parser.ParseException; - -import java.io.IOException; - -import static org.junit.jupiter.api.Assertions.*; - -/** - * @author Simon Thoresen Hult - */ -public abstract class AssertSearchBuilder { - - public static void assertBuilds(String searchDefinitionFileName) throws IOException, ParseException { - assertNotNull(ApplicationBuilder.buildFromFile(searchDefinitionFileName)); - } - - public static void assertBuildFails(String searchDefinitionFileName, String expectedException) - throws IOException, ParseException { - try { - ApplicationBuilder.buildFromFile(searchDefinitionFileName); - fail(searchDefinitionFileName); - } catch (IllegalArgumentException e) { - assertEquals(expectedException, e.getMessage()); - } - } -} diff --git a/config-model/src/test/java/com/yahoo/schema/processing/IndexingInputsTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/IndexingInputsTestCase.java index d420623f233..675168ca6c2 100644 --- a/config-model/src/test/java/com/yahoo/schema/processing/IndexingInputsTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/processing/IndexingInputsTestCase.java @@ -1,45 +1,165 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.schema.processing; +import com.yahoo.schema.ApplicationBuilder; import com.yahoo.schema.parser.ParseException; +import com.yahoo.yolean.Exceptions; import org.junit.jupiter.api.Test; -import java.io.IOException; - -import static com.yahoo.schema.processing.AssertSearchBuilder.assertBuildFails; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.fail; /** * @author Simon Thoresen Hult + * @author bratseth */ public class IndexingInputsTestCase { @Test - void requireThatExtraFieldInputExtraFieldThrows() throws IOException, ParseException { - assertBuildFails("src/test/examples/indexing_extra_field_input_extra_field.sd", - "For schema 'indexing_extra_field_input_extra_field', field 'bar': Indexing script refers " + - "to field 'bar' which is neither a field in document type " + - "'indexing_extra_field_input_extra_field' nor a mutable attribute"); + void requireThatExtraFieldInputExtraFieldThrows() throws ParseException { + try { + var schema = """ + search indexing_extra_field_input_extra_field { + document indexing_extra_field_input_extra_field { + } + field foo type string { + } + field bar type string { + indexing: input bar | index + } + } + """; + ApplicationBuilder.createFromString(schema); + fail("Expected exception"); + } + catch (IllegalArgumentException e) { + assertEquals("For schema 'indexing_extra_field_input_extra_field', field 'bar': Indexing script refers " + + "to field 'bar' which is neither a field in document type " + + "'indexing_extra_field_input_extra_field' nor a mutable attribute", + Exceptions.toMessageString(e)); + } + } + + @Test + void requireThatExtraFieldInputImplicitThrows() throws ParseException { + try { + var schema = """ + search indexing_extra_field_input_implicit { + document indexing_extra_field_input_implicit { + } + field foo type string { + indexing: index + } + } + """; + ApplicationBuilder.createFromString(schema); + fail("Expected exception"); + } + catch (IllegalArgumentException e) { + assertEquals("For schema 'indexing_extra_field_input_implicit', field 'foo': " + + "For expression '{ tokenize normalize stem:\"BEST\" | index foo; }': Expected string input, but no input is specified", + Exceptions.toMessageString(e)); + } + } + + @Test + void requireThatExtraFieldInputNullThrows() throws ParseException { + try { + var schema = """ + search indexing_extra_field_input_null { + document indexing_extra_field_input_null { + } + field foo type string { + indexing: input foo | index + } + } + """; + ApplicationBuilder.createFromString(schema); + fail("Expected exception"); + } + catch (IllegalArgumentException e) { + assertEquals("For schema 'indexing_extra_field_input_null', field 'foo': Indexing script refers to field " + + "'foo' which is neither a field in document type 'indexing_extra_field_input_null' nor a mutable attribute", + Exceptions.toMessageString(e)); + } + } + + @Test + void requireThatExtraFieldInputSelfThrows() throws ParseException { + try { + var schema = """ + search indexing_extra_field_input_self { + document indexing_extra_field_input_self { + } + field foo type string { + indexing: input foo | index + } + } + """; + ApplicationBuilder.createFromString(schema); + fail("Expected exception"); + } + catch (IllegalArgumentException e) { + assertEquals("For schema 'indexing_extra_field_input_self', field 'foo': Indexing script refers to field " + + "'foo' which is neither a field in document type 'indexing_extra_field_input_self' nor a mutable attribute", + Exceptions.toMessageString(e)); + } } @Test - void requireThatExtraFieldInputImplicitThrows() throws IOException, ParseException { - assertBuildFails("src/test/examples/indexing_extra_field_input_implicit.sd", - "For schema 'indexing_extra_field_input_implicit', field 'foo': Indexing script refers to " + - "field 'foo' which is neither a field in document type 'indexing_extra_field_input_implicit' nor a mutable attribute"); + void testPlainInputInDerivedField() throws ParseException { + var schema = """ + schema test { + document test { + field field1 type int { + } + } + field derived1 type int { + indexing: input field1 | attribute + } + } + """; + ApplicationBuilder.createFromString(schema); } @Test - void requireThatExtraFieldInputNullThrows() throws IOException, ParseException { - assertBuildFails("src/test/examples/indexing_extra_field_input_null.sd", - "For schema 'indexing_extra_field_input_null', field 'foo': Indexing script refers to field " + - "'foo' which is neither a field in document type 'indexing_extra_field_input_null' nor a mutable attribute"); + void testWrappedInputInDerivedField() throws ParseException { + var schema = """ + schema test { + document test { + field field1 type int { + } + } + field derived1 type int { + indexing: if (input field1 == 0) { 0 } else { 1 } | attribute + } + } + """; + ApplicationBuilder.createFromString(schema); } @Test - void requireThatExtraFieldInputSelfThrows() throws IOException, ParseException { - assertBuildFails("src/test/examples/indexing_extra_field_input_self.sd", - "For schema 'indexing_extra_field_input_self', field 'foo': Indexing script refers to field " + - "'foo' which is neither a field in document type 'indexing_extra_field_input_self' nor a mutable attribute"); + void testNoInputInDerivedField() throws ParseException { + try { + var schema = """ + schema test { + document test { + field field1 type int { + } + } + field derived1 type int { + indexing: attribute + } + } + """; + ApplicationBuilder.createFromString(schema); + fail("Expected exception"); + } + catch (IllegalArgumentException e) { + assertEquals("For schema 'test', field 'derived1': For expression '{ attribute derived1; }': " + + "Expected any input, but no input is specified", + Exceptions.toMessageString(e)); + } } } diff --git a/config-model/src/test/java/com/yahoo/schema/processing/IndexingOutputsTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/IndexingOutputsTestCase.java index e707d203381..7557ca5b725 100644 --- a/config-model/src/test/java/com/yahoo/schema/processing/IndexingOutputsTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/processing/IndexingOutputsTestCase.java @@ -1,13 +1,13 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.schema.processing; +import com.yahoo.schema.ApplicationBuilder; import com.yahoo.schema.parser.ParseException; +import com.yahoo.yolean.Exceptions; import org.junit.jupiter.api.Test; -import java.io.IOException; - -import static com.yahoo.schema.processing.AssertSearchBuilder.assertBuildFails; - +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.fail; /** * @author Simon Thoresen Hult @@ -15,16 +15,51 @@ import static com.yahoo.schema.processing.AssertSearchBuilder.assertBuildFails; public class IndexingOutputsTestCase { @Test - void requireThatOutputOtherFieldThrows() throws IOException, ParseException { - assertBuildFails("src/test/examples/indexing_output_other_field.sd", - "For schema 'indexing_output_other_field', field 'foo': Indexing expression 'index bar' " + - "attempts to write to a field other than 'foo'."); + void requireThatOutputOtherFieldThrows() throws ParseException { + try { + var schema = """ + search indexing_output_other_field { + document indexing_output_other_field { + field foo type string { + indexing: index bar + } + } + field bar type string { + } + } + """; + ApplicationBuilder.createFromString(schema); + fail("Expected exception"); + } + catch (IllegalArgumentException e) { + assertEquals("For schema 'indexing_output_other_field', field 'foo': Indexing expression 'index bar' " + + "attempts to write to a field other than 'foo'.", + Exceptions.toMessageString(e)); + } } @Test - void requireThatOutputConflictThrows() throws IOException, ParseException { - assertBuildFails("src/test/examples/indexing_output_conflict.sd", - "For schema 'indexing_output_confict', field 'bar': For expression 'index bar': Attempting " + - "to assign conflicting values to field 'bar'."); + void requireThatOutputConflictThrows() throws ParseException { + try { + var schema = """ + search indexing_output_confict { + document indexing_output_confict { + field foo type string { + } + } + field bar type string { + indexing: input foo | attribute | lowercase | index + } + } + """; + ApplicationBuilder.createFromString(schema); + fail("Expected exception"); + } + catch (IllegalArgumentException e) { + assertEquals("For schema 'indexing_output_confict', field 'bar': For expression 'index bar': Attempting " + + "to assign conflicting values to field 'bar'.", + Exceptions.toMessageString(e)); + } } + } diff --git a/config-model/src/test/java/com/yahoo/schema/processing/IndexingValidationTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/IndexingValidationTestCase.java index aa8a2922e8f..4343abbf548 100644 --- a/config-model/src/test/java/com/yahoo/schema/processing/IndexingValidationTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/processing/IndexingValidationTestCase.java @@ -4,13 +4,15 @@ package com.yahoo.schema.processing; import com.yahoo.schema.ApplicationBuilder; import com.yahoo.schema.derived.AbstractExportingTestCase; import com.yahoo.schema.parser.ParseException; +import com.yahoo.yolean.Exceptions; import org.junit.jupiter.api.Test; import java.io.IOException; import java.util.Arrays; import static com.yahoo.schema.processing.AssertIndexingScript.assertIndexing; -import static com.yahoo.schema.processing.AssertSearchBuilder.assertBuildFails; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.fail; /** * @author Simon Thoresen Hult @@ -18,45 +20,135 @@ import static com.yahoo.schema.processing.AssertSearchBuilder.assertBuildFails; public class IndexingValidationTestCase extends AbstractExportingTestCase { @Test - void testAttributeChanged() throws IOException, ParseException { - assertBuildFails("src/test/examples/indexing_attribute_changed.sd", - "For schema 'indexing_attribute_changed', field 'foo': For expression 'attribute foo': " + - "Attempting to assign conflicting values to field 'foo'."); + void testAttributeChanged() throws ParseException { + try { + var schema = """ + search indexing_attribute_changed { + document indexing_attribute_changed { + field foo type string { + indexing: summary | lowercase | attribute + } + } + } + """; + ApplicationBuilder.createFromString(schema); + fail("Expected exception"); + } + catch (IllegalArgumentException e) { + assertEquals("For schema 'indexing_attribute_changed', field 'foo': For expression 'attribute foo': " + + "Attempting to assign conflicting values to field 'foo'.", + Exceptions.toMessageString(e)); + } } @Test - void testAttributeOther() throws IOException, ParseException { - assertBuildFails("src/test/examples/indexing_attribute_other.sd", - "For schema 'indexing_attribute_other', field 'foo': Indexing expression 'attribute bar' " + - "attempts to write to a field other than 'foo'."); + void testAttributeOther() throws ParseException { + try { + var schema = """ + search indexing_attribute_other { + document indexing_attribute_other { + field foo type string { + indexing: attribute bar + } + } + } + """; + ApplicationBuilder.createFromString(schema); + fail("Expected exception"); + } + catch (IllegalArgumentException e) { + assertEquals("For schema 'indexing_attribute_other', field 'foo': Indexing expression 'attribute bar' " + + "attempts to write to a field other than 'foo'.", + Exceptions.toMessageString(e)); + } } @Test - void testIndexChanged() throws IOException, ParseException { - assertBuildFails("src/test/examples/indexing_index_changed.sd", - "For schema 'indexing_index_changed', field 'foo': For expression 'index foo': " + - "Attempting to assign conflicting values to field 'foo'."); + void testIndexChanged() throws ParseException { + try { + var schema = """ + search indexing_index_changed { + document indexing_index_changed { + field foo type string { + indexing: attribute | lowercase | index + } + } + } + """; + ApplicationBuilder.createFromString(schema); + fail("Expected exception"); + } + catch (IllegalArgumentException e) { + assertEquals("For schema 'indexing_index_changed', field 'foo': For expression 'index foo': " + + "Attempting to assign conflicting values to field 'foo'.", + Exceptions.toMessageString(e)); + } } @Test - void testIndexOther() throws IOException, ParseException { - assertBuildFails("src/test/examples/indexing_index_other.sd", - "For schema 'indexing_index_other', field 'foo': Indexing expression 'index bar' " + - "attempts to write to a field other than 'foo'."); + void testIndexOther() throws ParseException { + try { + var schema = """ + search indexing_index_other { + document indexing_index_other { + field foo type string { + indexing: index bar\s + } + } + } + """; + ApplicationBuilder.createFromString(schema); + fail("Expected exception"); + } + catch (IllegalArgumentException e) { + assertEquals("For schema 'indexing_index_other', field 'foo': Indexing expression 'index bar' " + + "attempts to write to a field other than 'foo'.", + Exceptions.toMessageString(e)); + } } @Test - void testSummaryChanged() throws IOException, ParseException { - assertBuildFails("src/test/examples/indexing_summary_changed.sd", - "For schema 'indexing_summary_fail', field 'foo': For expression 'summary foo': Attempting " + - "to assign conflicting values to field 'foo'."); + void testSummaryChanged() throws ParseException { + try { + var schema = """ + search indexing_summary_fail { + document indexing_summary_fail { + field foo type string { + indexing: index | lowercase | summary\s + } + } + } + """; + ApplicationBuilder.createFromString(schema); + fail("Expected exception"); + } + catch (IllegalArgumentException e) { + assertEquals("For schema 'indexing_summary_fail', field 'foo': For expression 'summary foo': Attempting " + + "to assign conflicting values to field 'foo'.", + Exceptions.toMessageString(e)); + } } @Test - void testSummaryOther() throws IOException, ParseException { - assertBuildFails("src/test/examples/indexing_summary_other.sd", - "For schema 'indexing_summary_other', field 'foo': Indexing expression 'summary bar' " + - "attempts to write to a field other than 'foo'."); + void testSummaryOther() throws ParseException { + try { + var schema = """ + search indexing_summary_other { + document indexing_summary_other { + field foo type string { + indexing: summary bar + } + } + } + """; + ApplicationBuilder.createFromString(schema); + fail("Expected exception"); + } + catch (IllegalArgumentException e) { + assertEquals("For schema 'indexing_summary_other', field 'foo': Indexing expression 'summary bar' " + + "attempts to write to a field other than 'foo'.", + Exceptions.toMessageString(e)); + } } @Test @@ -68,9 +160,35 @@ public class IndexingValidationTestCase extends AbstractExportingTestCase { } @Test - void requireThatMultilineOutputConflictThrows() throws IOException, ParseException { - assertBuildFails("src/test/examples/indexing_multiline_output_conflict.sd", - "For schema 'indexing_multiline_output_confict', field 'cox': For expression 'index cox': " + - "Attempting to assign conflicting values to field 'cox'."); + void requireThatMultilineOutputConflictThrows() throws ParseException { + try { + var schema = """ + search indexing_multiline_output_confict { + document indexing_multiline_output_confict { + field foo type string { + } + field bar type string { + } + field baz type string { + } + } + field cox type string { + indexing { + input foo | attribute; + input bar | index; + input baz | summary; + } + } + } + """; + ApplicationBuilder.createFromString(schema); + fail("Expected exception"); + } + catch (IllegalArgumentException e) { + assertEquals("For schema 'indexing_multiline_output_confict', field 'cox': For expression 'index cox': " + + "Attempting to assign conflicting values to field 'cox'.", + Exceptions.toMessageString(e)); + } } + } diff --git a/config-model/src/test/java/com/yahoo/schema/processing/IndexingValuesTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/IndexingValuesTestCase.java index 1f723924db6..c46b4fc2c7d 100644 --- a/config-model/src/test/java/com/yahoo/schema/processing/IndexingValuesTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/processing/IndexingValuesTestCase.java @@ -1,13 +1,15 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.schema.processing; +import com.yahoo.schema.ApplicationBuilder; import com.yahoo.schema.parser.ParseException; +import com.yahoo.yolean.Exceptions; import org.junit.jupiter.api.Test; import java.io.IOException; -import static com.yahoo.schema.processing.AssertSearchBuilder.assertBuildFails; -import static com.yahoo.schema.processing.AssertSearchBuilder.assertBuilds; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.fail; /** * @author Simon Thoresen Hult @@ -15,16 +17,43 @@ import static com.yahoo.schema.processing.AssertSearchBuilder.assertBuilds; public class IndexingValuesTestCase { @Test - void requireThatModifyFieldNoOutputDoesNotThrow() throws IOException, ParseException { - assertBuilds("src/test/examples/indexing_modify_field_no_output.sd"); + void requireThatModifyFieldNoOutputDoesNotThrow() throws ParseException { + var schema = """ + search indexing_modify_field_no_output { + document indexing_modify_field_no_output { + field foo type string { + indexing: lowercase | echo + } + } + } + """; + ApplicationBuilder.createFromString(schema); } @Test void requireThatInputOtherFieldThrows() throws IOException, ParseException { - assertBuildFails("src/test/examples/indexing_input_other_field.sd", - "For schema 'indexing_input_other_field', field 'bar': Indexing expression 'input foo' " + - "attempts to modify the value of the document field 'bar'. " + - "Use a field outside the document block instead."); + try { + var schema = """ + search indexing_input_other_field { + document indexing_input_other_field { + field foo type string { + + } + field bar type string { + indexing: input foo | attribute | index | summary + } + } + } + """; + ApplicationBuilder.createFromString(schema); + fail("Expected exception"); + } + catch (IllegalArgumentException e) { + assertEquals("For schema 'indexing_input_other_field', field 'bar': Indexing expression 'input foo' " + + "attempts to modify the value of the document field 'bar'. " + + "Use a field outside the document block instead.", + Exceptions.toMessageString(e)); + } } } diff --git a/config-model/src/test/java/com/yahoo/schema/processing/MatchPhaseSettingsValidatorTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/MatchPhaseSettingsValidatorTestCase.java index cbddea8ea6a..85ec80d2610 100644 --- a/config-model/src/test/java/com/yahoo/schema/processing/MatchPhaseSettingsValidatorTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/processing/MatchPhaseSettingsValidatorTestCase.java @@ -1,9 +1,12 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.schema.processing; +import com.yahoo.schema.ApplicationBuilder; +import com.yahoo.yolean.Exceptions; import org.junit.jupiter.api.Test; -import static com.yahoo.schema.processing.AssertSearchBuilder.assertBuildFails; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.fail; public class MatchPhaseSettingsValidatorTestCase { @@ -13,25 +16,109 @@ public class MatchPhaseSettingsValidatorTestCase { @Test void requireThatAttributeMustExists() throws Exception { - assertBuildFails("src/test/examples/matchphase/non_existing_attribute.sd", - getMessagePrefix() + "does not exists"); + try { + var schema = """ + search test { + document test { + field foo type int { + indexing: summary + } + } + rank-profile default { + match-phase { + attribute: foo + max-hits: 100 + } + } + } + """; + ApplicationBuilder.createFromString(schema); + fail("Expected exception"); + } + catch (IllegalArgumentException e) { + assertEquals(getMessagePrefix() + "does not exists", Exceptions.toMessageString(e)); + } } @Test void requireThatAttributeMustBeNumeric() throws Exception { - assertBuildFails("src/test/examples/matchphase/wrong_data_type_attribute.sd", - getMessagePrefix() + "must be single value numeric, but it is 'string'"); + try { + var schema = """ + search test { + document test { + field foo type string { + indexing: attribute + } + } + rank-profile default { + match-phase { + attribute: foo + max-hits: 100 + } + } + } + """; + ApplicationBuilder.createFromString(schema); + fail("Expected exception"); + } + catch (IllegalArgumentException e) { + assertEquals(getMessagePrefix() + "must be single value numeric, but it is 'string'", + Exceptions.toMessageString(e)); + } } @Test void requireThatAttributeMustBeSingleValue() throws Exception { - assertBuildFails("src/test/examples/matchphase/wrong_collection_type_attribute.sd", - getMessagePrefix() + "must be single value numeric, but it is 'Array<int>'"); + try { + var schema = """ + search test { + document test { + field foo type array<int> { + indexing: attribute + } + } + rank-profile default { + match-phase { + attribute: foo + max-hits: 100 + } + } + } + """; + ApplicationBuilder.createFromString(schema); + fail("Expected exception"); + } + catch (IllegalArgumentException e) { + assertEquals(getMessagePrefix() + "must be single value numeric, but it is 'Array<int>'", + Exceptions.toMessageString(e)); + } } @Test void requireThatAttributeMustHaveFastSearch() throws Exception { - assertBuildFails("src/test/examples/matchphase/non_fast_search_attribute.sd", - getMessagePrefix() + "must be fast-search, but it is not"); + try { + var schema = """ + search test { + document test { + field foo type int { + indexing: attribute + } + } + rank-profile default { + match-phase { + attribute: foo + max-hits: 100 + } + } + } + """; + ApplicationBuilder.createFromString(schema); + fail("Expected exception"); + } + catch (IllegalArgumentException e) { + assertEquals(getMessagePrefix() + "must be fast-search, but it is not", + Exceptions.toMessageString(e)); + } } + } diff --git a/config-model/src/test/java/com/yahoo/schema/processing/NGramTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/NGramTestCase.java index 06ea202b9c3..551542a84ba 100644 --- a/config-model/src/test/java/com/yahoo/schema/processing/NGramTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/processing/NGramTestCase.java @@ -27,15 +27,15 @@ public class NGramTestCase extends AbstractSchemaTestCase { SDField gram1 = schema.getConcreteField("gram_1"); assertEquals(MatchType.GRAM, gram1.getMatching().getType()); - assertEquals(1, gram1.getMatching().getGramSize()); + assertEquals(1, gram1.getMatching().getGramSize().getAsInt()); SDField gram2 = schema.getConcreteField("gram_2"); assertEquals(MatchType.GRAM, gram2.getMatching().getType()); - assertEquals(-1, gram2.getMatching().getGramSize()); // Not set explicitly + assertTrue(gram2.getMatching().getGramSize().isEmpty()); SDField gram3 = schema.getConcreteField("gram_3"); assertEquals(MatchType.GRAM, gram3.getMatching().getType()); - assertEquals(3, gram3.getMatching().getGramSize()); + assertEquals(3, gram3.getMatching().getGramSize().getAsInt()); assertEquals("input gram_1 | ngram 1 | index gram_1 | summary gram_1", gram1.getIndexingScript().iterator().next().toString()); assertEquals("input gram_2 | ngram 2 | attribute gram_2 | index gram_2", gram2.getIndexingScript().iterator().next().toString()); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsConsumersTest.java b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsConsumersTest.java index 49019e47bc2..eae4f12f62c 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsConsumersTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsConsumersTest.java @@ -52,11 +52,12 @@ public class MetricsConsumersTest { @Test void consumers_are_set_up_for_hosted() { ConsumersConfig config = consumersConfigFromXml(servicesWithAdminOnly(), hosted); - assertEquals(4, config.consumer().size()); + assertEquals(5, config.consumer().size()); assertEquals(MetricsConsumer.vespa.id(), config.consumer(0).name()); assertEquals(MetricsConsumer.autoscaling.id(), config.consumer(1).name()); assertEquals(MetricsConsumer.defaultConsumer.id(), config.consumer(2).name()); assertEquals(MetricsProxyContainerCluster.NEW_DEFAULT_CONSUMER_ID, config.consumer(3).name()); + assertEquals(MetricsConsumer.vespa9.id(), config.consumer(4).name()); } @Test @@ -124,7 +125,7 @@ public class MetricsConsumersTest { ); VespaModel hostedModel = getModel(services, hosted); ConsumersConfig config = consumersConfigFromModel(hostedModel); - assertEquals(4, config.consumer().size()); + assertEquals(5, config.consumer().size()); // All default metrics are retained ConsumersConfig.Consumer vespaConsumer = config.consumer(0); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/JvmHeapSizeValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/JvmHeapSizeValidatorTest.java new file mode 100644 index 00000000000..9b3b659c252 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/JvmHeapSizeValidatorTest.java @@ -0,0 +1,130 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.vespa.model.application.validation; + +import com.yahoo.config.ModelReference; +import com.yahoo.config.application.api.ApplicationFile; +import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.config.model.NullConfigModelRegistry; +import com.yahoo.config.model.api.OnnxModelCost; +import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.config.model.deploy.TestProperties; +import com.yahoo.config.model.provision.InMemoryProvisioner; +import com.yahoo.config.model.test.MockApplicationPackage; +import com.yahoo.config.provision.NodeResources; +import com.yahoo.vespa.model.VespaModel; +import org.junit.jupiter.api.Test; +import org.xml.sax.SAXException; + +import java.io.IOException; +import java.net.URI; +import java.util.concurrent.atomic.AtomicLong; + +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * @author bjorncs + */ +class JvmHeapSizeValidatorTest { + + @Test + void fails_on_too_low_jvm_percentage() throws IOException, SAXException { + var deployState = createDeployState(8, 7L * 1024 * 1024 * 1024); + var model = new VespaModel(new NullConfigModelRegistry(), deployState); + var e = assertThrows(IllegalArgumentException.class, () -> new JvmHeapSizeValidator().validate(model, deployState)); + String expectedMessage = "Allocated percentage of memory of JVM in cluster 'container' is too low (3% < 15%). Estimated cost of ONNX models is 7.00GB"; + assertTrue(e.getMessage().contains(expectedMessage), e.getMessage()); + } + + @Test + void fails_on_too_low_heap_size() throws IOException, SAXException { + var deployState = createDeployState(2.2, 1024L * 1024 * 1024); + var model = new VespaModel(new NullConfigModelRegistry(), deployState); + var e = assertThrows(IllegalArgumentException.class, () -> new JvmHeapSizeValidator().validate(model, deployState)); + String expectedMessage = "Allocated memory to JVM in cluster 'container' is too low (0.50GB < 0.60GB). Estimated cost of ONNX models is 1.00GB."; + assertTrue(e.getMessage().contains(expectedMessage), e.getMessage()); + } + + @Test + void accepts_adequate_heap_size() throws IOException, SAXException { + var deployState = createDeployState(8, 1024L * 1024 * 1024); + var model = new VespaModel(new NullConfigModelRegistry(), deployState); + assertDoesNotThrow(() -> new JvmHeapSizeValidator().validate(model, deployState)); + } + + @Test + void accepts_services_with_explicit_jvm_size() throws IOException, SAXException { + String servicesXml = + """ + <?xml version="1.0" encoding="utf-8" ?> + <services version='1.0'> + <container version='1.0'> + <nodes count="2"> + <jvm allocated-memory='5%'/> + <resources vcpu="4" memory="2Gb" disk="125Gb"/> + </nodes> + <component id="hf-embedder" type="hugging-face-embedder"> + <transformer-model url="https://my/url/model.onnx"/> + <tokenizer-model path="app/tokenizer.json"/> + </component> + </container> + </services>"""; + var deployState = createDeployState(servicesXml, 2, 1024L * 1024 * 1024); + var model = new VespaModel(new NullConfigModelRegistry(), deployState); + assertDoesNotThrow(() -> new JvmHeapSizeValidator().validate(model, deployState)); + } + + private static DeployState createDeployState(String servicesXml, double nodeGb, long modelCostBytes) { + return new DeployState.Builder() + .applicationPackage( + new MockApplicationPackage.Builder() + .withServices(servicesXml) + .build()) + .modelHostProvisioner(new InMemoryProvisioner(5, new NodeResources(4, nodeGb, 125, 0.3), true)) + .properties(new TestProperties().setHostedVespa(true).setDynamicHeapSize(true)) + .onnxModelCost(new ModelCostDummy(modelCostBytes)) + .build(); + } + + private static DeployState createDeployState(double nodeGb, long modelCostBytes) { + String servicesXml = + """ + <?xml version="1.0" encoding="utf-8" ?> + <services version='1.0'> + <container version='1.0'> + <nodes count="2"> + <resources vcpu="4" memory="%fGb" disk="125Gb"/> + </nodes> + <component id="hf-embedder" type="hugging-face-embedder"> + <transformer-model url="https://my/url/model.onnx"/> + <tokenizer-model path="app/tokenizer.json"/> + </component> + </container> + </services>""".formatted(nodeGb); + return createDeployState(servicesXml, nodeGb, modelCostBytes); + } + + private static class ModelCostDummy implements OnnxModelCost, OnnxModelCost.Calculator { + final AtomicLong totalCost = new AtomicLong(); + final long modelCost; + + ModelCostDummy(long modelCost) { this.modelCost = modelCost; } + + @Override public Calculator newCalculator(ApplicationPackage appPkg, DeployLogger logger) { return this; } + @Override public long aggregatedModelCostInBytes() { return totalCost.get(); } + @Override public void registerModel(ApplicationFile path) {} + + @SuppressWarnings("removal") @Override public void registerModel(ModelReference ref) {} + + @Override + public void registerModel(URI uri) { + assertEquals("https://my/url/model.onnx", uri.toString()); + totalCost.addAndGet(modelCost); + } + } + +}
\ No newline at end of file diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/UrlConfigValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/UrlConfigValidatorTest.java new file mode 100644 index 00000000000..cef4d8c27dd --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/UrlConfigValidatorTest.java @@ -0,0 +1,107 @@ +package com.yahoo.vespa.model.application.validation; + +import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.config.model.NullConfigModelRegistry; +import com.yahoo.config.model.api.ConfigDefinitionRepo; +import com.yahoo.config.model.application.provider.MockFileRegistry; +import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.config.model.deploy.TestProperties; +import com.yahoo.config.model.test.MockApplicationPackage; +import com.yahoo.config.provision.RegionName; +import com.yahoo.config.provision.SystemName; +import com.yahoo.config.provision.Zone; +import com.yahoo.embedding.BertBaseEmbedderConfig; +import com.yahoo.vespa.config.ConfigDefinitionKey; +import com.yahoo.vespa.config.buildergen.ConfigDefinition; +import com.yahoo.vespa.model.VespaModel; +import org.junit.jupiter.api.Test; +import org.xml.sax.SAXException; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +import static com.yahoo.config.provision.Environment.prod; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +public class UrlConfigValidatorTest { + + @Test + void failsWhenContainerNodesNotExclusive() throws IOException, SAXException { + runValidatorOnApp(true, SystemName.Public); // Exclusive nodes in public => success + + assertEquals("Found s3:// urls in config for container cluster default. This is only supported in public systems", + assertThrows(IllegalArgumentException.class, + () -> runValidatorOnApp(false, SystemName.main)) + .getMessage()); + + assertEquals("Found s3:// urls in config for container cluster default. This is only supported in public systems", + assertThrows(IllegalArgumentException.class, + () -> runValidatorOnApp(true, SystemName.main)) + .getMessage()); + + assertEquals("Found s3:// urls in config for container cluster default. Nodes in the cluster need to be 'exclusive'," + + " see https://cloud.vespa.ai/en/reference/services#nodes", + assertThrows(IllegalArgumentException.class, + () -> runValidatorOnApp(false, SystemName.Public)) + .getMessage()); + } + + private static String containerXml(boolean isExclusive) { + return """ + <container version='1.0' id='default'> + <component id='transformer' class='ai.vespa.embedding.BertBaseEmbedder' bundle='model-integration'> + <config name='embedding.bert-base-embedder'> + <transformerModel url='s3://models/minilm-l6-v2/sentence_all_MiniLM_L6_v2.onnx' path='foo'/> + <tokenizerVocab url='s3://models/bert-base-uncased.txt'/> + </config> + </component> + <search/> + <document-api/> + <nodes count='2' exclusive='%s' /> + </container> + """.formatted(Boolean.toString(isExclusive)); + } + + private static void runValidatorOnApp(boolean isExclusive, SystemName systemName) throws IOException, SAXException { + String container = containerXml(isExclusive); + String servicesXml = """ + <services version='1.0'> + %s + </services> + """.formatted(container); + ApplicationPackage app = new MockApplicationPackage.Builder() + .withServices(servicesXml) + .build(); + DeployState deployState = createDeployState(app, systemName); + VespaModel model = new VespaModel(new NullConfigModelRegistry(), deployState); + new UrlConfigValidator().validate(model, deployState); + } + + private static DeployState createDeployState(ApplicationPackage app, SystemName systemName) { + boolean isHosted = true; + var builder = new DeployState.Builder() + .applicationPackage(app) + .zone(new Zone(systemName, prod, RegionName.from("us-east-3"))) + .properties(new TestProperties().setHostedVespa(isHosted)) + .fileRegistry(new MockFileRegistry()); + + Map<ConfigDefinitionKey, ConfigDefinition> defs = new HashMap<>(); + defs.put(new ConfigDefinitionKey(BertBaseEmbedderConfig.CONFIG_DEF_NAME, BertBaseEmbedderConfig.CONFIG_DEF_NAMESPACE), + new ConfigDefinition(BertBaseEmbedderConfig.CONFIG_DEF_NAME, BertBaseEmbedderConfig.CONFIG_DEF_SCHEMA)); + builder.configDefinitionRepo(new ConfigDefinitionRepo() { + @Override + public Map<ConfigDefinitionKey, com.yahoo.vespa.config.buildergen.ConfigDefinition> getConfigDefinitions() { + return defs; + } + + @Override + public com.yahoo.vespa.config.buildergen.ConfigDefinition get(ConfigDefinitionKey key) { + return defs.get(key); + } + }); + return builder.build(); + } + +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java index 894fc55c014..3bdb60a0a8d 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java @@ -42,14 +42,14 @@ import java.util.Objects; import java.util.OptionalInt; import java.util.Set; -import static com.yahoo.config.model.api.ApplicationClusterEndpoint.RoutingMethod.exclusive; -import static com.yahoo.config.model.api.ApplicationClusterEndpoint.RoutingMethod.shared; import static com.yahoo.config.model.api.ApplicationClusterEndpoint.RoutingMethod.sharedLayer4; import static com.yahoo.config.model.api.ApplicationClusterEndpoint.Scope.application; import static com.yahoo.config.model.api.ApplicationClusterEndpoint.Scope.global; -import static com.yahoo.config.provision.SystemName.cd; +import static com.yahoo.config.model.api.ApplicationClusterEndpoint.Scope.zone; import static com.yahoo.config.provision.SystemName.main; -import static org.junit.jupiter.api.Assertions.*; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; /** * @author Simon Thoresen Hult @@ -365,62 +365,23 @@ public class ContainerClusterTest { @Test void generatesCorrectRoutingInfo() { - // main system: assertNames(main, ApplicationId.from("t1", "a1", "i1"), - Set.of(), + Set.of(new ContainerEndpoint("search-cluster", zone, List.of("search-cluster.i1.a1.t1.endpoint.suffix"), OptionalInt.empty(), sharedLayer4)), List.of("search-cluster.i1.a1.t1.endpoint.suffix")); assertNames(main, ApplicationId.from("t1", "a1", "default"), - Set.of(), - List.of("search-cluster.a1.t1.endpoint.suffix")); - - assertNames(main, - ApplicationId.from("t1", "default", "default"), - Set.of(), - List.of("search-cluster.default.t1.endpoint.suffix")); - - assertNames(main, - ApplicationId.from("t1", "a1", "default"), - Set.of(new ContainerEndpoint("not-in-this-cluster", global, List.of("foo", "bar"))), + Set.of(new ContainerEndpoint("not-in-this-cluster", global, List.of("foo", "bar")), + new ContainerEndpoint("search-cluster", zone, List.of("search-cluster.a1.t1.endpoint.suffix"), OptionalInt.empty(), sharedLayer4)), List.of("search-cluster.a1.t1.endpoint.suffix")); assertNames(main, ApplicationId.from("t1", "a1", "default"), - Set.of(new ContainerEndpoint("search-cluster", global, List.of("rotation-1.x.y.z", "rotation-2.x.y.z"), OptionalInt.empty(), sharedLayer4), - new ContainerEndpoint("search-cluster", application, List.of("app-rotation.x.y.z"), OptionalInt.of(3), sharedLayer4)), + Set.of(new ContainerEndpoint("search-cluster", global, List.of("rotation-1.x.y.z", "rotation-2.x.y.z"), OptionalInt.empty(), sharedLayer4), + new ContainerEndpoint("search-cluster", application, List.of("app-rotation.x.y.z"), OptionalInt.of(3), sharedLayer4), + new ContainerEndpoint("search-cluster", zone, List.of("search-cluster.a1.t1.endpoint.suffix"), OptionalInt.empty(), sharedLayer4)), List.of("search-cluster.a1.t1.endpoint.suffix", "rotation-1.x.y.z", "rotation-2.x.y.z", "app-rotation.x.y.z")); - - // cd system: - assertNames(cd, - ApplicationId.from("t1", "a1", "i1"), - Set.of(), - List.of("search-cluster.cd.i1.a1.t1.endpoint.suffix")); - - assertNames(cd, - ApplicationId.from("t1", "a1", "default"), - Set.of(), - List.of("search-cluster.cd.a1.t1.endpoint.suffix")); - - assertNames(cd, - ApplicationId.from("t1", "default", "default"), - Set.of(), - List.of("search-cluster.cd.default.t1.endpoint.suffix")); - - assertNames(cd, - ApplicationId.from("t1", "a1", "default"), - Set.of(new ContainerEndpoint("not-in-this-cluster", global, List.of("foo", "bar"))), - List.of("search-cluster.cd.a1.t1.endpoint.suffix")); - - assertNames(cd, - ApplicationId.from("t1", "a1", "default"), - Set.of(new ContainerEndpoint("search-cluster", global, List.of("rotation-1.x.y.z", "rotation-2.x.y.z"), OptionalInt.empty(), sharedLayer4), - new ContainerEndpoint("search-cluster", global, List.of("a.b.x.y.z", "rotation-2.x.y.z"), OptionalInt.empty(), shared), - new ContainerEndpoint("search-cluster", application, List.of("app-rotation.x.y.z"), OptionalInt.of(3), sharedLayer4), - new ContainerEndpoint("not-supported", global, List.of("not.supported"), OptionalInt.empty(), exclusive)), - List.of("search-cluster.cd.a1.t1.endpoint.suffix", "rotation-1.x.y.z", "rotation-2.x.y.z", "app-rotation.x.y.z")); - } private void assertNames(SystemName systemName, ApplicationId appId, Set<ContainerEndpoint> globalEndpoints, List<String> expectedSharedL4Names) { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java index 94d98f526a0..2dbf49ba61c 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java @@ -59,7 +59,6 @@ import java.util.List; import java.util.Map; import java.util.Set; import java.util.function.Function; -import java.util.logging.Level; import static com.yahoo.config.model.test.TestUtil.joinLines; import static com.yahoo.test.LinePatternMatcher.containsLineWithPattern; diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/SearchBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/SearchBuilderTest.java index 25c6c0ccb1c..9e8e6ff9c71 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/SearchBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/SearchBuilderTest.java @@ -220,6 +220,26 @@ public class SearchBuilderTest extends ContainerModelBuilderTestBase { } @Test + void threadpool_configuration_allow_new_syntax() { + Element clusterElem = DomBuilderTest.parse( + "<container id='default' version='1.0'>", + " <search>", + " <threadpool>", + " <threads boost=\"10.2\">0.4</threads>", + " <queue>50</queue>", + " </threadpool>", + " </search>", + nodesXml, + "</container>"); + createModel(root, clusterElem); + ContainerThreadpoolConfig config = root.getConfig( + ContainerThreadpoolConfig.class, "default/component/" + SearchHandler.HANDLER_CLASSNAME + "/threadpool@search-handler"); + assertEquals(-10, config.maxThreads()); + assertEquals(-1, config.minThreads()); + assertEquals(-50, config.queueSize()); + } + + @Test void ExecutionFactory_gets_same_chains_config_as_SearchHandler() { createBasicSearchModel(); Component<?, ?> executionFactory = ((SearchHandler) getComponent("default", SearchHandler.HANDLER_CLASSNAME)) diff --git a/config-model/src/test/java/com/yahoo/vespa/model/filedistribution/UserConfiguredFilesTest.java b/config-model/src/test/java/com/yahoo/vespa/model/filedistribution/UserConfiguredFilesTest.java index bb5ba840c2c..deea1a820d9 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/filedistribution/UserConfiguredFilesTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/filedistribution/UserConfiguredFilesTest.java @@ -7,12 +7,14 @@ import com.yahoo.config.ModelReference; import com.yahoo.config.UrlReference; import com.yahoo.config.application.api.FileRegistry; import com.yahoo.config.model.application.provider.BaseDeployLogger; +import com.yahoo.config.model.deploy.TestProperties; import com.yahoo.config.model.producer.UserConfigRepo; import com.yahoo.config.model.test.MockRoot; import com.yahoo.vespa.config.ConfigDefinition; import com.yahoo.vespa.config.ConfigDefinitionKey; import com.yahoo.vespa.config.ConfigPayloadBuilder; import com.yahoo.vespa.model.SimpleConfigProducer; +import com.yahoo.vespa.model.container.ApplicationContainerCluster; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; @@ -30,6 +32,7 @@ import static org.junit.jupiter.api.Assertions.fail; /** * @author Ulf Lilleengen + * @author hmusum */ public class UserConfiguredFilesTest { @@ -68,7 +71,10 @@ public class UserConfiguredFilesTest { } private UserConfiguredFiles userConfiguredFiles() { - return new UserConfiguredFiles(fileRegistry, new BaseDeployLogger()); + return new UserConfiguredFiles(fileRegistry, + new BaseDeployLogger(), + new TestProperties(), + new ApplicationContainerCluster.UserConfiguredUrls()); } @BeforeEach @@ -263,7 +269,8 @@ public class UserConfiguredFilesTest { userConfiguredFiles().register(producer); fail("Should have thrown exception"); } catch (IllegalArgumentException e) { - assertEquals("Unable to register file specified in services.xml for config 'mynamespace.myname': No such file or directory 'foo.txt'", e.getMessage()); + assertEquals("Invalid config in services.xml for 'mynamespace.myname': No such file or directory 'foo.txt'", + e.getMessage()); } } @@ -276,7 +283,7 @@ public class UserConfiguredFilesTest { userConfiguredFiles().register(producer); fail("Should have thrown exception"); } catch (IllegalArgumentException e) { - assertEquals("Unable to register file specified in services.xml for config 'mynamespace.myname': Unable to register file for field 'fileVal': Invalid config value '.'", + assertEquals("Invalid config in services.xml for 'mynamespace.myname': Invalid config value '.' for field 'fileVal", e.getMessage()); } } @@ -291,8 +298,7 @@ public class UserConfiguredFilesTest { userConfiguredFiles().register(producer); fail("Should have thrown exception"); } catch (IllegalArgumentException e) { - assertEquals("Unable to register file specified in services.xml for config 'mynamespace.myname': Directory '" + - relativeTempDir + "' is empty", + assertEquals("Invalid config in services.xml for 'mynamespace.myname': Directory '" + relativeTempDir + "' is empty", e.getMessage()); } } diff --git a/config-model/src/test/schema-test-files/services.xml b/config-model/src/test/schema-test-files/services.xml index e5cb7e8ef54..8a80c485b9d 100644 --- a/config-model/src/test/schema-test-files/services.xml +++ b/config-model/src/test/schema-test-files/services.xml @@ -252,5 +252,11 @@ <aws-parameter-store account="foo" aws-region="us-east-1"/> </store> </secret-store> + <search> + <threadpool> + <threads boost="32.0">8.0</threads> + <queue>40.0</queue> + </threadpool> + </search> </container> </services> |