diff options
Diffstat (limited to 'config-model/src/test/java')
31 files changed, 720 insertions, 171 deletions
diff --git a/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java b/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java index 5ead9812b56..737ac4c248c 100644 --- a/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java +++ b/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java @@ -60,7 +60,7 @@ import static com.yahoo.config.provision.NodeResources.DiskSpeed; import static com.yahoo.config.provision.NodeResources.StorageType; import static com.yahoo.vespa.defaults.Defaults.getDefaults; import static com.yahoo.vespa.model.Host.memoryOverheadGb; -import static com.yahoo.vespa.model.search.NodeResourcesTuning.GB; +import static com.yahoo.vespa.model.search.NodeResourcesTuning.GiB; import static com.yahoo.vespa.model.test.utils.ApplicationPackageUtils.generateSchemas; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; @@ -151,7 +151,7 @@ public class ModelProvisioningTest { assertEquals("-Xlog:gc", mydisc2.getContainers().get(1).getJvmOptions()); assertEquals("lib/blablamalloc.so", mydisc2.getContainers().get(0).getPreLoad()); assertEquals("lib/blablamalloc.so", mydisc2.getContainers().get(1).getPreLoad()); - assertEquals(45, mydisc2.getMemoryPercentage().get().percentage()); + assertEquals(45, mydisc2.getMemoryPercentage().get().ofContainerAvailable()); assertEquals(Optional.of("-XX:+UseParNewGC"), mydisc2.getJvmGCOptions()); QrStartConfig.Builder qrStartBuilder = new QrStartConfig.Builder(); mydisc2.getConfig(qrStartBuilder); @@ -234,8 +234,8 @@ public class ModelProvisioningTest { assertEquals(2, model.getContentClusters().get("content1").getRootGroup().getNodes().size(), "Nodes in content1"); assertEquals(1, model.getContainerClusters().get("container1").getContainers().size(), "Nodes in container1"); assertEquals(2, model.getContentClusters().get("content").getRootGroup().getNodes().size(), "Nodes in cluster without ID"); - assertEquals(65, physicalMemoryPercentage(model.getContainerClusters().get("container1")), "Heap size for container1"); - assertEquals(84, physicalMemoryPercentage(model.getContainerClusters().get("container2")), "Heap size for container2"); + assertEquals(85, physicalMemoryPercentage(model.getContainerClusters().get("container1")), "Heap size for container1"); + assertEquals(85, physicalMemoryPercentage(model.getContainerClusters().get("container2")), "Heap size for container2"); assertProvisioned(2, ClusterSpec.Id.from("content1"), ClusterSpec.Type.content, model); assertProvisioned(1, ClusterSpec.Id.from("container1"), ClusterSpec.Type.container, model); assertProvisioned(2, ClusterSpec.Id.from("content"), ClusterSpec.Type.content, model); @@ -287,8 +287,8 @@ public class ModelProvisioningTest { VespaModel model = tester.createModel(xmlWithNodes, true, deployStateWithClusterEndpoints("container1").deployLogger(logger)); assertEquals(2, model.getContentClusters().get("content1").getRootGroup().getNodes().size(), "Nodes in content1"); assertEquals(2, model.getContainerClusters().get("container1").getContainers().size(), "Nodes in container1"); - assertEquals(18, physicalMemoryPercentage(model.getContainerClusters().get("container1")), "Heap size is lowered with combined clusters"); - assertEquals(2025077080L, protonMemorySize(model.getContentClusters().get("content1")), "Memory for proton is lowered to account for the jvm heap"); + assertEquals(24, physicalMemoryPercentage(model.getContainerClusters().get("container1")), "Heap size is lowered with combined clusters"); + assertEquals(1876900708, protonMemorySize(model.getContentClusters().get("content1")), "Memory for proton is lowered to account for the jvm heap"); assertProvisioned(0, ClusterSpec.Id.from("container1"), ClusterSpec.Type.container, model); assertProvisioned(2, ClusterSpec.Id.from("content1"), ClusterSpec.Id.from("container1"), ClusterSpec.Type.combined, model); var msgs = logger.msgs().stream().filter(m -> m.level().equals(Level.WARNING)).toList(); @@ -356,7 +356,7 @@ public class ModelProvisioningTest { VespaModel model = tester.createModel(xmlWithNodes, true, deployStateWithClusterEndpoints("container1")); assertEquals(2, model.getContentClusters().get("content1").getRootGroup().getNodes().size(), "Nodes in content1"); assertEquals(2, model.getContainerClusters().get("container1").getContainers().size(), "Nodes in container1"); - assertEquals(65, physicalMemoryPercentage(model.getContainerClusters().get("container1")), "Heap size is normal"); + assertEquals(85, physicalMemoryPercentage(model.getContainerClusters().get("container1")), "Heap size is normal"); assertEquals((long) ((3 - memoryOverheadGb) * (Math.pow(1024, 3))), protonMemorySize(model.getContentClusters().get("content1")), "Memory for proton is normal"); } @@ -2580,7 +2580,7 @@ public class ModelProvisioningTest { ProtonConfig cfg = getProtonConfig(model, cluster.getSearchNodes().get(0).getConfigId()); assertEquals(2000, cfg.flush().memory().maxtlssize()); // from config override assertEquals(1000, cfg.flush().memory().maxmemory()); // from explicit tuning - assertEquals((long) ((128 - memoryOverheadGb) * GB * 0.08), cfg.flush().memory().each().maxmemory()); // from default node flavor tuning + assertEquals((long) ((128 - memoryOverheadGb) * GiB * 0.08), cfg.flush().memory().each().maxmemory()); // from default node flavor tuning } private static ProtonConfig getProtonConfig(VespaModel model, String configId) { diff --git a/config-model/src/test/java/com/yahoo/schema/DiversityTestCase.java b/config-model/src/test/java/com/yahoo/schema/DiversityTestCase.java index df71e30c7d9..d9355e2df7d 100644 --- a/config-model/src/test/java/com/yahoo/schema/DiversityTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/DiversityTestCase.java @@ -3,49 +3,58 @@ package com.yahoo.schema; import com.yahoo.search.query.ranking.Diversity; import com.yahoo.schema.parser.ParseException; +import com.yahoo.vespa.model.test.utils.DeployLoggerStub; import org.junit.jupiter.api.Test; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.fail; /** * @author baldersheim */ public class DiversityTestCase { - @Test - void testDiversity() throws ParseException { + private static void verifyDiversity(DeployLoggerStub logger, boolean atRankProfile, boolean atMatchPhase) throws ParseException { RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); - ApplicationBuilder builder = new ApplicationBuilder(rankProfileRegistry); + ApplicationBuilder builder = new ApplicationBuilder(logger, rankProfileRegistry); + String diversitySpec = """ + diversity { + attribute: b + min-groups: 74 + cutoff-factor: 17.3 + cutoff-strategy: strict + } + """; builder.addSchema( - "search test {\n" + - " document test { \n" + - " field a type int { \n" + - " indexing: attribute \n" + - " attribute: fast-search\n" + - " }\n" + - " field b type int {\n" + - " indexing: attribute \n" + - " }\n" + - " }\n" + - " \n" + - " rank-profile parent {\n" + - " match-phase {\n" + - " diversity {\n" + - " attribute: b\n" + - " min-groups: 74\n" + - " cutoff-factor: 17.3\n" + - " cutoff-strategy: strict" + - " }\n" + - " attribute: a\n" + - " max-hits: 120\n" + - " max-filter-coverage: 0.065" + - " }\n" + - " }\n" + - "}\n"); + """ + search test { + document test { + field a type int { + indexing: attribute + attribute: fast-search + } + field b type int { + indexing: attribute + } + } + rank-profile parent { + match-phase {""" + + (atMatchPhase ? diversitySpec : "") + + """ + attribute: a + max-hits: 120 + max-filter-coverage: 0.065 + }""" + + (atRankProfile ? diversitySpec : "") + + """ + } + } + """); builder.build(true); Schema s = builder.getSchema(); - RankProfile.MatchPhaseSettings matchPhase = rankProfileRegistry.get(s, "parent").getMatchPhaseSettings(); - RankProfile.DiversitySettings diversity = matchPhase.getDiversity(); + RankProfile parent = rankProfileRegistry.get(s, "parent"); + RankProfile.MatchPhaseSettings matchPhase = parent.getMatchPhase(); + RankProfile.DiversitySettings diversity = parent.getDiversity(); assertEquals("b", diversity.getAttribute()); assertEquals(74, diversity.getMinGroups()); assertEquals(17.3, diversity.getCutoffFactor(), 1e-16); @@ -54,6 +63,44 @@ public class DiversityTestCase { assertEquals("a", matchPhase.getAttribute()); assertEquals(0.065, matchPhase.getMaxFilterCoverage(), 1e-16); } + @Test + void testDiversity() throws ParseException { + DeployLoggerStub logger = new DeployLoggerStub(); + verifyDiversity(logger, true, false); + assertTrue(logger.entries.isEmpty()); + verifyDiversity(logger, false, true); + assertEquals(1, logger.entries.size()); + assertEquals("'diversity is deprecated inside 'match-phase'. Specify it at 'rank-profile' level.", logger.entries.get(0).message); + try { + verifyDiversity(logger, true, true); + fail("Should throw."); + } catch (Exception e) { + assertEquals("rank-profile 'parent' error: already has diversity", e.getMessage()); + } + } + + @Test + void requireMatchPhaseOrSecondPhase() throws ParseException { + ApplicationBuilder builder = new ApplicationBuilder(new RankProfileRegistry()); + builder.addSchema(""" + search test { + document test { + field b type int { indexing: attribute } + } + rank-profile parent { + diversity { + attribute: b + min-groups: 74 + } + } + }"""); + try { + builder.build(true); + fail("Should throw."); + } catch (IllegalArgumentException e) { + assertEquals("In schema 'test', rank-profile 'parent': 'diversity' requires either 'match-phase' or 'second-phase' to be specified.", e.getMessage()); + } + } private static String getMessagePrefix() { return "In search definition 'test', rank-profile 'parent': diversity attribute 'b' "; @@ -82,30 +129,29 @@ public class DiversityTestCase { assertEquals(getMessagePrefix() + "must be single value numeric, or enumerated attribute, but it is 'Array<int>'", e.getMessage()); } } - private ApplicationBuilder getSearchBuilder(String diversity) throws ParseException { - RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); - ApplicationBuilder builder = new ApplicationBuilder(rankProfileRegistry); - builder.addSchema( - "search test {\n" + - " document test { \n" + - " field a type int { \n" + - " indexing: attribute \n" + - " attribute: fast-search\n" + - " }\n" + - diversity + - " }\n" + - " \n" + - " rank-profile parent {\n" + - " match-phase {\n" + - " diversity {\n" + - " attribute: b\n" + - " min-groups: 74\n" + - " }\n" + - " attribute: a\n" + - " max-hits: 120\n" + - " }\n" + - " }\n" + - "}\n"); + private ApplicationBuilder getSearchBuilder(String diversityField) throws ParseException { + ApplicationBuilder builder = new ApplicationBuilder(new RankProfileRegistry()); + builder.addSchema(""" + search test { + document test { + field a type int { + indexing: attribute + attribute: fast-search + }""" + + diversityField + + """ + } + rank-profile parent { + match-phase { + attribute: a + max-hits: 120 + } + diversity { + attribute: b + min-groups: 74 + } + } + }"""); return builder; } } diff --git a/config-model/src/test/java/com/yahoo/schema/RankProfileTestCase.java b/config-model/src/test/java/com/yahoo/schema/RankProfileTestCase.java index 59887bfd57e..564bfd4a990 100644 --- a/config-model/src/test/java/com/yahoo/schema/RankProfileTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/RankProfileTestCase.java @@ -530,4 +530,28 @@ public class RankProfileTestCase extends AbstractSchemaTestCase { "}"); } + @Test + public void secondPhaseRankScoreDropLimitIsAddedToRankProperties() throws ParseException { + RankProfileRegistry registry = new RankProfileRegistry(); + ApplicationBuilder builder = new ApplicationBuilder(registry); + String input = """ + schema test { + document test { + } + rank-profile test inherits default { + second-phase { + rank-score-drop-limit: 17.0 + } + } + } + """; + builder.addSchema(input); + builder.build(true); + Schema schema = builder.getSchema(); + + assertEquals(3, registry.all().size()); + RawRankProfile rawProfile = createRawRankProfile(registry.get(schema, "test"), schema); + assertEquals("17.0", findProperty(rawProfile.configProperties(), "vespa.hitcollector.secondphase.rankscoredroplimit").get()); + } + } diff --git a/config-model/src/test/java/com/yahoo/schema/SummaryTestCase.java b/config-model/src/test/java/com/yahoo/schema/SummaryTestCase.java index 8ffbab84fd7..539ec91ee5f 100644 --- a/config-model/src/test/java/com/yahoo/schema/SummaryTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/SummaryTestCase.java @@ -6,16 +6,13 @@ import com.yahoo.vespa.documentmodel.DocumentSummary; import com.yahoo.vespa.model.test.utils.DeployLoggerStub; import com.yahoo.vespa.objects.FieldBase; import com.yahoo.yolean.Exceptions; -import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import static com.yahoo.config.model.test.TestUtil.joinLines; import java.util.Collection; import java.util.List; -import java.util.Optional; import java.util.logging.Level; -import java.util.stream.Collectors; import static org.junit.jupiter.api.Assertions.*; @@ -233,16 +230,12 @@ public class SummaryTestCase { " document-summary test_summary inherits nonesuch {" + " }" + "}"); - DeployLoggerStub logger = new DeployLoggerStub(); - ApplicationBuilder.createFromStrings(logger, schema); - assertEquals("document-summary 'test_summary' inherits 'nonesuch' but this is not present in schema 'test'", - logger.entries.get(0).message); - // fail("Expected failure"); + ApplicationBuilder.createFromString(schema); + fail("Expected failure"); } catch (IllegalArgumentException e) { - fail(); - // assertEquals("document-summary 'test_summary' inherits nonesuch but this is not present in schema 'test'", - // e.getMessage()); + assertEquals("document-summary 'test_summary' inherits 'nonesuch', but this is not present in schema 'test'", + e.getMessage()); } } diff --git a/config-model/src/test/java/com/yahoo/schema/parser/IntermediateCollectionTestCase.java b/config-model/src/test/java/com/yahoo/schema/parser/IntermediateCollectionTestCase.java index 8d2c98439ca..af6fb82ee7b 100644 --- a/config-model/src/test/java/com/yahoo/schema/parser/IntermediateCollectionTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/parser/IntermediateCollectionTestCase.java @@ -1,6 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.schema.parser; +import com.yahoo.document.config.DocumentmanagerConfig; import com.yahoo.io.reader.NamedReader; import static com.yahoo.config.model.test.TestUtil.joinLines; @@ -232,5 +233,16 @@ public class IntermediateCollectionTestCase { assertTrue(ex.getMessage().startsWith("Inheritance/reference cycle for documents: ")); } + @Test + void can_detect_errors_in_rank_profile_outside_schema() { + var collection = new IntermediateCollection(); + collection.addSchemaFromFile("src/test/derived/rankprofilemodularity/test.sd"); + var exception = assertThrows(ParseException.class, () -> { + collection.addRankProfileFile("test", "src/test/derived/rankprofilemodularity2/invalid_comment.profile"); + }); + var message = exception.getMessage(); + assertTrue(message.contains("Failed parsing rank-profile from 'src/test/derived/rankprofilemodularity2/invalid_comment.profile'")); + assertTrue(message.contains("Lexical error at line 3, column 6"), message); + } } diff --git a/config-model/src/test/java/com/yahoo/schema/parser/SchemaParserTestCase.java b/config-model/src/test/java/com/yahoo/schema/parser/SchemaParserTestCase.java index 5a2dc218da7..4186e352388 100644 --- a/config-model/src/test/java/com/yahoo/schema/parser/SchemaParserTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/parser/SchemaParserTestCase.java @@ -121,6 +121,39 @@ public class SchemaParserTestCase { } @Test + void significance_can_be_parsed() throws Exception { + String input = """ + schema foo { + rank-profile significance-ranking-0 inherits default { + significance { + use-model: true + } + } + rank-profile significance-ranking-1 { + significance { + use-model: false + } + } + } + """; + + ParsedSchema schema = parseString(input); + assertEquals("foo", schema.name()); + var rplist = schema.getRankProfiles(); + assertEquals(2, rplist.size()); + + var rp0 = rplist.get(0); + assertEquals("significance-ranking-0", rp0.name()); + assertTrue(rp0.isUseSignificanceModel().isPresent()); + assertTrue(rp0.isUseSignificanceModel().get()); + + var rp1 = rplist.get(1); + assertEquals("significance-ranking-1", rp1.name()); + assertTrue(rp1.isUseSignificanceModel().isPresent()); + assertFalse(rp1.isUseSignificanceModel().get()); + } + + @Test void maxOccurrencesCanBeParsed() throws Exception { String input = joinLines ("schema foo {", @@ -137,6 +170,23 @@ public class SchemaParserTestCase { assertEquals(11, field.matchSettings().getMaxTermOccurrences().get()); } + @Test + void maxTokenLengthCanBeParsed() throws Exception { + String input = joinLines + ("schema foo {", + " document foo {", + " field bar type string {", + " indexing: summary | index", + " match { max-token-length: 11 }", + " }", + " }", + "}"); + ParsedSchema schema = parseString(input); + var field = schema.getDocument().getFields().get(0); + assertEquals("bar", field.name()); + assertEquals(11, field.matchSettings().getMaxTokenLength().get()); + } + void checkFileParses(String fileName) throws Exception { var schema = parseFile(fileName); assertNotNull(schema); diff --git a/config-model/src/test/java/com/yahoo/schema/processing/IndexingScriptRewriterTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/IndexingScriptRewriterTestCase.java index de99d46b9ca..355a810f5ff 100644 --- a/config-model/src/test/java/com/yahoo/schema/processing/IndexingScriptRewriterTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/processing/IndexingScriptRewriterTestCase.java @@ -10,6 +10,7 @@ import com.yahoo.schema.Schema; import com.yahoo.schema.ApplicationBuilder; import com.yahoo.schema.AbstractSchemaTestCase; import com.yahoo.schema.document.BooleanIndexDefinition; +import com.yahoo.schema.document.MatchType; import com.yahoo.schema.document.SDDocumentType; import com.yahoo.schema.document.SDField; import com.yahoo.vespa.documentmodel.SummaryField; @@ -155,6 +156,24 @@ public class IndexingScriptRewriterTestCase extends AbstractSchemaTestCase { field); } + @Test + void requireThatMaxTokenLengthIsPropagated() { + var field = new SDField("test", DataType.STRING); + field.getMatching().maxTokenLength(10); + field.parseIndexingScript("test", "{ summary | index }"); + assertIndexingScript("{ input test | tokenize normalize stem:\"BEST\" max-token-length:10 | summary test | index test; }", + field); + } + + @Test + void requireThatMaxTokenLengthIsPropagatedForWordMatch() { + var field = new SDField("test", DataType.STRING); + field.getMatching().maxTokenLength(10).setType(MatchType.WORD); + field.parseIndexingScript("test", "{ summary | index }"); + assertIndexingScript("{ input test | exact max-token-length:10 | summary test | index test; }", + field); + } + private static void assertIndexingScript(String expectedScript, SDField unprocessedField) { assertEquals(expectedScript, processField(unprocessedField).toString()); diff --git a/config-model/src/test/java/com/yahoo/schema/processing/SummaryConsistencyTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/SummaryConsistencyTestCase.java index 9eca2106c5e..5ea097f13fb 100644 --- a/config-model/src/test/java/com/yahoo/schema/processing/SummaryConsistencyTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/processing/SummaryConsistencyTestCase.java @@ -9,6 +9,7 @@ import org.junit.jupiter.api.Test; import static com.yahoo.config.model.test.TestUtil.joinLines; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; public class SummaryConsistencyTestCase { @@ -42,4 +43,24 @@ public class SummaryConsistencyTestCase { Schema schema = ApplicationBuilder.createFromString(sd).getSchema(); assertEquals(SummaryTransform.ATTRIBUTECOMBINER, schema.getSummaryField("elem_array_unfiltered").getTransform()); } + + @Test + void testDocumentSummaryWithInheritanceOfNonExistingSummary() { + String schemaString = """ + schema foo { + document foo { + field foo type string { + indexing: summary + } + } + document-summary foo_summary inherits non-existent { + summary foo { + source: foo + } + } + } + """; + assertThrows(IllegalArgumentException.class, () -> ApplicationBuilder.createFromString(schemaString).getSchema()); + } + } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/ClusterInfoTest.java b/config-model/src/test/java/com/yahoo/vespa/model/ClusterInfoTest.java index 4df9f261dfe..7aa6eb76995 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/ClusterInfoTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/ClusterInfoTest.java @@ -258,7 +258,7 @@ public class ClusterInfoTest { .provisioned(provisioner.provisioned()) .build(); new VespaModel(new NullConfigModelRegistry(), deployState); - return deployState.provisioned().all(); + return deployState.provisioned().capacities(); } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/admin/otel/OpenTelemetryConfigGeneratorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/admin/otel/OpenTelemetryConfigGeneratorTest.java index 7c4968aac84..efe9cfe5060 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/admin/otel/OpenTelemetryConfigGeneratorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/admin/otel/OpenTelemetryConfigGeneratorTest.java @@ -2,8 +2,24 @@ package com.yahoo.vespa.model.admin.otel; import com.yahoo.config.model.ApplicationConfigProducerRoot.StatePortInfo; +import com.yahoo.config.model.producer.TreeConfigProducer; +import com.yahoo.config.model.test.MockRoot; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ClusterMembership; +import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.HostSpec; +import com.yahoo.config.provision.NodeResources; +import com.yahoo.config.provision.RegionName; +import com.yahoo.config.provision.SystemName; +import com.yahoo.config.provision.Zone; +import com.yahoo.vespa.model.AbstractService; +import com.yahoo.vespa.model.Host; +import com.yahoo.vespa.model.HostResource; +import com.yahoo.vespa.model.AbstractService; +import com.yahoo.vespa.model.PortAllocBridge; import org.junit.jupiter.api.Test; import java.util.List; +import java.util.Optional; import static org.junit.jupiter.api.Assertions.*; /** @@ -13,10 +29,66 @@ public class OpenTelemetryConfigGeneratorTest { @Test void testBuildsYaml() { - var generator = new OpenTelemetryConfigGenerator(null); - generator.addStatePorts(List.of(new StatePortInfo("localhost", 19098, "config-sentinel", "sentinel"))); + var mockZone = new Zone(SystemName.PublicCd, Environment.prod, RegionName.from("mock")); + var app = ApplicationId.from("mytenant", "myapp", "myinstance"); + var generator = new OpenTelemetryConfigGenerator(mockZone, app, true); + var root = new MockRoot(); + + var mockHost = new Host(root, "localhost2.local"); + var mockVersion = new com.yahoo.component.Version(8); + var mockCluster = ClusterMembership.from("container/feeding/2/3", mockVersion, Optional.empty()); + var noResource = NodeResources.unspecified(); + var mockHostSpec = new HostSpec("localhost1.local", + noResource, noResource, noResource, + mockCluster, + Optional.empty(), Optional.empty(), Optional.empty()); + var mockHostResource = new HostResource(mockHost, mockHostSpec); + var mockSvc1 = new MockService(root, "sentinel"); + mockSvc1.setHostResource(mockHostResource); + var mockPort1 = new StatePortInfo("localhost", 19098, mockSvc1); + + var mockSvc2 = new MockService(root, "searchnode"); + mockSvc2.setProp("clustername", "mycluster"); + mockSvc2.setProp("clustertype", "mockup"); + var mockPort2 = new StatePortInfo("other123x.host.local", 19102, mockSvc2); + + generator.addStatePorts(List.of(mockPort1, mockPort2)); String yaml = generator.generate(); + // System.err.println(">>>\n" + yaml + "\n<<<"); assertTrue(yaml.contains("sentinel")); + String want = """ + "parentHostname":"other123.host.local"""; + assertTrue(yaml.contains(want)); } + static class MockService extends AbstractService { + private final String name; + public MockService(TreeConfigProducer<?> parent, String name) { + super(parent, name); + this.name = name; + } + public String getServiceName() { return name; } + public String getServiceType() { return "dummy"; } + @Override public int getPortCount() { return 0; } + @Override public void allocatePorts(int start, PortAllocBridge from) { } + } + + @Test + void testFindParentHost() { + String result; + result = OpenTelemetryConfigGenerator.findParentHost("n1234c.foo.bar.some.cloud"); + assertEquals("n1234.foo.bar.some.cloud", result); + result = OpenTelemetryConfigGenerator.findParentHost("n1234-v6-7.foo.bar.some.cloud"); + assertEquals("n1234.foo.bar.some.cloud", result); + result = OpenTelemetryConfigGenerator.findParentHost("2000a.foo.bar.some.cloud"); + assertEquals("2000.foo.bar.some.cloud", result); + result = OpenTelemetryConfigGenerator.findParentHost("2000-v6-10.foo.bar.some.cloud"); + assertEquals("2000.foo.bar.some.cloud", result); + result = OpenTelemetryConfigGenerator.findParentHost("foobar.some.cloud"); + assertNull(result); + result = OpenTelemetryConfigGenerator.findParentHost("foo123bar.some.cloud"); + assertNull(result); + result = OpenTelemetryConfigGenerator.findParentHost("foo123.some.cloud"); + assertNull(result); + } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ComplexFieldsValidatorTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ComplexFieldsValidatorTestCase.java index ae1db366c9f..2e51a425f6d 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ComplexFieldsValidatorTestCase.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ComplexFieldsValidatorTestCase.java @@ -143,6 +143,32 @@ public class ComplexFieldsValidatorTestCase { } @Test + void logs_warning_when_complex_fields_have_struct_fields_with_index_and_exact_match() throws IOException, SAXException { + var logger = new MyLogger(); + createModelAndValidate(joinLines( + "schema test {", + " document test {", + " field nesteds type array<nested> {", + " struct-field foo {", + " indexing: attribute | index", + " match {", + " exact", + " exact-terminator: '@@'", + " }", + " }", + " }", + " struct nested {", + " field foo type string {}", + " }", + " }", + "}"), logger); + assertTrue(logger.message.toString().contains("For cluster 'mycluster', schema 'test': " + + "The following complex fields have struct fields with 'indexing: index' which is " + + "not supported and has no effect: nesteds (nesteds.foo). " + + "Remove setting or change to 'indexing: attribute' if needed for matching.")); + } + + @Test void validation_passes_when_only_supported_struct_field_attributes_are_used() throws IOException, SAXException { createModelAndValidate(joinLines("search test {", " document test {", diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/JvmHeapSizeValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/JvmHeapSizeValidatorTest.java index 45125c8eb68..340968f89d1 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/JvmHeapSizeValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/JvmHeapSizeValidatorTest.java @@ -45,10 +45,10 @@ class JvmHeapSizeValidatorTest { @Test void fails_on_too_low_heap_size() throws IOException, SAXException { - var deployState = createDeployState(2.2, 1024L * 1024 * 1024); + var deployState = createDeployState(2.3, 1024L * 1024 * 1024); var model = new VespaModel(new NullConfigModelRegistry(), deployState); ValidationTester.expect(new JvmHeapSizeValidator(), model, deployState, - "Allocated memory to JVM in cluster 'container' is too low (0.50GB < 0.60GB). Estimated cost of ONNX models is 1.00GB."); + "Allocated memory to JVM in cluster 'container' is too low (0.51GB < 0.60GB). Estimated cost of ONNX models is 1.00GB."); } @Test diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/QuotaValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/QuotaValidatorTest.java index 89f81dfdaef..590433757c3 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/QuotaValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/QuotaValidatorTest.java @@ -54,6 +54,12 @@ public class QuotaValidatorTest { } @Test + void test_deploy_within_quota_budget_because_in_dev() { + var tester = new ValidationTester(13, false, new TestProperties().setHostedVespa(true).setQuota(quota).setZone(devZone)); + tester.deploy(null, getServices(10), Environment.dev, null, CONTAINER_CLUSTER); + } + + @Test void test_deploy_above_quota_budget_in_publiccd() { var tester = new ValidationTester(13, false, new TestProperties().setHostedVespa(true).setQuota(quota.withBudget(BigDecimal.ONE)).setZone(publicCdZone)); try { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ResourcesReductionValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ResourcesReductionValidatorTest.java index b7e612127c1..36839e72e10 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ResourcesReductionValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ResourcesReductionValidatorTest.java @@ -209,7 +209,7 @@ public class ResourcesReductionValidatorTest { String resourcesStr = resources == null ? "" : String.format(" <resources vcpu='%.0f' memory='%.0fG' disk='%.0fG'/>", - resources.vcpu(), resources.memoryGb(), resources.diskGb()); + resources.vcpu(), resources.memoryGiB(), resources.diskGb()); return "<services version='1.0'>" + " <container id='default' version='1.0'>" + " <nodes count='" + nodes + "'>" + @@ -223,7 +223,7 @@ public class ResourcesReductionValidatorTest { String resourcesStr = resources == null ? "" : String.format(" <resources vcpu='%.0f' memory='%.0fG' disk='%.0fG'/>", - resources.vcpu(), resources.memoryGb(), resources.diskGb()); + resources.vcpu(), resources.memoryGiB(), resources.diskGb()); return "<services version='1.0'>" + " <content id='default' version='1.0'>" + " <redundancy>1</redundancy>" + diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/RestartOnDeployForLocalLLMValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/RestartOnDeployForLocalLLMValidatorTest.java new file mode 100644 index 00000000000..13e91f60712 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/RestartOnDeployForLocalLLMValidatorTest.java @@ -0,0 +1,79 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.application.validation.change; + +import com.yahoo.config.model.api.ConfigChangeAction; +import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.config.model.deploy.TestProperties; +import com.yahoo.vespa.model.VespaModel; +import com.yahoo.vespa.model.application.validation.ValidationTester; +import com.yahoo.vespa.model.test.utils.VespaModelCreatorWithMockPkg; +import org.junit.jupiter.api.Test; + +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * @author lesters + */ +public class RestartOnDeployForLocalLLMValidatorTest { + + private static final String LOCAL_LLM_COMPONENT = RestartOnDeployForLocalLLMValidator.LOCAL_LLM_COMPONENT; + + @Test + void validate_no_restart_on_deploy() { + VespaModel current = createModel(); + VespaModel next = createModel(withComponent(LOCAL_LLM_COMPONENT)); + List<ConfigChangeAction> result = validateModel(current, next); + assertEquals(0, result.size()); + } + + @Test + void validate_restart_on_deploy() { + VespaModel current = createModel(withComponent(LOCAL_LLM_COMPONENT)); + VespaModel next = createModel(withComponent(LOCAL_LLM_COMPONENT)); + List<ConfigChangeAction> result = validateModel(current, next); + assertEquals(1, result.size()); + assertTrue(result.get(0).validationId().isEmpty()); + assertEquals("Need to restart services in cluster 'cluster1' due to use of local LLM", result.get(0).getMessage()); + } + + private static List<ConfigChangeAction> validateModel(VespaModel current, VespaModel next) { + return ValidationTester.validateChanges(new RestartOnDeployForLocalLLMValidator(), + next, + deployStateBuilder().previousModel(current).build()); + } + + private static VespaModel createModel(String component) { + var xml = """ + <services version='1.0'> + <container id='cluster1' version='1.0'> + <http> + <server id='server1' port='8080'/> + </http> + %s + </container> + </services> + """.formatted(component); + DeployState.Builder builder = deployStateBuilder(); + return new VespaModelCreatorWithMockPkg(null, xml).create(builder); + } + + private static VespaModel createModel() { + return createModel(""); + } + + private static String withComponent(String componentClass) { + return "<component id='llm' class='%s' />".formatted(componentClass); + } + + private static DeployState.Builder deployStateBuilder() { + return new DeployState.Builder().properties(new TestProperties()); + } + + private static void assertStartsWith(String expected, List<ConfigChangeAction> result) { + assertTrue(result.get(0).getMessage().startsWith(expected)); + } + +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/RestartOnDeployForOnnxModelChangesValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/RestartOnDeployForOnnxModelChangesValidatorTest.java index 4c0786ea879..484a938476d 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/RestartOnDeployForOnnxModelChangesValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/RestartOnDeployForOnnxModelChangesValidatorTest.java @@ -33,7 +33,7 @@ public class RestartOnDeployForOnnxModelChangesValidatorTest { // Must be so large that changing model set or options requires restart (due to using more memory than available), // but not so large that deployment will not work at all with one model - private static final long defaultCost = 723456789; + private static final long defaultCost = 635241309; private static final long defaultHash = 0; @@ -47,8 +47,8 @@ public class RestartOnDeployForOnnxModelChangesValidatorTest { @Test void validate_changed_estimated_cost() { - VespaModel current = createModel(onnxModelCost(70000000, defaultHash)); - VespaModel next = createModel(onnxModelCost(723456789, defaultHash)); + VespaModel current = createModel(onnxModelCost(defaultCost, defaultHash)); + VespaModel next = createModel(onnxModelCost(19 * defaultCost / 20, defaultHash)); List<ConfigChangeAction> result = validateModel(current, next); assertEquals(1, result.size()); assertTrue(result.get(0).validationId().isEmpty()); @@ -58,8 +58,8 @@ public class RestartOnDeployForOnnxModelChangesValidatorTest { @Test void validate_changed_estimated_cost_non_hosted() { boolean hosted = false; - VespaModel current = createModel(onnxModelCost(70000000, defaultHash), hosted); - VespaModel next = createModel(onnxModelCost(723456789, defaultHash), hosted); + VespaModel current = createModel(onnxModelCost(defaultCost, defaultHash), hosted); + VespaModel next = createModel(onnxModelCost(19 * defaultCost / 20, defaultHash), hosted); List<ConfigChangeAction> result = validateModel(current, next, hosted); assertEquals(0, result.size()); } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/ContentClusterFixture.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/ContentClusterFixture.java index 8778f0c26c0..0677cabafb0 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/ContentClusterFixture.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/ContentClusterFixture.java @@ -29,11 +29,18 @@ public abstract class ContentClusterFixture { nextCluster = createCluster(nextSd); } + protected ContentClusterFixture(ContentCluster currentCluster, ContentCluster nextCluster) { + this.currentCluster = currentCluster; + this.nextCluster = nextCluster; + } + public ContentClusterFixture(String entireSd) throws Exception { - currentCluster = new ContentClusterBuilder().build( - ContentClusterUtils.createMockRoot(List.of(entireSd))); - nextCluster = new ContentClusterBuilder().build( - ContentClusterUtils.createMockRoot(List.of(entireSd))); + currentCluster = createClusterFromEntireSd(entireSd); + nextCluster = createClusterFromEntireSd(entireSd); + } + + protected static ContentCluster createClusterFromEntireSd(String sdContent) throws Exception { + return new ContentClusterBuilder().build(ContentClusterUtils.createMockRoot(List.of(sdContent))); } private static ContentCluster createCluster(String sdContent) throws Exception { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/IndexingScriptChangeValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/IndexingScriptChangeValidatorTest.java index cd54a20523f..247f01068fa 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/IndexingScriptChangeValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/IndexingScriptChangeValidatorTest.java @@ -39,6 +39,21 @@ public class IndexingScriptChangeValidatorTest { } } + private static class ComplexFixture extends ContentClusterFixture { + IndexingScriptChangeValidator validator; + public ComplexFixture(String currentSd, String nextSd) throws Exception { + super(createClusterFromEntireSd(currentSd), createClusterFromEntireSd(nextSd)); + validator = new IndexingScriptChangeValidator(ClusterSpec.Id.from("test"), + currentDb().getDerivedConfiguration().getSchema(), + nextDb().getDerivedConfiguration().getSchema()); + } + + @Override + public List<VespaConfigChangeAction> validate() { + return validator.validate(); + } + } + private static class ScriptFixture { private final ScriptExpression currentScript; @@ -56,6 +71,9 @@ public class IndexingScriptChangeValidatorTest { private static final String FIELD = "field f1 type string"; private static final String FIELD_F2 = "field f2 type string"; + private static final String TENSOR_FIELD_F1 = "field f1 type tensor(x[2])"; + private static final String TENSOR_FIELD_F2 = "field f2 type tensor(x[2])"; + private static final String TENSOR_FIELD_F3 = "field f3 type tensor(x[2])"; private static VespaConfigChangeAction expectedReindexingAction(String changedMsg, String fromScript, String toScript) { return expectedReindexingAction("f1", changedMsg, fromScript, toScript); @@ -115,6 +133,28 @@ public class IndexingScriptChangeValidatorTest { } @Test + void requireThatAddingIndexAspectForExtraTensorFieldWithChangedInputRequireReindexing() throws Exception { + new ComplexFixture(joinLines("schema test {", + " document test {", + " " + TENSOR_FIELD_F1 + " { }", + " " + TENSOR_FIELD_F2 + " { }", + " }", + " " + TENSOR_FIELD_F3 + " { indexing: input f1 | attribute }", + "}"), + joinLines("schema test {", + " document test {", + " " + TENSOR_FIELD_F1 + " { }", + " " + TENSOR_FIELD_F2 + " { }", + " }", + " " + TENSOR_FIELD_F3 + " { indexing: input f2 | index | attribute }", + "}")). + assertValidation(List.of(expectedReindexingAction("f3", "add index aspect", + "{ input f1 | attribute f3; }", + "{ input f2 | index f3 | attribute f3; }"))); + } + + + @Test void requireThatSettingDynamicSummaryIsOk() throws Exception { new Fixture(FIELD + " { indexing: summary }", FIELD + " { indexing: summary \n summary: dynamic }"). diff --git a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java index 924419daeae..3144748e7cc 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java @@ -157,7 +157,7 @@ public class ContentBuilderTest extends DomBuilderTest { assertEquals("clu/storage/0", c.getRootGroup().getNodes().get(0).getConfigId()); // Due to reuse. assertEquals(1, c.getRoot().hostSystem().getHosts().size()); HostResource h = c.getRoot().hostSystem().getHost("mockhost"); - String [] expectedServices = {"configserver", "logserver", "logd", "container-clustercontroller", "metricsproxy-container", "slobrok", "configproxy", "config-sentinel", "container", "storagenode", "searchnode", "distributor", "transactionlogserver"}; + String [] expectedServices = {"configserver", "logserver", "logd", "container-clustercontroller", "metricsproxy-container", "slobrok", "configproxy", "config-sentinel", "container", "storagenode", "searchnode", "distributor"}; assertServices(h, expectedServices); assertEquals("clu/storage/0", h.getService("storagenode").getConfigId()); assertEquals("clu/search/cluster.clu/0", h.getService("searchnode").getConfigId()); @@ -205,7 +205,7 @@ public class ContentBuilderTest extends DomBuilderTest { HostResource h = cluster.getRoot().hostSystem().getHost("mockhost"); String [] expectedServices = { "logd", "configproxy", "config-sentinel", "configserver", "container", "logserver", - "slobrok", "storagenode", "distributor", "searchnode", "transactionlogserver", + "slobrok", "storagenode", "distributor", "searchnode", CLUSTERCONTROLLER_CONTAINER.serviceName, METRICS_PROXY_CONTAINER.serviceName }; assertServices(h, expectedServices); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/NodesSpecificationTest.java b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/NodesSpecificationTest.java index 344471cada0..e0193d10e23 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/NodesSpecificationTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/NodesSpecificationTest.java @@ -42,8 +42,8 @@ public class NodesSpecificationTest { assertEquals(2, spec.minResources().nodeResources().vcpu(), 1e-9); assertEquals(2, spec.maxResources().nodeResources().vcpu(), 1e-9); - assertEquals(3e15, spec.minResources().nodeResources().memoryGb(), 1e-9); - assertEquals(3e15, spec.maxResources().nodeResources().memoryGb(), 1e-9); + assertEquals(3e15, spec.minResources().nodeResources().memoryGiB(), 1e-9); + assertEquals(3e15, spec.maxResources().nodeResources().memoryGiB(), 1e-9); assertEquals(4e3, spec.minResources().nodeResources().diskGb(), 1e-9); assertEquals(4e3, spec.maxResources().nodeResources().diskGb(), 1e-9); @@ -54,8 +54,8 @@ public class NodesSpecificationTest { assertEquals(1 << 30, spec.minResources().nodeResources().gpuResources().count()); assertEquals(1 << 30, spec.maxResources().nodeResources().gpuResources().count()); - assertEquals(3e-9, spec.minResources().nodeResources().gpuResources().memoryGb(), 1e-12); - assertEquals(3e-9, spec.maxResources().nodeResources().gpuResources().memoryGb(), 1e-12); + assertEquals(3e-9, spec.minResources().nodeResources().gpuResources().memoryGiB(), 1e-12); + assertEquals(3e-9, spec.maxResources().nodeResources().gpuResources().memoryGiB(), 1e-12); assertEquals(DiskSpeed.fast, spec.minResources().nodeResources().diskSpeed()); assertEquals(DiskSpeed.fast, spec.maxResources().nodeResources().diskSpeed()); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java index 2ea4249883d..93362e144a6 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java @@ -49,6 +49,7 @@ import static com.yahoo.config.model.api.ApplicationClusterEndpoint.Scope.zone; import static com.yahoo.config.provision.SystemName.main; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; /** @@ -221,24 +222,6 @@ public class ContainerClusterTest { } @Test - void requireThatJvmOmitStackTraceInFastThrowOptionWorks() { - // Empty option if option not set in property - MockRoot root = createRoot(new DeployState.Builder().build()); - ApplicationContainerCluster cluster = newClusterWithSearch(root); - addContainer(root, cluster, "c1", "host-c1"); - ApplicationContainer container = cluster.getContainers().get(0); - assertEquals("", container.getJvmOptions()); - - String jvmOption = "-XX:-foo"; - DeployState deployState = new DeployState.Builder().properties(new TestProperties().setJvmOmitStackTraceInFastThrowOption(jvmOption)).build(); - root = createRoot(deployState); - cluster = newClusterWithSearch(root); - addContainer(root, cluster, "c1", "host-c1"); - container = cluster.getContainers().get(0); - assertEquals(jvmOption, container.getJvmOptions()); - } - - @Test void requireThatWeCanHandleNullJvmOptions() { MockRoot root = createRoot(false); ApplicationContainerCluster cluster = newClusterWithSearch(root); @@ -516,4 +499,28 @@ public class ContainerClusterTest { (extra.isEmpty() ? "" : " " + extra); } + @Test + void testTruncationTo4Bits() { + assertEquals(0, ApplicationContainerCluster.truncateTo4SignificantBits(0)); + assertEquals(1, ApplicationContainerCluster.truncateTo4SignificantBits(1)); + assertEquals(15, ApplicationContainerCluster.truncateTo4SignificantBits(15)); + assertEquals(16, ApplicationContainerCluster.truncateTo4SignificantBits(16)); + assertEquals(16, ApplicationContainerCluster.truncateTo4SignificantBits(17)); + assertEquals(18, ApplicationContainerCluster.truncateTo4SignificantBits(18)); + assertEquals(18, ApplicationContainerCluster.truncateTo4SignificantBits(19)); + assertEquals(30, ApplicationContainerCluster.truncateTo4SignificantBits(30)); + assertEquals(30, ApplicationContainerCluster.truncateTo4SignificantBits(31)); + assertEquals(32, ApplicationContainerCluster.truncateTo4SignificantBits(32)); + assertEquals(32, ApplicationContainerCluster.truncateTo4SignificantBits(33)); + assertEquals(32, ApplicationContainerCluster.truncateTo4SignificantBits(34)); + assertEquals(32, ApplicationContainerCluster.truncateTo4SignificantBits(35)); + assertEquals(36, ApplicationContainerCluster.truncateTo4SignificantBits(36)); + assertEquals(0x78000000, ApplicationContainerCluster.truncateTo4SignificantBits(0x78000000)); + assertEquals(0x78000000, ApplicationContainerCluster.truncateTo4SignificantBits(0x7fffffff)); + assertEquals(0x80000000, ApplicationContainerCluster.truncateTo4SignificantBits(0x80000000)); + assertEquals(0b10001000000000000000000000000000, ApplicationContainerCluster.truncateTo4SignificantBits(0b10000000000000000000000000000001)); + assertEquals(0b11000100000000000000000000000000, ApplicationContainerCluster.truncateTo4SignificantBits(0b11000000000000000000000000000001)); + assertEquals(0b11111111111111111111111111100010, ApplicationContainerCluster.truncateTo4SignificantBits(0b11111111111111111111111111100001)); + } + } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/search/searchchain/SchemaChainsTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/search/searchchain/SchemaChainsTest.java index ea43f5c8124..b782366655f 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/search/searchchain/SchemaChainsTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/search/searchchain/SchemaChainsTest.java @@ -142,7 +142,7 @@ public class SchemaChainsTest extends SchemaChainsTestBase { assertTrue(chain.phases().isEmpty()); assertEquals(1, chain.inherits().size()); assertEquals("native", chain.inherits(0)); - assertEquals(10, chain.components().size()); + assertEquals(11, chain.components().size()); assertEquals("com.yahoo.prelude.querytransform.PhrasingSearcher@vespa", chain.components(0)); assertEquals("com.yahoo.prelude.searcher.FieldCollapsingSearcher@vespa", chain.components(1)); assertEquals("com.yahoo.search.yql.MinimalQueryInserter@vespa", chain.components(2)); @@ -153,13 +153,14 @@ public class SchemaChainsTest extends SchemaChainsTestBase { assertEquals("com.yahoo.prelude.semantics.SemanticSearcher@vespa", chain.components(7)); assertEquals("com.yahoo.search.grouping.GroupingQueryParser@vespa", chain.components(8)); assertEquals("com.yahoo.search.querytransform.WeakAndReplacementSearcher@vespa", chain.components(9)); + assertEquals("com.yahoo.search.searchers.OpportunisticWeakAndSearcher@vespa", chain.components(10)); assertTrue(chain.excludes().isEmpty()); assertEquals(ChainsConfig.Chains.Type.SEARCH, chain.type()); } @Test public void require_all_default_chains_are_correct() { - assertEquals(63, chainsConfig.components().size()); + assertEquals(64, chainsConfig.components().size()); assertEquals(10, chainsConfig.chains().size()); validateVespaPhasesChain(findChain("vespaPhases")); validateNativeChain(findChain("native")); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java index e704da08d18..60ea37cad3f 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java @@ -17,6 +17,7 @@ import com.yahoo.config.model.provision.InMemoryProvisioner; import com.yahoo.config.model.provision.SingleNodeProvisioner; import com.yahoo.config.model.test.MockApplicationPackage; import com.yahoo.config.model.test.MockRoot; +import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.CloudAccount; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.Flavor; @@ -49,6 +50,7 @@ import com.yahoo.vespa.model.container.component.Handler; import com.yahoo.vespa.model.content.utils.ContentClusterUtils; import com.yahoo.vespa.model.test.VespaModelTester; import com.yahoo.vespa.model.test.utils.VespaModelCreatorWithFilePkg; +import com.yahoo.yolean.Exceptions; import org.junit.jupiter.api.Test; import org.w3c.dom.Element; import org.xml.sax.SAXException; @@ -598,7 +600,7 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase { .setCloudAccount(cloudAccount)) .build()); assertEquals(2, model.hostSystem().getHosts().size()); - assertEquals(List.of(cloudAccount), model.provisioned().all().values() + assertEquals(List.of(cloudAccount), model.provisioned().capacities().values() .stream() .map(capacity -> capacity.cloudAccount().get()) .toList()); @@ -741,6 +743,29 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase { } @Test + void testerContainer() { + createModelWithTesterNodes("<nodes count='1' docker-image='foo/bar/baz'><resources vcpu='0.1' memory='1Gb' disk='1Gb'/></nodes>"); + + assertEquals("In container cluster 'default': tester cannot run on more than 1 node, but 2 nodes were specified", + Exceptions.toMessageString(assertThrows(IllegalArgumentException.class, + () -> createModelWithTesterNodes("<nodes count='2'/>")))); + + assertEquals("In container cluster 'default': tester resources must be absolute, but min and max resources differ: specification of dedicated " + + "min 1 nodes with [vcpu: 0.0, memory: 1.0 Gb, disk: 0.0 Gb, bandwidth: 0.3 Gbps, architecture: any] " + + "max 1 nodes with [vcpu: 0.0, memory: 2.0 Gb, disk: 0.0 Gb, bandwidth: 0.3 Gbps, architecture: any]", + Exceptions.toMessageString(assertThrows(IllegalArgumentException.class, + () -> createModelWithTesterNodes("<nodes><resources memory='[1Gb, 2Gb]'/></nodes>")))); + } + + void createModelWithTesterNodes(String testerNodesXml) { + String containerXml = "<container id='default' version='1.0'>%s</container>".formatted(testerNodesXml); + VespaModelTester tester = new VespaModelTester(); + tester.setApplicationId("t", "a", "i-t"); + tester.addHosts(3); + tester.createModel(containerXml, true); + } + + @Test void cluster_with_zookeeper() { Function<Integer, String> servicesXml = (nodeCount) -> "<container version='1.0' id='default'>" + "<nodes count='" + nodeCount + "'/>" + diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java index 786caa4b317..a890f35caa8 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java @@ -1,6 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.content; +import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.model.api.ApplicationClusterEndpoint; import com.yahoo.config.model.api.ContainerEndpoint; import com.yahoo.config.model.api.ModelContext; @@ -48,6 +49,8 @@ import java.util.Map; import java.util.Optional; import java.util.OptionalInt; import java.util.Set; +import java.util.function.Consumer; +import java.util.logging.Level; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; @@ -941,8 +944,19 @@ public class ContentClusterTest extends ContentBaseTest { } private static ContentCluster createOneNodeCluster(String clusterXml, TestProperties props, Optional<Flavor> flavor) throws Exception { + return createOneNodeCluster(clusterXml, props, flavor, new StringBuffer()); + } + + private static ContentCluster createOneNodeCluster(String clusterXml, TestProperties props, + Optional<Flavor> flavor, StringBuffer deployWarningsBuffer) throws Exception { + DeployLogger logger = (level, message) -> { + if (level == Level.WARNING) { // only care about warnings + deployWarningsBuffer.append("%s\n".formatted(message)); + } + }; DeployState.Builder deployStateBuilder = new DeployState.Builder() - .properties(props); + .properties(props) + .deployLogger(logger); MockRoot root = flavor.isPresent() ? ContentClusterUtils.createMockRoot(new SingleNodeProvisioner(flavor.get()), List.of(), deployStateBuilder) : @@ -1264,7 +1278,7 @@ public class ContentClusterTest extends ContentBaseTest { void verify_max_tls_size() throws Exception { var flavor = new Flavor(new FlavorsConfig.Flavor(new FlavorsConfig.Flavor.Builder().name("test").minDiskAvailableGb(100))); assertEquals(21474836480L, resolveMaxTLSSize(Optional.empty())); - assertEquals(2147483648L, resolveMaxTLSSize(Optional.of(flavor))); + assertEquals(2_000_000_000, resolveMaxTLSSize(Optional.of(flavor))); } void assertZookeeperServerImplementation(String expectedClassName, @@ -1471,13 +1485,18 @@ public class ContentClusterTest extends ContentBaseTest { assertEquals(expectedGroupsAllowedDown, config.max_number_of_groups_allowed_to_be_down()); } - private boolean resolveDistributorOperationCancellationConfig(Integer featureLevel) throws Exception { + private StorDistributormanagerConfig resolveDistributorConfig(Consumer<TestProperties> propertyMutator) throws Exception { var properties = new TestProperties(); - if (featureLevel != null) { - properties.setContentLayerMetadataFeatureLevel(featureLevel); - } - var cfg = resolveStorDistributormanagerConfig(properties); - return cfg.enable_operation_cancellation(); + propertyMutator.accept(properties); + return resolveStorDistributormanagerConfig(properties); + } + + private boolean resolveDistributorOperationCancellationConfig(Integer featureLevel) throws Exception { + return resolveDistributorConfig((props) -> { + if (featureLevel != null) { + props.setContentLayerMetadataFeatureLevel(featureLevel); + } + }).enable_operation_cancellation(); } @Test @@ -1488,6 +1507,21 @@ public class ContentClusterTest extends ContentBaseTest { assertTrue(resolveDistributorOperationCancellationConfig(2)); } + private boolean resolveDistributorSymmetricReplicaSelectionConfig(Boolean flagValue) throws Exception { + return resolveDistributorConfig((props) -> { + if (flagValue != null) { + props.setSymmetricPutAndActivateReplicaSelection(flagValue); + } + }).symmetric_put_and_activate_replica_selection(); + } + + @Test + void distributor_symmetric_replica_selection_config_controlled_by_properties() throws Exception { + assertFalse(resolveDistributorSymmetricReplicaSelectionConfig(null)); // defaults to false + assertFalse(resolveDistributorSymmetricReplicaSelectionConfig(false)); + assertTrue(resolveDistributorSymmetricReplicaSelectionConfig(true)); + } + @Test void node_distribution_key_outside_legal_range_is_disallowed() { // Only [0, UINT16_MAX - 1] is a valid range. UINT16_MAX is a special content layer-internal @@ -1506,6 +1540,68 @@ public class ContentClusterTest extends ContentBaseTest { } } + private String createClusterAndGetDeploymentWarnings(String xml) { + var warningBuf = new StringBuffer(); + try { + createOneNodeCluster(xml, new TestProperties(), Optional.empty(), warningBuf); + } catch (Exception e) { + throw new RuntimeException(e); + } + return warningBuf.toString(); + }; + + private static String clusterXmlWithNodeDistributionKey(int key) { + return "<content version=\"1.0\" id=\"mockcluster\">" + + " <redundancy>1</redundancy>" + + " <documents/>" + + " <group>" + + " <node distribution-key=\"%d\" hostalias=\"mockhost\"/>".formatted(key) + + " </group>" + + "</content>"; + } + + @Test + void distribution_key_much_higher_than_node_count_logs_deployment_warning() { + // "much higher" is somewhat arbitrary, but needs to be kept in track with threshold in `ContentCluster`. + String warnings = createClusterAndGetDeploymentWarnings(clusterXmlWithNodeDistributionKey(101)); + assertEquals(warnings, "Content cluster 'mockcluster' has 1 node(s), but the highest distribution " + + "key is 101. Having much higher distribution keys than the number of nodes " + + "is not recommended, as it may negatively affect performance. " + + "See https://docs.vespa.ai/en/reference/services-content.html#node\n"); + } + + @Test + void distribution_key_not_much_higher_than_node_count_does_not_log_deployment_warning() { + String warnings = createClusterAndGetDeploymentWarnings(clusterXmlWithNodeDistributionKey(100)); + assertEquals(warnings, ""); + } + + private void checkStrictlyIncreasingClusterStateVersionConfig(Boolean flagValue, boolean expected) throws Exception { + var props = new TestProperties(); + if (flagValue != null) { + props.setEnforceStrictlyIncreasingClusterStateVersions(flagValue); + } + var cc = createOneNodeCluster(props); + + // stor-server config should be the same for both distributors and storage nodes + var builder = new StorServerConfig.Builder(); + cc.getStorageCluster().getConfig(builder); + var cfg = builder.build(); + assertEquals(expected, cfg.require_strictly_increasing_cluster_state_versions()); + + builder = new StorServerConfig.Builder(); + cc.getDistributorNodes().getConfig(builder); + cfg = builder.build(); + assertEquals(expected, cfg.require_strictly_increasing_cluster_state_versions()); + } + + @Test + void strictly_increasing_cluster_state_versions_config_controlled_by_feature_flag() throws Exception { + checkStrictlyIncreasingClusterStateVersionConfig(null, false); // TODO change default + checkStrictlyIncreasingClusterStateVersionConfig(false, false); + checkStrictlyIncreasingClusterStateVersionConfig(true, true); + } + private String servicesWithGroups(int groupCount, double minGroupUpRatio) { String services = String.format("<?xml version='1.0' encoding='UTF-8' ?>" + "<services version='1.0'>" + diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/IndexedTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/IndexedTest.java index e9e96d8b0cf..920b2f7a868 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/IndexedTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/IndexedTest.java @@ -131,7 +131,7 @@ public class IndexedTest extends ContentBaseTest { // String [] expectedServices = {"logserver", "configserver", "adminserver", "slobrok", // "logd", "configproxy","config-sentinel", // "container", "fleetcontroller", - // "storagenode", "searchnode", "distributor", "transactionlogserver"}; + // "storagenode", "searchnode", "distributor"}; // DomContentBuilderTest.assertServices(h, expectedServices); Routing routing = model.getRouting(); assertNotNull(routing); @@ -159,8 +159,7 @@ public class IndexedTest extends ContentBaseTest { // HostResource h = model.getHostSystem().getHosts().get(0); // String [] expectedServices = {"logserver", "configserver", "adminserver", "slobrok", // "logd", "configproxy","config-sentinel", - // "container", "storagenode", "searchnode", "distributor", - // "transactionlogserver"}; + // "container", "storagenode", "searchnode", "distributor"}; // DomContentBuilderTest.assertServices(h, expectedServices); ContentCluster s = model.getContentClusters().get("test"); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/ml/ImportedModelTester.java b/config-model/src/test/java/com/yahoo/vespa/model/ml/ImportedModelTester.java index 4fd61f59ed7..6114cc7d8bf 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/ml/ImportedModelTester.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/ml/ImportedModelTester.java @@ -8,9 +8,14 @@ import ai.vespa.rankingexpression.importer.tensorflow.TensorFlowImporter; import ai.vespa.rankingexpression.importer.vespa.VespaImporter; import ai.vespa.rankingexpression.importer.xgboost.XGBoostImporter; import com.yahoo.config.FileReference; +import com.yahoo.config.application.api.ApplicationFile; import com.yahoo.config.model.ApplicationPackageTester; import com.yahoo.config.model.api.ApplicationClusterEndpoint; import com.yahoo.config.model.api.ContainerEndpoint; +import com.yahoo.config.model.api.OnnxModelCost; +import com.yahoo.config.model.api.OnnxModelCost.Calculator; +import com.yahoo.config.model.api.OnnxModelCost.ModelInfo; +import com.yahoo.config.model.api.OnnxModelOptions; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.io.GrowableByteBuffer; import com.yahoo.io.IOUtils; @@ -22,7 +27,10 @@ import org.xml.sax.SAXException; import java.io.IOException; import java.io.UncheckedIOException; +import java.net.URI; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.Set; @@ -47,6 +55,7 @@ public class ImportedModelTester { private final String modelName; private final Path applicationDir; private final DeployState deployState; + public final Calculator calculator = new MockCalculator(); public ImportedModelTester(String modelName, Path applicationDir) { this(modelName, applicationDir, new DeployState.Builder()); @@ -58,6 +67,7 @@ public class ImportedModelTester { deployState = deployStateBuilder.applicationPackage(ApplicationPackageTester.create(applicationDir.toString()).app()) .endpoints(Set.of(new ContainerEndpoint("container", ApplicationClusterEndpoint.Scope.zone, List.of("default.example.com")))) .modelImporters(importers) + .onnxModelCost((pkg, app, cluster) -> calculator) .build(); } @@ -98,4 +108,19 @@ public class ImportedModelTester { } } + public static class MockCalculator implements OnnxModelCost.Calculator { + private final Map<String, ModelInfo> models = new HashMap<>(); + @Override public long aggregatedModelCostInBytes() { return models.size(); } + @Override public void registerModel(ApplicationFile path, OnnxModelOptions onnxModelOptions) { + models.put(path.toString(), new ModelInfo(path.toString(), 1, 1, onnxModelOptions)); + } + @Override public void registerModel(URI uri, OnnxModelOptions onnxModelOptions) { + models.put(uri.toString(), new ModelInfo(uri.toString(), 1, 1, onnxModelOptions)); + } + @Override public Map<String, ModelInfo> models() { return Map.copyOf(models); } + @Override public void setRestartOnDeploy() { } + @Override public boolean restartOnDeploy() { return false; } + @Override public void store() { } + } + } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/ml/ModelEvaluationTest.java b/config-model/src/test/java/com/yahoo/vespa/model/ml/ModelEvaluationTest.java index cc33c8561fc..bced4c546c6 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/ml/ModelEvaluationTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/ml/ModelEvaluationTest.java @@ -55,6 +55,7 @@ public class ModelEvaluationTest { RankProfilesConfig config = new RankProfilesConfig(b); assertEquals(0, config.rankprofile().size()); + assertEquals(0, tester.calculator.aggregatedModelCostInBytes()); } finally { IOUtils.recursiveDeleteDir(appDir.append(ApplicationPackage.MODELS_GENERATED_DIR).toFile()); @@ -69,6 +70,7 @@ public class ModelEvaluationTest { try { ImportedModelTester tester = new ImportedModelTester("ml_serving", appDir); assertHasMlModels(tester.createVespaModel(), appDir); + assertEquals(3, tester.calculator.aggregatedModelCostInBytes()); // At this point the expression is stored - copy application to another location which do not have a models dir storedAppDir.toFile().mkdirs(); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/search/NodeResourcesTuningTest.java b/config-model/src/test/java/com/yahoo/vespa/model/search/NodeResourcesTuningTest.java index fbc4f6768a2..998437b64f5 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/search/NodeResourcesTuningTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/search/NodeResourcesTuningTest.java @@ -9,7 +9,8 @@ import org.junit.jupiter.api.Test; import static com.yahoo.vespa.model.Host.memoryOverheadGb; import static org.junit.jupiter.api.Assertions.assertEquals; -import static com.yahoo.vespa.model.search.NodeResourcesTuning.MB; +import static com.yahoo.vespa.model.search.NodeResourcesTuning.MiB; +import static com.yahoo.vespa.model.search.NodeResourcesTuning.GiB; import static com.yahoo.vespa.model.search.NodeResourcesTuning.GB; /** @@ -28,7 +29,7 @@ public class NodeResourcesTuningTest { @Test void require_that_hwinfo_memory_size_is_set() { - assertEquals(24 * GB, configFromMemorySetting(24 + memoryOverheadGb, 0).hwinfo().memory().size()); + assertEquals(24 * GiB, configFromMemorySetting(24 + memoryOverheadGb, 0).hwinfo().memory().size()); assertEquals(1.9585050869E10, configFromMemorySetting(24 + memoryOverheadGb, ApplicationContainerCluster.heapSizePercentageOfTotalAvailableMemoryWhenCombinedCluster * 0.01).hwinfo().memory().size(), 1000); } @@ -84,26 +85,26 @@ public class NodeResourcesTuningTest { @Test void require_that_document_store_maxfilesize_is_set_based_on_available_memory() { - assertDocumentStoreMaxFileSize(256 * MB, 4); - assertDocumentStoreMaxFileSize(256 * MB, 6); - assertDocumentStoreMaxFileSize(256 * MB, 8); - assertDocumentStoreMaxFileSize(256 * MB, 12); - assertDocumentStoreMaxFileSize((long) (16 * GB * 0.02), 16); - assertDocumentStoreMaxFileSize((long) (24 * GB * 0.02), 24); - assertDocumentStoreMaxFileSize((long) (32 * GB * 0.02), 32); - assertDocumentStoreMaxFileSize((long) (48 * GB * 0.02), 48); - assertDocumentStoreMaxFileSize((long) (64 * GB * 0.02), 64); - assertDocumentStoreMaxFileSize((long) (128 * GB * 0.02), 128); - assertDocumentStoreMaxFileSize((long) (256 * GB * 0.02), 256); - assertDocumentStoreMaxFileSize((long) (512 * GB * 0.02), 512); + assertDocumentStoreMaxFileSize(256 * MiB, 4); + assertDocumentStoreMaxFileSize(256 * MiB, 6); + assertDocumentStoreMaxFileSize(256 * MiB, 8); + assertDocumentStoreMaxFileSize(256 * MiB, 12); + assertDocumentStoreMaxFileSize((long) (16 * GiB * 0.02), 16); + assertDocumentStoreMaxFileSize((long) (24 * GiB * 0.02), 24); + assertDocumentStoreMaxFileSize((long) (32 * GiB * 0.02), 32); + assertDocumentStoreMaxFileSize((long) (48 * GiB * 0.02), 48); + assertDocumentStoreMaxFileSize((long) (64 * GiB * 0.02), 64); + assertDocumentStoreMaxFileSize((long) (128 * GiB * 0.02), 128); + assertDocumentStoreMaxFileSize((long) (256 * GiB * 0.02), 256); + assertDocumentStoreMaxFileSize((long) (512 * GiB * 0.02), 512); } @Test void require_that_flush_strategy_memory_limits_are_set_based_on_available_memory() { - assertFlushStrategyMemory((long) (4 * GB * DEFAULT_MEMORY_GAIN), 4); - assertFlushStrategyMemory((long) (8 * GB * DEFAULT_MEMORY_GAIN), 8); - assertFlushStrategyMemory((long) (24 * GB * DEFAULT_MEMORY_GAIN), 24); - assertFlushStrategyMemory((long) (64 * GB * DEFAULT_MEMORY_GAIN), 64); + assertFlushStrategyMemory((long) (4 * GiB * DEFAULT_MEMORY_GAIN), 4); + assertFlushStrategyMemory((long) (8 * GiB * DEFAULT_MEMORY_GAIN), 8); + assertFlushStrategyMemory((long) (24 * GiB * DEFAULT_MEMORY_GAIN), 24); + assertFlushStrategyMemory((long) (64 * GiB * DEFAULT_MEMORY_GAIN), 64); } @Test @@ -129,8 +130,8 @@ public class NodeResourcesTuningTest { @Test void require_that_summary_cache_max_bytes_is_set_based_on_memory() { - assertEquals(1 * GB / 25, configFromMemorySetting(1 + memoryOverheadGb, 0).summary().cache().maxbytes()); - assertEquals(256 * GB / 25, configFromMemorySetting(256 + memoryOverheadGb, 0).summary().cache().maxbytes()); + assertEquals(1 * GiB / 25, configFromMemorySetting(1 + memoryOverheadGb, 0).summary().cache().maxbytes()); + assertEquals(256 * GiB / 25, configFromMemorySetting(256 + memoryOverheadGb, 0).summary().cache().maxbytes()); } @Test diff --git a/config-model/src/test/java/com/yahoo/vespa/model/search/test/SearchNodeTest.java b/config-model/src/test/java/com/yahoo/vespa/model/search/test/SearchNodeTest.java index bc981c3de7c..499df4e5669 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/search/test/SearchNodeTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/search/test/SearchNodeTest.java @@ -12,7 +12,6 @@ import com.yahoo.vespa.model.Host; import com.yahoo.vespa.model.HostResource; import com.yahoo.vespa.model.search.NodeSpec; import com.yahoo.vespa.model.search.SearchNode; -import com.yahoo.vespa.model.search.TransactionLogServer; import org.junit.jupiter.api.Test; import static org.junit.jupiter.api.Assertions.*; @@ -31,13 +30,8 @@ public class SearchNodeTest { assertEquals(expected, cfg.basedir()); } - private void prepare(MockRoot root, SearchNode node, Boolean useFsync) { + private void prepare(MockRoot root, SearchNode node) { Host host = new Host(root, "mockhost"); - TransactionLogServer tls = new TransactionLogServer(root, "mycluster", useFsync); - tls.setHostResource(new HostResource(host)); - tls.setBasePort(100); - tls.initService(root.getDeployState()); - node.setTls(tls); node.setHostResource(new HostResource(host)); node.setBasePort(200); node.initService(root.getDeployState()); @@ -45,13 +39,15 @@ public class SearchNodeTest { } private static SearchNode createSearchNode(MockRoot root, String name, int distributionKey, NodeSpec nodeSpec, - boolean flushOnShutDown, boolean isHosted, ModelContext.FeatureFlags featureFlags) { + boolean flushOnShutDown, boolean isHosted, + ModelContext.FeatureFlags featureFlags, + Boolean syncTransactionLog) { return SearchNode.create(root, name, distributionKey, nodeSpec, "mycluster", null, flushOnShutDown, - null, null, isHosted, 0.0, featureFlags); + null, null, isHosted, 0.0, featureFlags, syncTransactionLog); } - private static SearchNode createSearchNode(MockRoot root) { - return createSearchNode(root, "mynode", 3, new NodeSpec(7, 5), true, true, new TestProperties()); + private static SearchNode createSearchNode(MockRoot root, Boolean syncTransactionLog) { + return createSearchNode(root, "mynode", 3, new NodeSpec(7, 5), true, true, new TestProperties(), syncTransactionLog); } @Test @@ -64,15 +60,17 @@ public class SearchNodeTest { @Test void requireThatBasedirIsCorrectForElasticMode() { MockRoot root = new MockRoot(""); - SearchNode node = createSearchNode(root, "mynode", 3, new NodeSpec(7, 5), false, root.getDeployState().isHosted(), new TestProperties()); - prepare(root, node, true); + SearchNode node = createSearchNode(root, "mynode", 3, new NodeSpec(7, 5), false, + root.getDeployState().isHosted(), new TestProperties(), true); + prepare(root, node); assertBaseDir(Defaults.getDefaults().underVespaHome("var/db/vespa/search/cluster.mycluster/n3"), node); } @Test void requireThatPreShutdownCommandIsEmptyWhenNotActivated() { MockRoot root = new MockRoot(""); - SearchNode node = createSearchNode(root, "mynode", 3, new NodeSpec(7, 5), false, root.getDeployState().isHosted(), new TestProperties()); + SearchNode node = createSearchNode(root, "mynode", 3, new NodeSpec(7, 5), false, + root.getDeployState().isHosted(), new TestProperties(), true); node.setHostResource(new HostResource(new Host(node, "mynbode"))); node.initService(root.getDeployState()); assertFalse(node.getPreShutdownCommand().isPresent()); @@ -81,7 +79,8 @@ public class SearchNodeTest { @Test void requireThatPreShutdownCommandUsesPrepareRestartWhenActivated() { MockRoot root = new MockRoot(""); - SearchNode node = createSearchNode(root, "mynode2", 4, new NodeSpec(7, 5), true, root.getDeployState().isHosted(), new TestProperties()); + SearchNode node = createSearchNode(root, "mynode2", 4, new NodeSpec(7, 5), true, + root.getDeployState().isHosted(), new TestProperties(), true); node.setHostResource(new HostResource(new Host(node, "mynbode2"))); node.initService(root.getDeployState()); assertTrue(node.getPreShutdownCommand().isPresent()); @@ -90,7 +89,8 @@ public class SearchNodeTest { private void verifyCodePlacement(boolean hugePages) { MockRoot root = new MockRoot(""); - SearchNode node = createSearchNode(root, "mynode2", 4, new NodeSpec(7, 5), true, false, new TestProperties().loadCodeAsHugePages(hugePages)); + SearchNode node = createSearchNode(root, "mynode2", 4, new NodeSpec(7, 5), true, false, + new TestProperties().loadCodeAsHugePages(hugePages), true); node.setHostResource(new HostResource(new Host(node, "mynbode2"))); node.initService(root.getDeployState()); assertEquals(hugePages, node.getEnvVars().get("VESPA_LOAD_CODE_AS_HUGEPAGES") != null); @@ -104,7 +104,8 @@ public class SearchNodeTest { private void verifySharedStringRepoReclaim(boolean sharedStringRepoNoReclaim) { MockRoot root = new MockRoot(""); - SearchNode node = createSearchNode(root, "mynode2", 4, new NodeSpec(7, 5), true, false, new TestProperties().sharedStringRepoNoReclaim(sharedStringRepoNoReclaim)); + SearchNode node = createSearchNode(root, "mynode2", 4, new NodeSpec(7, 5), true, false, + new TestProperties().sharedStringRepoNoReclaim(sharedStringRepoNoReclaim), true); node.setHostResource(new HostResource(new Host(node, "mynbode2"))); node.initService(root.getDeployState()); assertEquals(sharedStringRepoNoReclaim, node.getEnvVars().get("VESPA_SHARED_STRING_REPO_NO_RECLAIM") != null); @@ -120,10 +121,10 @@ public class SearchNodeTest { return new MockRoot("", new DeployState.Builder().properties(properties).build()); } - private TranslogserverConfig getTlsConfig(ModelContext.Properties properties, Boolean useFsync) { + private TranslogserverConfig getTlsConfig(ModelContext.Properties properties, Boolean syncTransactionLog) { MockRoot root = createRoot(properties); - SearchNode node = createSearchNode(root); - prepare(root, node, useFsync); + SearchNode node = createSearchNode(root, syncTransactionLog); + prepare(root, node); TranslogserverConfig.Builder tlsBuilder = new TranslogserverConfig.Builder(); node.getConfig(tlsBuilder); return tlsBuilder.build(); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/significance/test/SignificanceModelTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/significance/test/SignificanceModelTestCase.java index 00e95a34287..26e8c67a226 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/significance/test/SignificanceModelTestCase.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/significance/test/SignificanceModelTestCase.java @@ -37,9 +37,6 @@ public class SignificanceModelTestCase { ApplicationContainerCluster containerCluster = vespaModel.getContainerClusters().get("container"); var significanceConfig = assertSignificancePresent(containerCluster); assertEquals(3, significanceConfig.model().size()); - assertEquals("en", significanceConfig.model().get(0).language()); - assertEquals("no", significanceConfig.model().get(1).language()); - assertEquals("ru", significanceConfig.model().get(2).language()); assertEquals("models/idf-norwegian-wiki.json.zst", modelReference(significanceConfig.model().get(1), "path").path().orElseThrow().value()); assertEquals("https://some/uri/blob.json", modelReference(significanceConfig.model().get(2), "path").url().orElseThrow().value()); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTester.java b/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTester.java index 7037480f853..887db9c9cc9 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTester.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTester.java @@ -88,7 +88,7 @@ public class VespaModelTester { // (for e.g cluster controllers and slobrok nodes) String hostname = String.format("%s-%02d", "node" + "-" + Math.round(resources.vcpu()) + - "-" + Math.round(resources.memoryGb()) + + "-" + Math.round(resources.memoryGiB()) + "-" + Math.round(resources.diskGb()), count - i); hosts.add(new Host(hostname, List.of(), flavor)); |