diff options
86 files changed, 1163 insertions, 1741 deletions
diff --git a/client/go/jvm/env.go b/client/go/jvm/env.go index dbc957e3ccb..d2f0b2e6616 100644 --- a/client/go/jvm/env.go +++ b/client/go/jvm/env.go @@ -19,7 +19,7 @@ func (opts *Options) exportEnvSettings(ps *prog.Spec) { vlt := fmt.Sprintf("file:%s/vespa.log", lvd) lcd := fmt.Sprintf("%s/var/db/vespa/logcontrol", vespaHome) lcf := fmt.Sprintf("%s/%s.logcontrol", lcd, c.ServiceName()) - dlp := fmt.Sprintf("%s/lib64", vespaHome) + dlp := fmt.Sprintf("%s/lib64:/opt/vespa-deps/lib64", vespaHome) opts.fixSpec.FixDir(lvd) opts.fixSpec.FixDir(lcd) ps.Setenv(envvars.VESPA_LOG_TARGET, vlt) diff --git a/client/go/prog/common_env.go b/client/go/prog/common_env.go index a970895bf84..150a4aaa38f 100644 --- a/client/go/prog/common_env.go +++ b/client/go/prog/common_env.go @@ -15,7 +15,7 @@ func (spec *Spec) configureCommonEnv() { os.Unsetenv(envvars.LD_PRELOAD) spec.Setenv(envvars.STD_THREAD_PREVENT_TRY_CATCH, "true") spec.Setenv(envvars.GLIBCXX_FORCE_NEW, "1") - spec.Setenv(envvars.LD_LIBRARY_PATH, vespa.FindHome()+"/lib64") + spec.Setenv(envvars.LD_LIBRARY_PATH, vespa.FindHome()+"/lib64:/opt/vespa-deps/lib64") spec.Setenv(envvars.MALLOC_ARENA_MAX, "1") // fallback from old env.vars: diff --git a/client/go/script-utils/startcbinary/common_env.go b/client/go/script-utils/startcbinary/common_env.go index 812f182a757..eadb0aafb1b 100644 --- a/client/go/script-utils/startcbinary/common_env.go +++ b/client/go/script-utils/startcbinary/common_env.go @@ -26,7 +26,7 @@ func (spec *ProgSpec) configureCommonEnv() { os.Unsetenv(envvars.LD_PRELOAD) spec.setenv(envvars.STD_THREAD_PREVENT_TRY_CATCH, "true") spec.setenv(envvars.GLIBCXX_FORCE_NEW, "1") - spec.setenv(envvars.LD_LIBRARY_PATH, vespa.FindHome()+"/lib64") + spec.setenv(envvars.LD_LIBRARY_PATH, vespa.FindHome()+"/lib64:/opt/vespa-deps/lib64") spec.setenv(envvars.MALLOC_ARENA_MAX, "1") // fallback from old env.vars: diff --git a/config-application-package/pom.xml b/config-application-package/pom.xml index 41e65f19b79..dc2c8d91db1 100644 --- a/config-application-package/pom.xml +++ b/config-application-package/pom.xml @@ -109,12 +109,6 @@ <artifactId>junit</artifactId> <scope>test</scope> </dependency> - <dependency> - <groupId>xmlunit</groupId> - <artifactId>xmlunit</artifactId> - <version>1.5</version> - <scope>test</scope> - </dependency> </dependencies> <build> <plugins> diff --git a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/DeployData.java b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/DeployData.java index c3e9b99f562..de4ea62f671 100644 --- a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/DeployData.java +++ b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/DeployData.java @@ -2,7 +2,6 @@ package com.yahoo.config.model.application.provider; import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.Tags; import java.util.Set; @@ -15,8 +14,6 @@ public class DeployData { private final ApplicationId applicationId; - private final Tags tags; - /** The absolute path to the directory holding the application */ private final String deployedFromDir; @@ -32,14 +29,12 @@ public class DeployData { public DeployData(String deployedFromDir, ApplicationId applicationId, - Tags tags, Long deployTimestamp, boolean internalRedeploy, Long generation, long currentlyActiveGeneration) { this.deployedFromDir = deployedFromDir; this.applicationId = applicationId; - this.tags = tags; this.deployTimestamp = deployTimestamp; this.internalRedeploy = internalRedeploy; this.generation = generation; @@ -58,6 +53,4 @@ public class DeployData { public ApplicationId getApplicationId() { return applicationId; } - public Tags getTags() { return tags; } - } diff --git a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java index 2d91f811e8b..c095b9ad586 100644 --- a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java +++ b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java @@ -142,7 +142,6 @@ public class FilesApplicationPackage extends AbstractApplicationPackage { deployData.getDeployTimestamp(), deployData.isInternalRedeploy(), deployData.getApplicationId(), - deployData.getTags(), computeCheckSum(appDir), deployData.getGeneration(), deployData.getCurrentlyActiveGeneration()); @@ -489,7 +488,6 @@ public class FilesApplicationPackage extends AbstractApplicationPackage { ApplicationId.from(TenantName.defaultName(), ApplicationName.from(originalAppDir), InstanceName.defaultName()), - Tags.empty(), "", 0L, 0L); diff --git a/config-application-package/src/test/java/com/yahoo/config/application/HostedOverrideProcessorComplexTest.java b/config-application-package/src/test/java/com/yahoo/config/application/HostedOverrideProcessorComplexTest.java index 902b4126a29..93e038c786a 100644 --- a/config-application-package/src/test/java/com/yahoo/config/application/HostedOverrideProcessorComplexTest.java +++ b/config-application-package/src/test/java/com/yahoo/config/application/HostedOverrideProcessorComplexTest.java @@ -1,15 +1,18 @@ package com.yahoo.config.application; +import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.config.application.api.DeploymentInstanceSpec; +import com.yahoo.config.application.api.xml.DeploymentSpecXmlReader; +import com.yahoo.config.model.application.provider.FilesApplicationPackage; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.Tags; -import com.yahoo.io.IOUtils; -import org.custommonkey.xmlunit.XMLUnit; import org.junit.Test; import org.w3c.dom.Document; import javax.xml.transform.TransformerException; +import java.io.File; import java.io.IOException; import static org.junit.Assert.assertEquals; @@ -18,10 +21,6 @@ public class HostedOverrideProcessorComplexTest { private static final String servicesFile = "src/test/resources/complex-app/services.xml"; - static { - XMLUnit.setIgnoreWhitespace(true); - } - @Test public void testProdBetaUsWest2a() throws TransformerException, IOException { String expected = @@ -62,7 +61,6 @@ public class HostedOverrideProcessorComplexTest { assertOverride(InstanceName.from("beta1"), Environment.prod, RegionName.from("aws-us-west-2a"), - Tags.fromString("beta beta-prod beta-prod-cd"), expected); } @@ -106,12 +104,16 @@ public class HostedOverrideProcessorComplexTest { assertOverride(InstanceName.from("beta1"), Environment.prod, RegionName.from("aws-us-east-1b"), - Tags.fromString("beta beta-prod beta-prod-cd"), expected); } - private void assertOverride(InstanceName instance, Environment environment, RegionName region, Tags tags, String expected) throws TransformerException, IOException { - Document inputDoc = Xml.getDocument(IOUtils.createReader(servicesFile)); + private void assertOverride(InstanceName instance, Environment environment, RegionName region, String expected) throws TransformerException { + ApplicationPackage app = FilesApplicationPackage.fromFile(new File(servicesFile).getParentFile()); + Document inputDoc = Xml.getDocument(app.getServices()); + Tags tags = app.getDeployment() + .map(new DeploymentSpecXmlReader(false)::read) + .flatMap(spec -> spec.instance(instance).map(DeploymentInstanceSpec::tags)) + .orElse(Tags.empty()); Document newDoc = new OverrideProcessor(instance, environment, region, tags).process(inputDoc); assertEquals(expected, Xml.documentAsString(newDoc, true)); } diff --git a/config-application-package/src/test/java/com/yahoo/config/application/HostedOverrideProcessorTagsTest.java b/config-application-package/src/test/java/com/yahoo/config/application/HostedOverrideProcessorTagsTest.java index 8d7431d33b6..b8b09396fb4 100644 --- a/config-application-package/src/test/java/com/yahoo/config/application/HostedOverrideProcessorTagsTest.java +++ b/config-application-package/src/test/java/com/yahoo/config/application/HostedOverrideProcessorTagsTest.java @@ -5,7 +5,6 @@ import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.Tags; -import org.custommonkey.xmlunit.XMLUnit; import org.junit.Test; import org.w3c.dom.Document; @@ -17,10 +16,6 @@ import java.io.StringReader; */ public class HostedOverrideProcessorTagsTest { - static { - XMLUnit.setIgnoreWhitespace(true); - } - private static final String input = "<?xml version='1.0' encoding='UTF-8' standalone='no'?>" + "<services xmlns:deploy='vespa' xmlns:preprocess='?' version='1.0'>" + diff --git a/config-application-package/src/test/java/com/yahoo/config/application/HostedOverrideProcessorTest.java b/config-application-package/src/test/java/com/yahoo/config/application/HostedOverrideProcessorTest.java index 451c7a3c217..eb65c01522e 100644 --- a/config-application-package/src/test/java/com/yahoo/config/application/HostedOverrideProcessorTest.java +++ b/config-application-package/src/test/java/com/yahoo/config/application/HostedOverrideProcessorTest.java @@ -5,51 +5,42 @@ import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.Tags; -import org.custommonkey.xmlunit.XMLUnit; import org.junit.Test; import org.w3c.dom.Document; -import org.xml.sax.SAXException; -import javax.xml.parsers.ParserConfigurationException; -import javax.xml.stream.XMLStreamException; import javax.xml.transform.TransformerException; -import java.io.IOException; import java.io.StringReader; -import java.util.List; /** * @author bratseth */ public class HostedOverrideProcessorTest { - static { - XMLUnit.setIgnoreWhitespace(true); - } - private static final String input = - "<?xml version='1.0' encoding='UTF-8' standalone='no'?>" + - "<services xmlns:deploy='vespa' xmlns:preprocess='?' version='1.0'>" + - " <container id='foo' version='1.0'>" + - " <nodes count='1'/>" + - " <nodes count='3' deploy:environment='perf'/>" + - " <nodes deploy:environment='staging' count='2' required='true'/>" + - " <nodes deploy:environment='prod' count='3' flavor='v-4-8-100'/>" + - " <nodes deploy:environment='prod' deploy:region='us-west' count='4'/>" + - " <nodes deploy:environment='prod' deploy:region='us-east-3' flavor='v-8-8-100' count='5'/>" + - " <nodes deploy:instance='myinstance' deploy:environment='prod' deploy:region='us-west' count='1'/>" + - " </container>" + - "</services>"; + """ + <?xml version='1.0' encoding='UTF-8' standalone='no'?> + <services xmlns:deploy='vespa' xmlns:preprocess='?' version='1.0'> + <container id='foo' version='1.0'> + <nodes count='1'/> + <nodes count='3' deploy:environment='perf'/> + <nodes deploy:environment='staging' count='2' required='true'/> + <nodes deploy:environment='prod' count='3' flavor='v-4-8-100'/> + <nodes deploy:environment='prod' deploy:region='us-west' count='4'/> + <nodes deploy:environment='prod' deploy:region='us-east-3' flavor='v-8-8-100' count='5'/> + <nodes deploy:instance='myinstance' deploy:environment='prod' deploy:region='us-west' count='1'/> + </container></services>"""; @Test public void testParsingDefault() throws TransformerException { - String expected = - "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>" + - "<services xmlns:deploy=\"vespa\" xmlns:preprocess=\"?\" version=\"1.0\">" + - " <container id=\"foo\" version=\"1.0\">" + - " <nodes count='1'/>" + - " </container>" + - "</services>"; + String expected = + """ + <?xml version="1.0" encoding="UTF-8" standalone="no"?> + <services xmlns:deploy="vespa" xmlns:preprocess="?" version="1.0"> + <container id="foo" version="1.0"> + <nodes count='1'/> + </container> + </services>"""; assertOverride(InstanceName.defaultName(), Environment.test, RegionName.defaultName(), @@ -59,13 +50,14 @@ public class HostedOverrideProcessorTest { @Test public void testParsingEnvironmentAndRegion() throws TransformerException { - String expected = - "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>" + - "<services xmlns:deploy=\"vespa\" xmlns:preprocess=\"?\" version=\"1.0\">" + - " <container id=\"foo\" version=\"1.0\">" + - " <nodes count='4' required='true'/>" + - " </container>" + - "</services>"; + String expected = + """ + <?xml version="1.0" encoding="UTF-8" standalone="no"?> + <services xmlns:deploy="vespa" xmlns:preprocess="?" version="1.0"> + <container id="foo" version="1.0"> + <nodes count='4' required='true'/> + </container> + </services>"""; assertOverride(InstanceName.defaultName(), Environment.from("prod"), RegionName.from("us-west"), diff --git a/config-application-package/src/test/java/com/yahoo/config/application/IncludeProcessorTest.java b/config-application-package/src/test/java/com/yahoo/config/application/IncludeProcessorTest.java index 3de624c78ac..697d8c208d3 100644 --- a/config-application-package/src/test/java/com/yahoo/config/application/IncludeProcessorTest.java +++ b/config-application-package/src/test/java/com/yahoo/config/application/IncludeProcessorTest.java @@ -2,7 +2,6 @@ package com.yahoo.config.application; import com.yahoo.config.application.api.ApplicationPackage; -import org.junit.Assert; import org.junit.Test; import org.w3c.dom.Document; import org.xml.sax.SAXException; @@ -28,54 +27,58 @@ public class IncludeProcessorTest { DocumentBuilder docBuilder = Xml.getPreprocessDocumentBuilder(); String expected = - "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>" + - "<services xmlns:deploy=\"vespa\" xmlns:preprocess=\"properties\" version=\"1.0\">\n" + - " <preprocess:properties>\n" + - " <qrs.port>4099</qrs.port>\n" + - " <qrs.port>5000</qrs.port>\n" + - " </preprocess:properties>\n" + - " <preprocess:properties deploy:environment='prod'>\n" + - " <qrs.port deploy:region='us-west'>5001</qrs.port>" + - " <qrs.port deploy:region='us-east us-central'>5002</qrs.port>" + - " </preprocess:properties>\n" + - " <admin version=\"2.0\">\n" + - " <adminserver hostalias=\"node0\"/>\n" + - " </admin>\n" + - " <admin deploy:environment=\"staging prod\" deploy:region=\"us-east us-central\" version=\"2.0\">\n" + - " <adminserver hostalias=\"node1\"/>\n" + - " </admin>\n" + - " <content id=\"foo\" version=\"1.0\">\n" + - " <redundancy>1</redundancy><documents>\n" + - " <document mode=\"index\" type=\"music.sd\"/>\n" + - " </documents><nodes>\n" + - " <node distribution-key=\"0\" hostalias=\"node0\"/>\n" + - " </nodes>" + - " <nodes deploy:environment=\"prod\">\n" + - " <node distribution-key=\"0\" hostalias=\"node0\"/>\n" + - " <node distribution-key=\"1\" hostalias=\"node1\"/>\n" + - " </nodes>" + - " <nodes deploy:environment=\"prod\" deploy:region=\"us-west\">\n" + - " <node distribution-key=\"0\" hostalias=\"node0\"/>\n" + - " <node distribution-key=\"1\" hostalias=\"node1\"/>\n" + - " <node distribution-key=\"2\" hostalias=\"node2\"/>\n" + - " </nodes>" + - "</content>\n" + - "<container id=\"stateless\" version=\"1.0\">\n" + - " <search deploy:environment=\"prod\">\n" + - " <chain id=\"common\">\n" + - " <searcher id=\"MySearcher1\" />\n" + - " <searcher deploy:environment=\"prod\" id=\"MySearcher2\" />\n" + - " </chain>\n" + - " </search>\n" + - " <search/>\n" + - " <component id=\"foo\" class=\"MyFoo\" bundle=\"foobundle\" />\n" + - " <component id=\"bar\" class=\"TestBar\" bundle=\"foobundle\" deploy:environment=\"dev\" />\n" + - " <component id=\"bar\" class=\"ProdBar\" bundle=\"foobundle\" deploy:environment=\"prod\" />\n" + - " <component id=\"baz\" class=\"ProdBaz\" bundle=\"foobundle\" deploy:environment=\"prod\" />\n" + - " <nodes>\n" + - " <node baseport=\"${qrs.port}\" hostalias=\"node0\"/>\n" + - " </nodes>\n" + - "</container></services>"; + """ + <?xml version="1.0" encoding="UTF-8" standalone="no"?> + <!-- Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --><services xmlns:deploy="vespa" xmlns:preprocess="properties" version="1.0"> + <preprocess:properties> + <qrs.port>4099</qrs.port> + <qrs.port>5000</qrs.port> + </preprocess:properties> + <preprocess:properties deploy:environment="prod"> + <qrs.port deploy:region="us-west">5001</qrs.port> + <qrs.port deploy:region="us-east us-central">5002</qrs.port> + </preprocess:properties> + <admin version="2.0"> + <adminserver hostalias="node0"/> + </admin> + <admin deploy:environment="staging prod" deploy:region="us-east us-central" version="2.0"> + <adminserver hostalias="node1"/> + </admin> + <content id="foo" version="1.0"> + <redundancy>1</redundancy> + <documents> + <document mode="index" type="music.sd"/> + </documents> + <nodes> + <node distribution-key="0" hostalias="node0"/> + </nodes> + <nodes deploy:environment="prod"> + <node distribution-key="0" hostalias="node0"/> + <node distribution-key="1" hostalias="node1"/> + </nodes> + <nodes deploy:environment="prod" deploy:region="us-west"> + <node distribution-key="0" hostalias="node0"/> + <node distribution-key="1" hostalias="node1"/> + <node distribution-key="2" hostalias="node2"/> + </nodes> + </content> + <container id="stateless" version="1.0"> + <search deploy:environment="prod"> + <chain id="common"> + <searcher id="MySearcher1"/> + <searcher deploy:environment="prod" id="MySearcher2"/> + </chain> + </search> + <search/> + <component bundle="foobundle" class="MyFoo" id="foo"/> + <component bundle="foobundle" class="TestBar" deploy:environment="dev" id="bar"/> + <component bundle="foobundle" class="ProdBar" deploy:environment="prod" id="bar"/> + <component bundle="foobundle" class="ProdBaz" deploy:environment="prod" id="baz"/> + <nodes> + <node baseport="${qrs.port}" hostalias="node0"/> + </nodes> + </container> + </services>"""; Document doc = new IncludeProcessor(app).process(docBuilder.parse(getServices(app))); // System.out.println(Xml.documentAsString(doc)); diff --git a/config-application-package/src/test/java/com/yahoo/config/application/MultiOverrideProcessorTest.java b/config-application-package/src/test/java/com/yahoo/config/application/MultiOverrideProcessorTest.java index debde6c1438..8602b2955aa 100644 --- a/config-application-package/src/test/java/com/yahoo/config/application/MultiOverrideProcessorTest.java +++ b/config-application-package/src/test/java/com/yahoo/config/application/MultiOverrideProcessorTest.java @@ -5,7 +5,6 @@ import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.Tags; -import org.custommonkey.xmlunit.XMLUnit; import org.junit.Test; import org.w3c.dom.Document; @@ -19,107 +18,109 @@ import java.io.StringReader; */ public class MultiOverrideProcessorTest { - static { - XMLUnit.setIgnoreWhitespace(true); - } - private static final String input = - "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" + - "<services version=\"1.0\" xmlns:deploy=\"vespa\">\n" + - " <container id='default' version='1.0'>\n" + - " <component id=\"comp-B\" class=\"com.yahoo.ls.MyComponent\" bundle=\"lsbe-hv\">\n" + - " <config name=\"ls.config.resource-pool\">\n" + - " <resource>\n" + - " <item>\n" + - " <id>comp-B-item-0</id>\n" + - " <type></type>\n" + - " </item>\n" + - " <item deploy:environment=\"dev perf test staging prod\" deploy:region=\"us-west-1 us-east-3\">\n" + - " <id>comp-B-item-1</id>\n" + - " <type></type>\n" + - " </item>\n" + - " <item>\n" + - " <id>comp-B-item-2</id>\n" + - " <type></type>\n" + - " </item>\n" + - " </resource>\n" + - " </config>\n" + - " </component>\n" + - " </container>\n" + - "</services>\n"; + """ + <?xml version="1.0" encoding="UTF-8"?> + <services version="1.0" xmlns:deploy="vespa"> + <container id='default' version='1.0'> + <component id="comp-B" class="com.yahoo.ls.MyComponent" bundle="lsbe-hv"> + <config name="ls.config.resource-pool"> + <resource> + <item> + <id>comp-B-item-0</id> + <type></type> + </item> + <item deploy:environment="dev perf test staging prod" deploy:region="us-west-1 us-east-3"> + <id>comp-B-item-1</id> + <type></type> + </item> + <item> + <id>comp-B-item-2</id> + <type></type> + </item> + </resource> + </config> + </component> + </container> + </services> + """; private static final String inputWithIds = - "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" + - "<services version=\"1.0\" xmlns:deploy=\"vespa\">\n" + - " <container id='default' version='1.0'>\n" + - " <component id=\"comp-B\" class=\"com.yahoo.ls.MyComponent\" bundle=\"lsbe-hv\">\n" + - " <config name=\"ls.config.resource-pool\">\n" + - " <resource>\n" + - " <item id='1'>\n" + - " <id>comp-B-item-0</id>\n" + - " <type></type>\n" + - " </item>\n" + - " <item id='2' deploy:environment=\"dev perf test staging prod\" deploy:region=\"us-west-1 us-east-3\">\n" + - " <id>comp-B-item-1</id>\n" + - " <type></type>\n" + - " </item>\n" + - " <item id='3'>\n" + - " <id>comp-B-item-2</id>\n" + - " <type></type>\n" + - " </item>\n" + - " </resource>\n" + - " </config>\n" + - " </component>\n" + - " </container>\n" + - "</services>\n"; + """ + <?xml version="1.0" encoding="UTF-8"?> + <services version="1.0" xmlns:deploy="vespa"> + <container id='default' version='1.0'> + <component id="comp-B" class="com.yahoo.ls.MyComponent" bundle="lsbe-hv"> + <config name="ls.config.resource-pool"> + <resource> + <item id='1'> + <id>comp-B-item-0</id> + <type></type> + </item> + <item id='2' deploy:environment="dev perf test staging prod" deploy:region="us-west-1 us-east-3"> + <id>comp-B-item-1</id> + <type></type> + </item> + <item id='3'> + <id>comp-B-item-2</id> + <type></type> + </item> + </resource> + </config> + </component> + </container> + </services> + """; @Test public void testParsingDev() throws TransformerException { String expected = - "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" + - "<services version=\"1.0\" xmlns:deploy=\"vespa\">\n" + - " <container id='default' version='1.0'>\n" + - " <component id=\"comp-B\" class=\"com.yahoo.ls.MyComponent\" bundle=\"lsbe-hv\">\n" + - " <config name=\"ls.config.resource-pool\">\n" + - " <resource>\n" + - " <item>\n" + - " <id>comp-B-item-1</id>\n" + - " <type></type>\n" + - " </item>\n" + - " </resource>\n" + - " </config>\n" + - " </component>\n" + - " </container>\n" + - "</services>"; + """ + <?xml version="1.0" encoding="UTF-8"?> + <services version="1.0" xmlns:deploy="vespa"> + <container id='default' version='1.0'> + <component id="comp-B" class="com.yahoo.ls.MyComponent" bundle="lsbe-hv"> + <config name="ls.config.resource-pool"> + <resource> + <item> + <id>comp-B-item-1</id> + <type></type> + </item> + </resource> + </config> + </component> + </container> + </services>"""; assertOverride(Environment.dev, RegionName.from("us-east-3"), expected); } @Test public void testParsingDevWithIds() throws TransformerException { String expected = - "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" + - "<services version=\"1.0\" xmlns:deploy=\"vespa\">\n" + - " <container id='default' version='1.0'>\n" + - " <component id=\"comp-B\" class=\"com.yahoo.ls.MyComponent\" bundle=\"lsbe-hv\">\n" + - " <config name=\"ls.config.resource-pool\">\n" + - " <resource>\n" + - " <item id='1'>\n" + - " <id>comp-B-item-0</id>\n" + - " <type></type>\n" + - " </item>\n" + - " <item id='2'>\n" + - " <id>comp-B-item-1</id>\n" + - " <type></type>\n" + - " </item>\n" + - " <item id='3'>\n" + - " <id>comp-B-item-2</id>\n" + - " <type></type>\n" + - " </item>\n" + - " </resource>\n" + - " </config>\n" + - " </component>\n" + - " </container>\n" + - "</services>"; + """ + <?xml version="1.0" encoding="UTF-8"?> + <services version="1.0" xmlns:deploy="vespa"> + <container id='default' version='1.0'> + <component id="comp-B" class="com.yahoo.ls.MyComponent" bundle="lsbe-hv"> + <config name="ls.config.resource-pool"> + <resource> + <item id='1'> + <id>comp-B-item-0</id> + <type></type> + </item> + <item id='2'> + <id>comp-B-item-1</id> + <type></type> + </item> + <item id='3'> + <id>comp-B-item-2</id> + <type></type> + </item> + </resource> + </config> + </component> + </container> + </services>"""; assertOverrideWithIds(Environment.dev, RegionName.from("us-east-3"), expected); } diff --git a/config-application-package/src/test/java/com/yahoo/config/application/OverrideProcessorTest.java b/config-application-package/src/test/java/com/yahoo/config/application/OverrideProcessorTest.java index 150999390d8..ca074ef9704 100644 --- a/config-application-package/src/test/java/com/yahoo/config/application/OverrideProcessorTest.java +++ b/config-application-package/src/test/java/com/yahoo/config/application/OverrideProcessorTest.java @@ -5,7 +5,6 @@ import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.Tags; -import org.custommonkey.xmlunit.XMLUnit; import org.junit.Test; import org.w3c.dom.Document; @@ -17,10 +16,6 @@ import java.io.StringReader; */ public class OverrideProcessorTest { - static { - XMLUnit.setIgnoreWhitespace(true); - } - private static final String input = "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>" + "<services xmlns:deploy=\"vespa\" xmlns:preprocess=\"?\" version=\"1.0\">" + diff --git a/config-application-package/src/test/java/com/yahoo/config/application/PropertiesProcessorTest.java b/config-application-package/src/test/java/com/yahoo/config/application/PropertiesProcessorTest.java index 3bf0f25766c..b8896ca2b79 100644 --- a/config-application-package/src/test/java/com/yahoo/config/application/PropertiesProcessorTest.java +++ b/config-application-package/src/test/java/com/yahoo/config/application/PropertiesProcessorTest.java @@ -1,7 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.config.application; -import org.custommonkey.xmlunit.XMLUnit; import org.junit.Test; import org.w3c.dom.Document; @@ -16,25 +15,22 @@ import static org.junit.Assert.assertEquals; */ public class PropertiesProcessorTest { - static { - XMLUnit.setIgnoreWhitespace(true); - } - @Test public void testPropertyValues() throws TransformerException { - String input = "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>" + - "<services xmlns:deploy=\"vespa\" xmlns:preprocess=\"properties\" version=\"1.0\">" + - " <preprocess:properties>" + - " <slobrok.port>4099</slobrok.port>" + - " <redundancy>2</redundancy>" + - " </preprocess:properties>" + - " <admin version=\"2.0\">" + - " <adminserver hostalias=\"node0\"/>" + - " <slobroks>" + - " <slobrok hostalias=\"node1\" baseport=\"${slobrok.port}\"/>" + - " </slobroks>" + - " </admin>" + - "</services>"; + String input = """ + <?xml version="1.0" encoding="UTF-8" standalone="no"?> + <services xmlns:deploy="vespa" xmlns:preprocess="properties" version="1.0"> + <preprocess:properties> + <slobrok.port>4099</slobrok.port> + <redundancy>2</redundancy> + </preprocess:properties> + <admin version="2.0"> + <adminserver hostalias="node0"/> + <slobroks> + <slobrok hostalias="node1" baseport="${slobrok.port}"/> + </slobroks> + </admin> + </services>"""; PropertiesProcessor p = new PropertiesProcessor(); p.process(Xml.getDocument(new StringReader(input))); @@ -46,49 +42,51 @@ public class PropertiesProcessorTest { @Test public void testPropertyApplying() throws TransformerException { - String input = "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>" + - "<services xmlns:deploy=\"vespa\" xmlns:preprocess=\"properties\" version=\"1.0\">" + - " <preprocess:properties>" + - " <slobrok.port>4099</slobrok.port>" + - " <redundancy>2</redundancy>" + - " <doctype>music</doctype>" + - " <zero>0</zero>" + - " </preprocess:properties>" + - " <admin version=\"2.0\">" + - " <adminserver hostalias=\"node0\"/>" + - " <slobroks>" + - " <slobrok hostalias=\"node1\" baseport=\"${slobrok.port}\"/>" + - " </slobroks>" + - " </admin>" + - " <content id=\"foo\" version=\"1.0\">" + - " <redundancy>${redundancy}</redundancy>" + - " <documents>" + - " <document mode=\"index\" type=\"${doctype}.sd\"/>" + - " </documents>" + - " <nodes>" + - " <node distribution-key=\"${zero}\" hostalias=\"node${zero}\"/>" + - " </nodes>" + - " </content>" + - "</services>"; - - String expected = "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>" + - "<services xmlns:deploy=\"vespa\" xmlns:preprocess=\"properties\" version=\"1.0\">" + - " <admin version=\"2.0\">" + - " <adminserver hostalias=\"node0\"/>" + - " <slobroks>" + - " <slobrok hostalias=\"node1\" baseport=\"4099\"/>" + - " </slobroks>" + - " </admin>" + - " <content id=\"foo\" version=\"1.0\">" + - " <redundancy>2</redundancy>" + - " <documents>" + - " <document mode=\"index\" type=\"music.sd\"/>" + - " </documents>" + - " <nodes>" + - " <node distribution-key=\"0\" hostalias=\"node0\"/>" + - " </nodes>" + - " </content>" + - "</services>"; + String input = """ + <?xml version="1.0" encoding="UTF-8" standalone="no"?> + <services xmlns:deploy="vespa" xmlns:preprocess="properties" version="1.0"> + <preprocess:properties> + <slobrok.port>4099</slobrok.port> + <redundancy>2</redundancy> + <doctype>music</doctype> + <zero>0</zero> + </preprocess:properties> + <admin version="2.0"> + <adminserver hostalias="node0"/> + <slobroks> + <slobrok hostalias="node1" baseport="${slobrok.port}"/> + </slobroks> + </admin> + <content id="foo" version="1.0"> + <redundancy>${redundancy}</redundancy> + <documents> + <document mode="index" type="${doctype}.sd"/> + </documents> + <nodes> + <node distribution-key="${zero}" hostalias="node${zero}"/> + </nodes> + </content> + </services>"""; + + String expected = """ + <?xml version="1.0" encoding="UTF-8" standalone="no"?> + <services xmlns:deploy="vespa" xmlns:preprocess="properties" version="1.0"> + <admin version="2.0"> + <adminserver hostalias="node0"/> + <slobroks> + <slobrok hostalias="node1" baseport="4099"/> + </slobroks> + </admin> + <content id="foo" version="1.0"> + <redundancy>2</redundancy> + <documents> + <document mode="index" type="music.sd"/> + </documents> + <nodes> + <node distribution-key="0" hostalias="node0"/> + </nodes> + </content> + </services>"""; Document inputDoc = Xml.getDocument(new StringReader(input)); @@ -100,31 +98,34 @@ public class PropertiesProcessorTest { // TODO: Check that warning is actually logged @Test public void testWarnIfDuplicatePropertyForSameEnvironment() throws TransformerException { - String input = "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>" + - "<services xmlns:deploy=\"vespa\" xmlns:preprocess=\"properties\" version=\"1.0\">" + - " <preprocess:properties>" + - " <slobrok.port>4099</slobrok.port>" + - " <slobrok.port>5000</slobrok.port>" + - " <redundancy>2</redundancy>" + - " </preprocess:properties>" + - " <admin version=\"2.0\">" + - " <adminserver hostalias=\"node0\"/>" + - " <slobroks>" + - " <slobrok hostalias=\"node1\" baseport=\"${slobrok.port}\"/>" + - " </slobroks>" + - " </admin>" + - "</services>"; - - - String expected = "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>" + - "<services xmlns:deploy=\"vespa\" xmlns:preprocess=\"properties\" version=\"1.0\">" + - " <admin version=\"2.0\">" + - " <adminserver hostalias=\"node0\"/>" + - " <slobroks>" + - " <slobrok hostalias=\"node1\" baseport=\"5000\"/>" + // Should get the last defined value - " </slobroks>" + - " </admin>" + - "</services>"; + String input = """ + <?xml version="1.0" encoding="UTF-8" standalone="no"?> + <services xmlns:deploy="vespa" xmlns:preprocess="properties" version="1.0"> + <preprocess:properties> + <slobrok.port>4099</slobrok.port> + <slobrok.port>5000</slobrok.port> + <redundancy>2</redundancy> + </preprocess:properties> + <admin version="2.0"> + <adminserver hostalias="node0"/> + <slobroks> + <slobrok hostalias="node1" baseport="${slobrok.port}"/> + </slobroks> + </admin> + </services>"""; + + + // Should get the last defined value + String expected = """ + <?xml version="1.0" encoding="UTF-8" standalone="no"?> + <services xmlns:deploy="vespa" xmlns:preprocess="properties" version="1.0"> + <admin version="2.0"> + <adminserver hostalias="node0"/> + <slobroks> + <slobrok hostalias="node1" baseport="5000"/> + </slobroks> + </admin> + </services>"""; Document inputDoc = Xml.getDocument(new StringReader(input)); Document newDoc = new PropertiesProcessor().process(inputDoc); diff --git a/config-application-package/src/test/java/com/yahoo/config/application/TestBase.java b/config-application-package/src/test/java/com/yahoo/config/application/TestBase.java index 06172c2698f..7e3a4758f8e 100644 --- a/config-application-package/src/test/java/com/yahoo/config/application/TestBase.java +++ b/config-application-package/src/test/java/com/yahoo/config/application/TestBase.java @@ -1,14 +1,13 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.config.application; -import org.custommonkey.xmlunit.Diff; -import org.custommonkey.xmlunit.XMLUnit; import org.w3c.dom.Document; +import javax.xml.transform.TransformerException; import java.io.Reader; import java.io.StringReader; -import static org.junit.Assert.assertTrue; +import static org.junit.Assert.assertEquals; /** * Utilities for tests @@ -16,14 +15,15 @@ import static org.junit.Assert.assertTrue; * @author hmusum */ public class TestBase { - static { - XMLUnit.setIgnoreWhitespace(true); - } static void assertDocument(String expected, Document output) { - Document expectedDoc = Xml.getDocument(new StringReader(expected)); - Diff diff = new Diff(expectedDoc, output); - assertTrue(diff.toString(), diff.identical()); + try { + assertEquals(Xml.documentAsString(Xml.getDocument(new StringReader(expected)), true), + Xml.documentAsString(output, true)); + } + catch (TransformerException e) { + throw new AssertionError(e); + } } public static void assertDocument(String expectedDocument, Reader document) { diff --git a/config-application-package/src/test/java/com/yahoo/config/application/XmlPreprocessorTest.java b/config-application-package/src/test/java/com/yahoo/config/application/XmlPreprocessorTest.java index 0da94b69e58..49dbacbae3d 100644 --- a/config-application-package/src/test/java/com/yahoo/config/application/XmlPreprocessorTest.java +++ b/config-application-package/src/test/java/com/yahoo/config/application/XmlPreprocessorTest.java @@ -23,29 +23,31 @@ public class XmlPreprocessorTest { @Test public void testPreProcessing() throws Exception { String expectedDev = - "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>" + - "<services xmlns:deploy=\"vespa\" xmlns:preprocess=\"properties\" version=\"1.0\">\n" + - " <admin version=\"2.0\">\n" + - " <adminserver hostalias=\"node0\"/>\n" + - " </admin>\n" + - " <content id=\"foo\" version=\"1.0\">\n" + - " <redundancy>1</redundancy>\n" + - " <documents>\n" + - " <document mode=\"index\" type=\"music.sd\"/>\n" + - " </documents>\n" + - " <nodes>\n" + - " <node distribution-key=\"0\" hostalias=\"node0\"/>\n" + - " </nodes>\n" + - " </content>\n" + - " <container id=\"stateless\" version=\"1.0\">\n" + - " <search/>\n" + - " <component bundle=\"foobundle\" class=\"MyFoo\" id=\"foo\"/>\n" + - " <component bundle=\"foobundle\" class=\"TestBar\" id=\"bar\"/>\n" + - " <nodes>\n" + - " <node hostalias=\"node0\" baseport=\"5000\"/>\n" + - " </nodes>\n" + - " </container>\n" + - "</services>"; + """ + <?xml version="1.0" encoding="UTF-8" standalone="no"?> + <!-- Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> + <services xmlns:deploy="vespa" xmlns:preprocess="properties" version="1.0"> + <admin version="2.0"> + <adminserver hostalias="node0"/> + </admin> + <content id="foo" version="1.0"> + <redundancy>1</redundancy> + <documents> + <document mode="index" type="music.sd"/> + </documents> + <nodes> + <node distribution-key="0" hostalias="node0"/> + </nodes> + </content> + <container id="stateless" version="1.0"> + <search/> + <component bundle="foobundle" class="MyFoo" id="foo"/> + <component bundle="foobundle" class="TestBar" id="bar"/> + <nodes> + <node hostalias="node0" baseport="5000"/> + </nodes> + </container> + </services>"""; TestBase.assertDocument(expectedDev, new XmlPreProcessor(appDir, services, @@ -54,30 +56,33 @@ public class XmlPreprocessorTest { RegionName.defaultName(), Tags.empty()).run()); + // Difference from dev: node1 + // Difference from dev: no TestBar String expectedStaging = - "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>" + - "<services xmlns:deploy=\"vespa\" xmlns:preprocess=\"properties\" version=\"1.0\">\n" + - " <admin version=\"2.0\">\n" + - " <adminserver hostalias=\"node1\"/>\n" + // Difference from dev: node1 - " </admin>\n" + - " <content id=\"foo\" version=\"1.0\">\n" + - " <redundancy>1</redundancy>\n" + - " <documents>\n" + - " <document mode=\"index\" type=\"music.sd\"/>\n" + - " </documents>\n" + - " <nodes>\n" + - " <node distribution-key=\"0\" hostalias=\"node0\"/>\n" + - " </nodes>\n" + - " </content>\n" + - " <container id=\"stateless\" version=\"1.0\">\n" + - " <search/>\n" + - " <component bundle=\"foobundle\" class=\"MyFoo\" id=\"foo\"/>\n" + - "" + // Difference from dev: no TestBar - " <nodes>\n" + - " <node hostalias=\"node0\" baseport=\"5000\"/>\n" + - " </nodes>\n" + - " </container>\n" + - "</services>"; + """ + <?xml version="1.0" encoding="UTF-8" standalone="no"?> + <!-- Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> + <services xmlns:deploy="vespa" xmlns:preprocess="properties" version="1.0"> + <admin version="2.0"> + <adminserver hostalias="node1"/> + </admin> + <content id="foo" version="1.0"> + <redundancy>1</redundancy> + <documents> + <document mode="index" type="music.sd"/> + </documents> + <nodes> + <node distribution-key="0" hostalias="node0"/> + </nodes> + </content> + <container id="stateless" version="1.0"> + <search/> + <component bundle="foobundle" class="MyFoo" id="foo"/> + <nodes> + <node hostalias="node0" baseport="5000"/> + </nodes> + </container> + </services>"""; TestBase.assertDocument(expectedStaging, new XmlPreProcessor(appDir, services, @@ -87,37 +92,39 @@ public class XmlPreprocessorTest { Tags.empty()).run()); String expectedUsWest = - "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>" + - "<services xmlns:deploy=\"vespa\" xmlns:preprocess=\"properties\" version=\"1.0\">\n" + - " <admin version=\"2.0\">\n" + - " <adminserver hostalias=\"node0\"/>\n" + - " </admin>\n" + - " <content id=\"foo\" version=\"1.0\">\n" + - " <redundancy>1</redundancy>\n" + - " <documents>\n" + - " <document mode=\"index\" type=\"music.sd\"/>\n" + - " </documents>\n" + - " <nodes>\n" + - " <node distribution-key=\"0\" hostalias=\"node0\"/>\n" + - " <node distribution-key=\"1\" hostalias=\"node1\"/>\n" + - " <node distribution-key=\"2\" hostalias=\"node2\"/>\n" + - " </nodes>\n" + - " </content>\n" + - " <container id=\"stateless\" version=\"1.0\">\n" + - " <search>\n" + - " <chain id=\"common\">\n" + - " <searcher id=\"MySearcher1\"/>\n" + - " <searcher id=\"MySearcher2\"/>\n" + - " </chain>\n" + - " </search>\n" + - " <component bundle=\"foobundle\" class=\"MyFoo\" id=\"foo\"/>\n" + - " <component bundle=\"foobundle\" class=\"ProdBar\" id=\"bar\"/>\n" + - " <component bundle=\"foobundle\" class=\"ProdBaz\" id=\"baz\"/>\n" + - " <nodes>\n" + - " <node hostalias=\"node0\" baseport=\"5001\"/>\n" + - " </nodes>\n" + - " </container>\n" + - "</services>"; + """ + <?xml version="1.0" encoding="UTF-8" standalone="no"?> + <!-- Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> + <services xmlns:deploy="vespa" xmlns:preprocess="properties" version="1.0"> + <admin version="2.0"> + <adminserver hostalias="node0"/> + </admin> + <content id="foo" version="1.0"> + <redundancy>1</redundancy> + <documents> + <document mode="index" type="music.sd"/> + </documents> + <nodes> + <node distribution-key="0" hostalias="node0"/> + <node distribution-key="1" hostalias="node1"/> + <node distribution-key="2" hostalias="node2"/> + </nodes> + </content> + <container id="stateless" version="1.0"> + <search> + <chain id="common"> + <searcher id="MySearcher1"/> + <searcher id="MySearcher2"/> + </chain> + </search> + <component bundle="foobundle" class="MyFoo" id="foo"/> + <component bundle="foobundle" class="ProdBar" id="bar"/> + <component bundle="foobundle" class="ProdBaz" id="baz"/> + <nodes> + <node hostalias="node0" baseport="5001"/> + </nodes> + </container> + </services>"""; TestBase.assertDocument(expectedUsWest, new XmlPreProcessor(appDir, services, @@ -127,36 +134,38 @@ public class XmlPreprocessorTest { Tags.empty()).run()); String expectedUsEastAndCentral = - "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>" + - "<services xmlns:deploy=\"vespa\" xmlns:preprocess=\"properties\" version=\"1.0\">\n" + - " <admin version=\"2.0\">\n" + - " <adminserver hostalias=\"node1\"/>\n" + - " </admin>\n" + - " <content id=\"foo\" version=\"1.0\">\n" + - " <redundancy>1</redundancy>\n" + - " <documents>\n" + - " <document mode=\"index\" type=\"music.sd\"/>\n" + - " </documents>\n" + - " <nodes>\n" + - " <node distribution-key=\"0\" hostalias=\"node0\"/>\n" + - " <node distribution-key=\"1\" hostalias=\"node1\"/>\n" + - " </nodes>\n" + - " </content>\n" + - " <container id=\"stateless\" version=\"1.0\">\n" + - " <search>\n" + - " <chain id=\"common\">\n" + - " <searcher id=\"MySearcher1\"/>\n" + - " <searcher id=\"MySearcher2\"/>\n" + - " </chain>\n" + - " </search>\n" + - " <component bundle=\"foobundle\" class=\"MyFoo\" id=\"foo\"/>\n" + - " <component bundle=\"foobundle\" class=\"ProdBar\" id=\"bar\"/>\n" + - " <component bundle=\"foobundle\" class=\"ProdBaz\" id=\"baz\"/>\n" + - " <nodes>\n" + - " <node hostalias=\"node0\" baseport=\"5002\"/>\n" + - " </nodes>\n" + - " </container>\n" + - "</services>"; + """ + <?xml version="1.0" encoding="UTF-8" standalone="no"?> + <!-- Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> + <services xmlns:deploy="vespa" xmlns:preprocess="properties" version="1.0"> + <admin version="2.0"> + <adminserver hostalias="node1"/> + </admin> + <content id="foo" version="1.0"> + <redundancy>1</redundancy> + <documents> + <document mode="index" type="music.sd"/> + </documents> + <nodes> + <node distribution-key="0" hostalias="node0"/> + <node distribution-key="1" hostalias="node1"/> + </nodes> + </content> + <container id="stateless" version="1.0"> + <search> + <chain id="common"> + <searcher id="MySearcher1"/> + <searcher id="MySearcher2"/> + </chain> + </search> + <component bundle="foobundle" class="MyFoo" id="foo"/> + <component bundle="foobundle" class="ProdBar" id="bar"/> + <component bundle="foobundle" class="ProdBaz" id="baz"/> + <nodes> + <node hostalias="node0" baseport="5002"/> + </nodes> + </container> + </services>"""; TestBase.assertDocument(expectedUsEastAndCentral, new XmlPreProcessor(appDir, services, @@ -176,44 +185,46 @@ public class XmlPreprocessorTest { @Test public void testPropertiesWithOverlappingNames() throws Exception { String input = - "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>" + - "<services xmlns:deploy=\"vespa\" xmlns:preprocess=\"properties\" version=\"1.0\">" + - " <preprocess:properties>" + - " <sherpa.host>gamma-usnc1.dht.yahoo.com</sherpa.host>" + - " <sherpa.port>4080</sherpa.port>" + - " <lidspacecompaction_interval>3600</lidspacecompaction_interval>" + - " <lidspacecompaction_interval deploy:environment='prod'>36000</lidspacecompaction_interval>" + - " <lidspacecompaction_allowedlidbloat>50000</lidspacecompaction_allowedlidbloat>" + - " <lidspacecompaction_allowedlidbloat deploy:environment='prod'>50000000</lidspacecompaction_allowedlidbloat>" + - " <lidspacecompaction_allowedlidbloatfactor>0.01</lidspacecompaction_allowedlidbloatfactor>" + - " <lidspacecompaction_allowedlidbloatfactor deploy:environment='prod'>0.91</lidspacecompaction_allowedlidbloatfactor>" + - " </preprocess:properties>" + - " <config name='a'>" + - " <a>${lidspacecompaction_interval}</a>" + - " <b>${lidspacecompaction_allowedlidbloat}</b>" + - " <c>${lidspacecompaction_allowedlidbloatfactor}</c>" + - " <host>${sherpa.host}</host>" + - " <port>${sherpa.port}</port>" + - " </config>" + - " <admin version=\"2.0\">" + - " <adminserver hostalias=\"node0\"/>" + - " </admin>" + - "</services>"; + """ + <?xml version="1.0" encoding="UTF-8" standalone="no"?> + <services xmlns:deploy="vespa" xmlns:preprocess="properties" version="1.0"> + <preprocess:properties> + <sherpa.host>gamma-usnc1.dht.yahoo.com</sherpa.host> + <sherpa.port>4080</sherpa.port> + <lidspacecompaction_interval>3600</lidspacecompaction_interval> + <lidspacecompaction_interval deploy:environment='prod'>36000</lidspacecompaction_interval> + <lidspacecompaction_allowedlidbloat>50000</lidspacecompaction_allowedlidbloat> + <lidspacecompaction_allowedlidbloat deploy:environment='prod'>50000000</lidspacecompaction_allowedlidbloat> + <lidspacecompaction_allowedlidbloatfactor>0.01</lidspacecompaction_allowedlidbloatfactor> + <lidspacecompaction_allowedlidbloatfactor deploy:environment='prod'>0.91</lidspacecompaction_allowedlidbloatfactor> + </preprocess:properties> + <config name='a'> + <a>${lidspacecompaction_interval}</a> + <b>${lidspacecompaction_allowedlidbloat}</b> + <c>${lidspacecompaction_allowedlidbloatfactor}</c> + <host>${sherpa.host}</host> + <port>${sherpa.port}</port> + </config> + <admin version="2.0"> + <adminserver hostalias="node0"/> + </admin> + </services>"""; String expectedProd = - "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>" + - "<services xmlns:deploy=\"vespa\" xmlns:preprocess=\"properties\" version=\"1.0\">" + - " <config name='a'>" + - " <a>36000</a>" + - " <b>50000000</b>" + - " <c>0.91</c>" + - " <host>gamma-usnc1.dht.yahoo.com</host>" + - " <port>4080</port>" + - " </config>" + - " <admin version=\"2.0\">" + - " <adminserver hostalias=\"node0\"/>" + - " </admin>" + - "</services>"; + """ + <?xml version="1.0" encoding="UTF-8" standalone="no"?> + <services xmlns:deploy="vespa" xmlns:preprocess="properties" version="1.0"> + <config name='a'> + <a>36000</a> + <b>50000000</b> + <c>0.91</c> + <host>gamma-usnc1.dht.yahoo.com</host> + <port>4080</port> + </config> + <admin version="2.0"> + <adminserver hostalias="node0"/> + </admin> + </services>"""; Document docDev = (new XmlPreProcessor(appDir, new StringReader(input), InstanceName.defaultName(), diff --git a/config-application-package/src/test/java/com/yahoo/config/model/application/provider/FilesApplicationPackageTest.java b/config-application-package/src/test/java/com/yahoo/config/model/application/provider/FilesApplicationPackageTest.java index 29bc2b6fbd6..6c83b2029ad 100644 --- a/config-application-package/src/test/java/com/yahoo/config/model/application/provider/FilesApplicationPackageTest.java +++ b/config-application-package/src/test/java/com/yahoo/config/model/application/provider/FilesApplicationPackageTest.java @@ -41,34 +41,40 @@ public class FilesApplicationPackageTest { ApplicationPackage processed = app.preprocess(new Zone(Environment.dev, RegionName.defaultName()), new BaseDeployLogger()); assertTrue(new File(appDir, ".preprocessed").exists()); - String expectedServices = "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?><services xmlns:deploy=\"vespa\" xmlns:preprocess=\"properties\" version=\"1.0\">\n" + - " <admin version=\"2.0\">\n" + - " <adminserver hostalias=\"node0\"/>\n" + - " </admin>\n" + - " <content id=\"foo\" version=\"1.0\">\n" + - " <redundancy>1</redundancy>\n" + - " <documents>\n" + - " <document mode=\"index\" type=\"music.sd\"/>\n" + - " </documents>\n" + - " <nodes>\n" + - " <node distribution-key=\"0\" hostalias=\"node0\"/>\n" + - " </nodes>\n" + - " </content>\n" + - " <container id=\"stateless\" version=\"1.0\">\n" + - " <search/>\n" + - " <component bundle=\"foobundle\" class=\"MyFoo\" id=\"foo\"/>\n" + - " <component bundle=\"foobundle\" class=\"TestBar\" id=\"bar\"/>\n" + - " <nodes>\n" + - " <node hostalias=\"node0\" baseport=\"5000\"/>\n" + - " </nodes>\n" + - " </container>\n" + - "</services>"; + String expectedServices = """ + <?xml version="1.0" encoding="UTF-8" standalone="no"?> + <!-- Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> + <services xmlns:deploy="vespa" xmlns:preprocess="properties" version="1.0"> + <admin version="2.0"> + <adminserver hostalias="node0"/> + </admin> + <content id="foo" version="1.0"> + <redundancy>1</redundancy> + <documents> + <document mode="index" type="music.sd"/> + </documents> + <nodes> + <node distribution-key="0" hostalias="node0"/> + </nodes> + </content> + <container id="stateless" version="1.0"> + <search/> + <component bundle="foobundle" class="MyFoo" id="foo"/> + <component bundle="foobundle" class="TestBar" id="bar"/> + <nodes> + <node hostalias="node0" baseport="5000"/> + </nodes> + </container> + </services>"""; TestBase.assertDocument(expectedServices, processed.getServices()); - String expectedHosts = "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?><hosts xmlns:deploy=\"vespa\" xmlns:preprocess=\"properties\">\n" + - " <host name=\"bar.yahoo.com\">\n" + - " <alias>node1</alias>\n" + - " </host>\n" + - "</hosts>"; + String expectedHosts = """ + <?xml version="1.0" encoding="UTF-8" standalone="no"?> + <!-- Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> + <hosts xmlns:deploy="vespa" xmlns:preprocess="properties"> + <host name="bar.yahoo.com"> + <alias>node1</alias> + </host> + </hosts>"""; TestBase.assertDocument(expectedHosts, processed.getHosts()); } diff --git a/config-model-api/abi-spec.json b/config-model-api/abi-spec.json deleted file mode 100644 index 9288debdb72..00000000000 --- a/config-model-api/abi-spec.json +++ /dev/null @@ -1,742 +0,0 @@ -{ - "com.yahoo.config.application.api.ApplicationFile$MetaData" : { - "superClass" : "java.lang.Object", - "interfaces" : [ ], - "attributes" : [ - "public" - ], - "methods" : [ - "public void <init>()", - "public void <init>(java.lang.String, java.lang.String)", - "public java.lang.String getStatus()", - "public java.lang.String getMd5()" - ], - "fields" : [ - "public java.lang.String status", - "public java.lang.String md5" - ] - }, - "com.yahoo.config.application.api.ApplicationFile$PathFilter" : { - "superClass" : "java.lang.Object", - "interfaces" : [ ], - "attributes" : [ - "public", - "interface", - "abstract" - ], - "methods" : [ - "public abstract boolean accept(com.yahoo.path.Path)" - ], - "fields" : [ ] - }, - "com.yahoo.config.application.api.ApplicationFile" : { - "superClass" : "java.lang.Object", - "interfaces" : [ - "java.lang.Comparable" - ], - "attributes" : [ - "public", - "abstract" - ], - "methods" : [ - "protected void <init>(com.yahoo.path.Path)", - "public abstract boolean isDirectory()", - "public abstract boolean exists()", - "public abstract java.io.Reader createReader()", - "public abstract java.io.InputStream createInputStream()", - "public abstract com.yahoo.config.application.api.ApplicationFile createDirectory()", - "public abstract com.yahoo.config.application.api.ApplicationFile writeFile(java.io.Reader)", - "public abstract com.yahoo.config.application.api.ApplicationFile appendFile(java.lang.String)", - "public java.util.List listFiles()", - "public abstract java.util.List listFiles(com.yahoo.config.application.api.ApplicationFile$PathFilter)", - "public java.util.List listFiles(boolean)", - "public abstract com.yahoo.config.application.api.ApplicationFile delete()", - "public com.yahoo.path.Path getPath()", - "public java.lang.String toString()", - "public boolean equals(java.lang.Object)", - "protected com.yahoo.path.Path getMetaPath()", - "public abstract com.yahoo.config.application.api.ApplicationFile$MetaData getMetaData()" - ], - "fields" : [ - "public static final java.lang.String ContentStatusNew", - "public static final java.lang.String ContentStatusChanged", - "public static final java.lang.String ContentStatusDeleted", - "protected final com.yahoo.path.Path path" - ] - }, - "com.yahoo.config.application.api.ApplicationMetaData" : { - "superClass" : "java.lang.Object", - "interfaces" : [ ], - "attributes" : [ - "public" - ], - "methods" : [ - "public void <init>(java.lang.String, java.lang.Long, boolean, com.yahoo.config.provision.ApplicationId, com.yahoo.config.provision.Tags, java.lang.String, java.lang.Long, long)", - "public void <init>(java.lang.String, java.lang.Long, boolean, com.yahoo.config.provision.ApplicationId, java.lang.String, java.lang.Long, long)", - "public void <init>(java.lang.String, java.lang.String, java.lang.Long, boolean, com.yahoo.config.provision.ApplicationId, java.lang.String, java.lang.Long, long)", - "public java.lang.String getDeployedByUser()", - "public java.lang.String getDeployPath()", - "public com.yahoo.config.provision.ApplicationId getApplicationId()", - "public com.yahoo.config.provision.Tags getTags()", - "public java.lang.Long getDeployTimestamp()", - "public java.lang.Long getGeneration()", - "public boolean isInternalRedeploy()", - "public java.lang.String getChecksum()", - "public long getPreviousActiveGeneration()", - "public java.lang.String toString()", - "public static com.yahoo.config.application.api.ApplicationMetaData fromJsonString(java.lang.String)", - "public com.yahoo.slime.Slime getSlime()", - "public java.lang.String asJsonString()", - "public byte[] asJsonBytes()" - ], - "fields" : [ ] - }, - "com.yahoo.config.application.api.ApplicationPackage" : { - "superClass" : "java.lang.Object", - "interfaces" : [ ], - "attributes" : [ - "public", - "interface", - "abstract" - ], - "methods" : [ - "public abstract com.yahoo.config.provision.ApplicationId getApplicationId()", - "public abstract java.io.Reader getServices()", - "public abstract java.io.Reader getHosts()", - "public java.util.List getUserIncludeDirs()", - "public void validateIncludeDir(java.lang.String)", - "public abstract java.util.Map getAllExistingConfigDefs()", - "public abstract java.util.List getFiles(com.yahoo.path.Path, java.lang.String, boolean)", - "public java.util.List getFiles(com.yahoo.path.Path, java.lang.String)", - "public java.util.Optional getMajorVersion()", - "public abstract com.yahoo.config.application.api.ApplicationFile getFile(com.yahoo.path.Path)", - "public java.util.List getQueryProfileFiles()", - "public java.util.List getQueryProfileTypeFiles()", - "public java.util.List getPageTemplateFiles()", - "public com.yahoo.config.application.api.ApplicationFile getClientSecurityFile()", - "public abstract java.lang.String getHostSource()", - "public abstract java.lang.String getServicesSource()", - "public abstract java.util.Optional getDeployment()", - "public abstract java.util.Optional getValidationOverrides()", - "public abstract java.util.List getComponentsInfo(com.yahoo.component.Version)", - "public abstract java.io.Reader getRankingExpression(java.lang.String)", - "public static java.lang.String getFileName(java.util.jar.JarEntry)", - "public abstract com.yahoo.config.application.api.ApplicationMetaData getMetaData()", - "public abstract java.io.File getFileReference(com.yahoo.path.Path)", - "public void validateXML()", - "public void validateXMLFor(java.util.Optional)", - "public void writeMetaData()", - "public java.util.Optional getAllocatedHosts()", - "public java.util.Map getFileRegistries()", - "public java.util.Map legacyOverrides()", - "public abstract java.util.Collection getSchemas()", - "public com.yahoo.config.application.api.ApplicationPackage preprocess(com.yahoo.config.provision.Zone, com.yahoo.config.application.api.DeployLogger)" - ], - "fields" : [ - "public static final java.lang.String HOSTS", - "public static final java.lang.String SERVICES", - "public static final com.yahoo.path.Path SCHEMAS_DIR", - "public static final com.yahoo.path.Path SEARCH_DEFINITIONS_DIR", - "public static final java.lang.String COMPONENT_DIR", - "public static final java.lang.String SEARCHCHAINS_DIR", - "public static final java.lang.String DOCPROCCHAINS_DIR", - "public static final java.lang.String PROCESSORCHAINS_DIR", - "public static final java.lang.String ROUTINGTABLES_DIR", - "public static final com.yahoo.path.Path MODELS_DIR", - "public static final com.yahoo.path.Path MODELS_GENERATED_DIR", - "public static final com.yahoo.path.Path MODELS_GENERATED_REPLICATED_DIR", - "public static final com.yahoo.path.Path CONSTANTS_DIR", - "public static final java.lang.String CONFIG_DEFINITIONS_DIR", - "public static final com.yahoo.path.Path QUERY_PROFILES_DIR", - "public static final com.yahoo.path.Path QUERY_PROFILE_TYPES_DIR", - "public static final com.yahoo.path.Path PAGE_TEMPLATES_DIR", - "public static final com.yahoo.path.Path RULES_DIR", - "public static final com.yahoo.path.Path DEPLOYMENT_FILE", - "public static final com.yahoo.path.Path VALIDATION_OVERRIDES", - "public static final com.yahoo.path.Path SECURITY_DIR", - "public static final java.lang.String SD_NAME_SUFFIX", - "public static final java.lang.String RANKEXPRESSION_NAME_SUFFIX", - "public static final java.lang.String RANKPROFILE_NAME_SUFFIX", - "public static final java.lang.String RULES_NAME_SUFFIX", - "public static final java.lang.String EXT_DIR", - "public static final java.lang.String PERMANENT_SERVICES" - ] - }, - "com.yahoo.config.application.api.ComponentInfo" : { - "superClass" : "java.lang.Object", - "interfaces" : [ ], - "attributes" : [ - "public" - ], - "methods" : [ - "public void <init>(java.lang.String)", - "public java.lang.String getPathRelativeToAppDir()", - "public java.lang.String toString()" - ], - "fields" : [ ] - }, - "com.yahoo.config.application.api.DeployLogger" : { - "superClass" : "java.lang.Object", - "interfaces" : [ ], - "attributes" : [ - "public", - "interface", - "abstract" - ], - "methods" : [ - "public abstract void log(java.util.logging.Level, java.lang.String)", - "public void logApplicationPackage(java.util.logging.Level, java.lang.String)" - ], - "fields" : [ ] - }, - "com.yahoo.config.application.api.DeploymentInstanceSpec" : { - "superClass" : "com.yahoo.config.application.api.DeploymentSpec$Steps", - "interfaces" : [ ], - "attributes" : [ - "public" - ], - "methods" : [ - "public void <init>(com.yahoo.config.provision.InstanceName, com.yahoo.config.provision.Tags, java.util.List, com.yahoo.config.application.api.DeploymentSpec$UpgradePolicy, com.yahoo.config.application.api.DeploymentSpec$RevisionTarget, com.yahoo.config.application.api.DeploymentSpec$RevisionChange, com.yahoo.config.application.api.DeploymentSpec$UpgradeRollout, int, int, int, java.util.List, java.util.Optional, java.util.Optional, java.util.Optional, com.yahoo.config.application.api.Notifications, java.util.List, java.time.Instant)", - "public com.yahoo.config.provision.InstanceName name()", - "public com.yahoo.config.provision.Tags tags()", - "public com.yahoo.config.application.api.DeploymentSpec$UpgradePolicy upgradePolicy()", - "public com.yahoo.config.application.api.DeploymentSpec$RevisionTarget revisionTarget()", - "public com.yahoo.config.application.api.DeploymentSpec$RevisionChange revisionChange()", - "public com.yahoo.config.application.api.DeploymentSpec$UpgradeRollout upgradeRollout()", - "public int minRisk()", - "public int maxRisk()", - "public int maxIdleHours()", - "public java.util.List changeBlocker()", - "public java.util.Optional globalServiceId()", - "public boolean canUpgradeAt(java.time.Instant)", - "public boolean canChangeRevisionAt(java.time.Instant)", - "public java.util.Optional athenzService(com.yahoo.config.provision.Environment, com.yahoo.config.provision.RegionName)", - "public java.util.Optional cloudAccount(com.yahoo.config.provision.Environment, java.util.Optional)", - "public com.yahoo.config.application.api.Notifications notifications()", - "public java.util.List endpoints()", - "public boolean deploysTo(com.yahoo.config.provision.Environment, com.yahoo.config.provision.RegionName)", - "public boolean equals(java.lang.Object)", - "public int hashCode()", - "public java.lang.String toString()" - ], - "fields" : [ ] - }, - "com.yahoo.config.application.api.DeploymentSpec$ChangeBlocker" : { - "superClass" : "java.lang.Object", - "interfaces" : [ ], - "attributes" : [ - "public" - ], - "methods" : [ - "public void <init>(boolean, boolean, com.yahoo.config.application.api.TimeWindow)", - "public boolean blocksRevisions()", - "public boolean blocksVersions()", - "public com.yahoo.config.application.api.TimeWindow window()", - "public java.lang.String toString()" - ], - "fields" : [ ] - }, - "com.yahoo.config.application.api.DeploymentSpec$DeclaredTest" : { - "superClass" : "com.yahoo.config.application.api.DeploymentSpec$Step", - "interfaces" : [ ], - "attributes" : [ - "public" - ], - "methods" : [ - "public void <init>(com.yahoo.config.provision.RegionName)", - "public boolean concerns(com.yahoo.config.provision.Environment, java.util.Optional)", - "public boolean isTest()", - "public com.yahoo.config.provision.RegionName region()", - "public boolean equals(java.lang.Object)", - "public int hashCode()", - "public java.lang.String toString()" - ], - "fields" : [ ] - }, - "com.yahoo.config.application.api.DeploymentSpec$DeclaredZone" : { - "superClass" : "com.yahoo.config.application.api.DeploymentSpec$Step", - "interfaces" : [ ], - "attributes" : [ - "public" - ], - "methods" : [ - "public void <init>(com.yahoo.config.provision.Environment)", - "public void <init>(com.yahoo.config.provision.Environment, java.util.Optional, boolean, java.util.Optional, java.util.Optional, java.util.Optional)", - "public com.yahoo.config.provision.Environment environment()", - "public java.util.Optional region()", - "public boolean active()", - "public java.util.Optional testerFlavor()", - "public java.util.Optional athenzService()", - "public java.util.Optional cloudAccount()", - "public java.util.List zones()", - "public boolean concerns(com.yahoo.config.provision.Environment, java.util.Optional)", - "public boolean isTest()", - "public int hashCode()", - "public boolean equals(java.lang.Object)", - "public java.lang.String toString()" - ], - "fields" : [ ] - }, - "com.yahoo.config.application.api.DeploymentSpec$Delay" : { - "superClass" : "com.yahoo.config.application.api.DeploymentSpec$Step", - "interfaces" : [ ], - "attributes" : [ - "public" - ], - "methods" : [ - "public void <init>(java.time.Duration)", - "public java.time.Duration delay()", - "public boolean concerns(com.yahoo.config.provision.Environment, java.util.Optional)", - "public java.lang.String toString()" - ], - "fields" : [ ] - }, - "com.yahoo.config.application.api.DeploymentSpec$DeprecatedElement" : { - "superClass" : "java.lang.Object", - "interfaces" : [ ], - "attributes" : [ - "public" - ], - "methods" : [ - "public void <init>(int, java.lang.String, java.util.List, java.lang.String)", - "public int majorVersion()", - "public java.lang.String humanReadableString()", - "public java.lang.String toString()" - ], - "fields" : [ ] - }, - "com.yahoo.config.application.api.DeploymentSpec$ParallelSteps" : { - "superClass" : "com.yahoo.config.application.api.DeploymentSpec$Steps", - "interfaces" : [ ], - "attributes" : [ - "public" - ], - "methods" : [ - "public void <init>(java.util.List)", - "public java.time.Duration delay()", - "public boolean isOrdered()", - "public boolean equals(java.lang.Object)", - "public int hashCode()", - "public java.lang.String toString()" - ], - "fields" : [ ] - }, - "com.yahoo.config.application.api.DeploymentSpec$RevisionChange" : { - "superClass" : "java.lang.Enum", - "interfaces" : [ ], - "attributes" : [ - "public", - "final", - "enum" - ], - "methods" : [ - "public static com.yahoo.config.application.api.DeploymentSpec$RevisionChange[] values()", - "public static com.yahoo.config.application.api.DeploymentSpec$RevisionChange valueOf(java.lang.String)" - ], - "fields" : [ - "public static final enum com.yahoo.config.application.api.DeploymentSpec$RevisionChange whenClear", - "public static final enum com.yahoo.config.application.api.DeploymentSpec$RevisionChange whenFailing", - "public static final enum com.yahoo.config.application.api.DeploymentSpec$RevisionChange always" - ] - }, - "com.yahoo.config.application.api.DeploymentSpec$RevisionTarget" : { - "superClass" : "java.lang.Enum", - "interfaces" : [ ], - "attributes" : [ - "public", - "final", - "enum" - ], - "methods" : [ - "public static com.yahoo.config.application.api.DeploymentSpec$RevisionTarget[] values()", - "public static com.yahoo.config.application.api.DeploymentSpec$RevisionTarget valueOf(java.lang.String)" - ], - "fields" : [ - "public static final enum com.yahoo.config.application.api.DeploymentSpec$RevisionTarget next", - "public static final enum com.yahoo.config.application.api.DeploymentSpec$RevisionTarget latest" - ] - }, - "com.yahoo.config.application.api.DeploymentSpec$Step" : { - "superClass" : "java.lang.Object", - "interfaces" : [ ], - "attributes" : [ - "public", - "abstract" - ], - "methods" : [ - "public void <init>()", - "public final boolean concerns(com.yahoo.config.provision.Environment)", - "public abstract boolean concerns(com.yahoo.config.provision.Environment, java.util.Optional)", - "public java.util.List zones()", - "public java.time.Duration delay()", - "public java.util.List steps()", - "public boolean isTest()", - "public boolean isOrdered()" - ], - "fields" : [ ] - }, - "com.yahoo.config.application.api.DeploymentSpec$Steps" : { - "superClass" : "com.yahoo.config.application.api.DeploymentSpec$Step", - "interfaces" : [ ], - "attributes" : [ - "public" - ], - "methods" : [ - "public void <init>(java.util.List)", - "public java.util.List zones()", - "public java.util.List steps()", - "public boolean concerns(com.yahoo.config.provision.Environment, java.util.Optional)", - "public java.time.Duration delay()", - "public boolean equals(java.lang.Object)", - "public int hashCode()", - "public java.lang.String toString()" - ], - "fields" : [ ] - }, - "com.yahoo.config.application.api.DeploymentSpec$UpgradePolicy" : { - "superClass" : "java.lang.Enum", - "interfaces" : [ ], - "attributes" : [ - "public", - "final", - "enum" - ], - "methods" : [ - "public static com.yahoo.config.application.api.DeploymentSpec$UpgradePolicy[] values()", - "public static com.yahoo.config.application.api.DeploymentSpec$UpgradePolicy valueOf(java.lang.String)" - ], - "fields" : [ - "public static final enum com.yahoo.config.application.api.DeploymentSpec$UpgradePolicy canary", - "public static final enum com.yahoo.config.application.api.DeploymentSpec$UpgradePolicy defaultPolicy", - "public static final enum com.yahoo.config.application.api.DeploymentSpec$UpgradePolicy conservative" - ] - }, - "com.yahoo.config.application.api.DeploymentSpec$UpgradeRollout" : { - "superClass" : "java.lang.Enum", - "interfaces" : [ ], - "attributes" : [ - "public", - "final", - "enum" - ], - "methods" : [ - "public static com.yahoo.config.application.api.DeploymentSpec$UpgradeRollout[] values()", - "public static com.yahoo.config.application.api.DeploymentSpec$UpgradeRollout valueOf(java.lang.String)" - ], - "fields" : [ - "public static final enum com.yahoo.config.application.api.DeploymentSpec$UpgradeRollout separate", - "public static final enum com.yahoo.config.application.api.DeploymentSpec$UpgradeRollout leading", - "public static final enum com.yahoo.config.application.api.DeploymentSpec$UpgradeRollout simultaneous" - ] - }, - "com.yahoo.config.application.api.DeploymentSpec" : { - "superClass" : "java.lang.Object", - "interfaces" : [ ], - "attributes" : [ - "public" - ], - "methods" : [ - "public void <init>(java.util.List, java.util.Optional, java.util.Optional, java.util.Optional, java.util.Optional, java.util.List, java.lang.String, java.util.List)", - "public java.util.Optional majorVersion()", - "public java.util.List steps()", - "public java.util.Optional athenzDomain()", - "public java.util.Optional athenzService()", - "public java.util.Optional cloudAccount()", - "public java.lang.String xmlForm()", - "public java.util.Optional instance(com.yahoo.config.provision.InstanceName)", - "public com.yahoo.config.application.api.DeploymentInstanceSpec requireInstance(java.lang.String)", - "public com.yahoo.config.application.api.DeploymentInstanceSpec requireInstance(com.yahoo.config.provision.InstanceName)", - "public java.util.List instanceNames()", - "public java.util.List instances()", - "public java.util.List endpoints()", - "public java.util.List deprecatedElements()", - "public static com.yahoo.config.application.api.DeploymentSpec fromXml(java.io.Reader)", - "public static com.yahoo.config.application.api.DeploymentSpec fromXml(java.lang.String)", - "public static com.yahoo.config.application.api.DeploymentSpec fromXml(java.lang.String, boolean)", - "public static java.lang.String toMessageString(java.lang.Throwable)", - "public boolean equals(java.lang.Object)", - "public int hashCode()", - "public int deployableHashCode()" - ], - "fields" : [ - "public static final com.yahoo.config.application.api.DeploymentSpec empty" - ] - }, - "com.yahoo.config.application.api.Endpoint$Level" : { - "superClass" : "java.lang.Enum", - "interfaces" : [ ], - "attributes" : [ - "public", - "final", - "enum" - ], - "methods" : [ - "public static com.yahoo.config.application.api.Endpoint$Level[] values()", - "public static com.yahoo.config.application.api.Endpoint$Level valueOf(java.lang.String)" - ], - "fields" : [ - "public static final enum com.yahoo.config.application.api.Endpoint$Level application", - "public static final enum com.yahoo.config.application.api.Endpoint$Level instance" - ] - }, - "com.yahoo.config.application.api.Endpoint$Target" : { - "superClass" : "java.lang.Object", - "interfaces" : [ ], - "attributes" : [ - "public" - ], - "methods" : [ - "public void <init>(com.yahoo.config.provision.RegionName, com.yahoo.config.provision.InstanceName, int)", - "public com.yahoo.config.provision.RegionName region()", - "public com.yahoo.config.provision.InstanceName instance()", - "public int weight()", - "public java.lang.String toString()", - "public boolean equals(java.lang.Object)", - "public int hashCode()" - ], - "fields" : [ ] - }, - "com.yahoo.config.application.api.Endpoint" : { - "superClass" : "java.lang.Object", - "interfaces" : [ ], - "attributes" : [ - "public" - ], - "methods" : [ - "public void <init>(java.lang.String, java.lang.String, com.yahoo.config.application.api.Endpoint$Level, java.util.List)", - "public java.lang.String endpointId()", - "public java.lang.String containerId()", - "public java.util.List regions()", - "public com.yahoo.config.application.api.Endpoint$Level level()", - "public java.util.List targets()", - "public com.yahoo.config.application.api.Endpoint withTargets(java.util.List)", - "public boolean equals(java.lang.Object)", - "public int hashCode()", - "public java.lang.String toString()" - ], - "fields" : [ - "public static final java.lang.String DEFAULT_ID" - ] - }, - "com.yahoo.config.application.api.FileRegistry$Entry" : { - "superClass" : "java.lang.Object", - "interfaces" : [ ], - "attributes" : [ - "public" - ], - "methods" : [ - "public void <init>(java.lang.String, com.yahoo.config.FileReference)" - ], - "fields" : [ - "public final java.lang.String relativePath", - "public final com.yahoo.config.FileReference reference" - ] - }, - "com.yahoo.config.application.api.FileRegistry" : { - "superClass" : "java.lang.Object", - "interfaces" : [ ], - "attributes" : [ - "public", - "interface", - "abstract" - ], - "methods" : [ - "public abstract com.yahoo.config.FileReference addFile(java.lang.String)", - "public abstract com.yahoo.config.FileReference addUri(java.lang.String)", - "public abstract com.yahoo.config.FileReference addBlob(java.lang.String, java.nio.ByteBuffer)", - "public com.yahoo.config.FileReference addApplicationPackage()", - "public abstract java.util.List export()" - ], - "fields" : [ ] - }, - "com.yahoo.config.application.api.Notifications$Role" : { - "superClass" : "java.lang.Enum", - "interfaces" : [ ], - "attributes" : [ - "public", - "final", - "enum" - ], - "methods" : [ - "public static com.yahoo.config.application.api.Notifications$Role[] values()", - "public static com.yahoo.config.application.api.Notifications$Role valueOf(java.lang.String)", - "public static java.lang.String toValue(com.yahoo.config.application.api.Notifications$Role)", - "public static com.yahoo.config.application.api.Notifications$Role fromValue(java.lang.String)" - ], - "fields" : [ - "public static final enum com.yahoo.config.application.api.Notifications$Role author" - ] - }, - "com.yahoo.config.application.api.Notifications$When" : { - "superClass" : "java.lang.Enum", - "interfaces" : [ ], - "attributes" : [ - "public", - "final", - "enum" - ], - "methods" : [ - "public static com.yahoo.config.application.api.Notifications$When[] values()", - "public static com.yahoo.config.application.api.Notifications$When valueOf(java.lang.String)", - "public static java.lang.String toValue(com.yahoo.config.application.api.Notifications$When)", - "public static com.yahoo.config.application.api.Notifications$When fromValue(java.lang.String)" - ], - "fields" : [ - "public static final enum com.yahoo.config.application.api.Notifications$When failing", - "public static final enum com.yahoo.config.application.api.Notifications$When failingCommit" - ] - }, - "com.yahoo.config.application.api.Notifications" : { - "superClass" : "java.lang.Object", - "interfaces" : [ ], - "attributes" : [ - "public" - ], - "methods" : [ - "public static com.yahoo.config.application.api.Notifications none()", - "public static com.yahoo.config.application.api.Notifications of(java.util.Map, java.util.Map)", - "public java.util.Set emailAddressesFor(com.yahoo.config.application.api.Notifications$When)", - "public java.util.Set emailRolesFor(com.yahoo.config.application.api.Notifications$When)" - ], - "fields" : [ ] - }, - "com.yahoo.config.application.api.TimeWindow$LocalDateRange" : { - "superClass" : "java.lang.Object", - "interfaces" : [ ], - "attributes" : [ - "public" - ], - "methods" : [ - "public java.util.Optional start()", - "public java.util.Optional end()", - "public java.lang.String toString()" - ], - "fields" : [ ] - }, - "com.yahoo.config.application.api.TimeWindow" : { - "superClass" : "java.lang.Object", - "interfaces" : [ ], - "attributes" : [ - "public" - ], - "methods" : [ - "public java.util.List days()", - "public java.util.List hours()", - "public java.time.ZoneId zone()", - "public com.yahoo.config.application.api.TimeWindow$LocalDateRange dateRange()", - "public boolean includes(java.time.Instant)", - "public java.lang.String toString()", - "public static com.yahoo.config.application.api.TimeWindow from(java.lang.String, java.lang.String, java.lang.String, java.lang.String, java.lang.String)" - ], - "fields" : [ ] - }, - "com.yahoo.config.application.api.UnparsedConfigDefinition" : { - "superClass" : "java.lang.Object", - "interfaces" : [ ], - "attributes" : [ - "public", - "interface", - "abstract" - ], - "methods" : [ - "public abstract com.yahoo.vespa.config.ConfigDefinition parse()", - "public abstract java.lang.String getUnparsedContent()" - ], - "fields" : [ ] - }, - "com.yahoo.config.application.api.ValidationId" : { - "superClass" : "java.lang.Enum", - "interfaces" : [ ], - "attributes" : [ - "public", - "final", - "enum" - ], - "methods" : [ - "public static com.yahoo.config.application.api.ValidationId[] values()", - "public static com.yahoo.config.application.api.ValidationId valueOf(java.lang.String)", - "public java.lang.String value()", - "public java.lang.String toString()", - "public static java.util.Optional from(java.lang.String)" - ], - "fields" : [ - "public static final enum com.yahoo.config.application.api.ValidationId indexingChange", - "public static final enum com.yahoo.config.application.api.ValidationId indexModeChange", - "public static final enum com.yahoo.config.application.api.ValidationId fieldTypeChange", - "public static final enum com.yahoo.config.application.api.ValidationId clusterSizeReduction", - "public static final enum com.yahoo.config.application.api.ValidationId tensorTypeChange", - "public static final enum com.yahoo.config.application.api.ValidationId resourcesReduction", - "public static final enum com.yahoo.config.application.api.ValidationId contentTypeRemoval", - "public static final enum com.yahoo.config.application.api.ValidationId contentClusterRemoval", - "public static final enum com.yahoo.config.application.api.ValidationId deploymentRemoval", - "public static final enum com.yahoo.config.application.api.ValidationId globalDocumentChange", - "public static final enum com.yahoo.config.application.api.ValidationId configModelVersionMismatch", - "public static final enum com.yahoo.config.application.api.ValidationId skipOldConfigModels", - "public static final enum com.yahoo.config.application.api.ValidationId accessControl", - "public static final enum com.yahoo.config.application.api.ValidationId globalEndpointChange", - "public static final enum com.yahoo.config.application.api.ValidationId redundancyIncrease", - "public static final enum com.yahoo.config.application.api.ValidationId redundancyOne", - "public static final enum com.yahoo.config.application.api.ValidationId pagedSettingRemoval", - "public static final enum com.yahoo.config.application.api.ValidationId certificateRemoval" - ] - }, - "com.yahoo.config.application.api.ValidationOverrides$Allow" : { - "superClass" : "java.lang.Object", - "interfaces" : [ ], - "attributes" : [ - "public" - ], - "methods" : [ - "public void <init>(com.yahoo.config.application.api.ValidationId, java.time.Instant)", - "public boolean allows(com.yahoo.config.application.api.ValidationId, java.time.Instant)", - "public java.lang.String toString()" - ], - "fields" : [ ] - }, - "com.yahoo.config.application.api.ValidationOverrides$ValidationException" : { - "superClass" : "java.lang.IllegalArgumentException", - "interfaces" : [ ], - "attributes" : [ - "public" - ], - "methods" : [ ], - "fields" : [ ] - }, - "com.yahoo.config.application.api.ValidationOverrides" : { - "superClass" : "java.lang.Object", - "interfaces" : [ ], - "attributes" : [ - "public" - ], - "methods" : [ - "public void <init>(java.util.List)", - "public void invalid(java.util.Map, java.time.Instant)", - "public void invalid(com.yahoo.config.application.api.ValidationId, java.lang.String, java.time.Instant)", - "public boolean allows(java.lang.String, java.time.Instant)", - "public boolean allows(com.yahoo.config.application.api.ValidationId, java.time.Instant)", - "public boolean validate(java.time.Instant)", - "public java.lang.String xmlForm()", - "public static java.lang.String toAllowMessage(com.yahoo.config.application.api.ValidationId)", - "public static com.yahoo.config.application.api.ValidationOverrides fromXml(java.io.Reader)", - "public static com.yahoo.config.application.api.ValidationOverrides fromXml(java.lang.String)" - ], - "fields" : [ - "public static final com.yahoo.config.application.api.ValidationOverrides empty" - ] - }, - "com.yahoo.config.application.api.xml.DeploymentSpecXmlReader" : { - "superClass" : "java.lang.Object", - "interfaces" : [ ], - "attributes" : [ - "public" - ], - "methods" : [ - "public void <init>(boolean, java.time.Clock)", - "public void <init>()", - "public void <init>(boolean)", - "public com.yahoo.config.application.api.DeploymentSpec read(java.io.Reader)", - "public com.yahoo.config.application.api.DeploymentSpec read(java.lang.String)" - ], - "fields" : [ ] - } -} diff --git a/config-model-api/pom.xml b/config-model-api/pom.xml index 1d28965fa3c..5053e4b3472 100644 --- a/config-model-api/pom.xml +++ b/config-model-api/pom.xml @@ -111,10 +111,6 @@ </execution> </executions> </plugin> - <plugin> - <groupId>com.yahoo.vespa</groupId> - <artifactId>abi-check-plugin</artifactId> - </plugin> </plugins> </build> </project> diff --git a/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationMetaData.java b/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationMetaData.java index c830c2baa1f..18d3b81197e 100644 --- a/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationMetaData.java +++ b/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationMetaData.java @@ -10,9 +10,6 @@ import com.yahoo.slime.SlimeUtils; import com.yahoo.text.Utf8; import java.io.IOException; -import java.util.Arrays; -import java.util.Set; -import java.util.stream.Collectors; /** * Metadata about an application package. @@ -25,28 +22,25 @@ public class ApplicationMetaData { private final long deployTimestamp; private final boolean internalRedeploy; private final ApplicationId applicationId; - private final Tags tags; private final String checksum; private final long generation; private final long previousActiveGeneration; public ApplicationMetaData(String deployedFromDir, Long deployTimestamp, boolean internalRedeploy, - ApplicationId applicationId, Tags tags, - String checksum, Long generation, long previousActiveGeneration) { + ApplicationId applicationId, String checksum, Long generation, long previousActiveGeneration) { this.deployedFromDir = deployedFromDir; this.deployTimestamp = deployTimestamp; this.internalRedeploy = internalRedeploy; this.applicationId = applicationId; - this.tags = tags; this.checksum = checksum; this.generation = generation; this.previousActiveGeneration = previousActiveGeneration; } @Deprecated // TODO: Remove on Vespa 9 - public ApplicationMetaData(String deployedFromDir, Long deployTimestamp, boolean internalRedeploy, - ApplicationId applicationId, String checksum, Long generation, long previousActiveGeneration) { - this(deployedFromDir, deployTimestamp, internalRedeploy, applicationId, Tags.empty(), checksum, generation, previousActiveGeneration); + public ApplicationMetaData(String deployedFromDir, Long deployTimestamp, boolean internalRedeploy, ApplicationId applicationId, + Tags ignored, String checksum, Long generation, long previousActiveGeneration) { + this(deployedFromDir, deployTimestamp, internalRedeploy, applicationId, checksum, generation, previousActiveGeneration); } @Deprecated // TODO: Remove on Vespa 9 @@ -63,6 +57,9 @@ public class ApplicationMetaData { @Deprecated // TODO: Remove in Vespa 9 public String getDeployedByUser() { return "unknown"; } + @Deprecated // TODO: Remove in Vespa 9 + public Tags getTags() { return Tags.empty(); } + /** * Gets the directory where the application was deployed from. * Will return null if a problem occurred while getting metadata @@ -73,8 +70,6 @@ public class ApplicationMetaData { public ApplicationId getApplicationId() { return applicationId; } - public Tags getTags() { return tags; } - /** * Gets the time the application was deployed. * Will return null if a problem occurred while getting metadata. @@ -95,7 +90,7 @@ public class ApplicationMetaData { */ public boolean isInternalRedeploy() { return internalRedeploy; } - /** Returns an md5 hash of the contents of the application package */ + /** Returns an MD5 hash of the contents of the application package */ public String getChecksum() { return checksum; } /** Returns the previously active generation at the point when this application was created. */ @@ -117,7 +112,6 @@ public class ApplicationMetaData { deploy.field("timestamp").asLong(), booleanField("internalRedeploy", false, deploy), ApplicationId.fromSerializedForm(app.field("id").asString()), - Tags.fromString(deploy.field("tags").asString()), app.field("checksum").asString(), app.field("generation").asLong(), app.field("previousActiveGeneration").asLong()); @@ -133,7 +127,6 @@ public class ApplicationMetaData { deploy.setString("from", deployedFromDir); deploy.setLong("timestamp", deployTimestamp); deploy.setBool("internalRedeploy", internalRedeploy); - deploy.setString("tags", tags.asString()); Cursor app = meta.setObject("application"); app.setString("id", applicationId.serializedForm()); app.setString("checksum", checksum); diff --git a/config-model-api/src/main/java/com/yahoo/config/application/api/package-info.java b/config-model-api/src/main/java/com/yahoo/config/application/api/package-info.java index 32536ebb778..c1f7d4bd844 100644 --- a/config-model-api/src/main/java/com/yahoo/config/application/api/package-info.java +++ b/config-model-api/src/main/java/com/yahoo/config/application/api/package-info.java @@ -1,7 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. @ExportPackage -@PublicApi package com.yahoo.config.application.api; -import com.yahoo.api.annotations.PublicApi; import com.yahoo.osgi.annotation.ExportPackage; diff --git a/config-model-api/src/main/java/com/yahoo/config/application/api/xml/package-info.java b/config-model-api/src/main/java/com/yahoo/config/application/api/xml/package-info.java index a88368469c1..5ec2ebfa8f9 100644 --- a/config-model-api/src/main/java/com/yahoo/config/application/api/xml/package-info.java +++ b/config-model-api/src/main/java/com/yahoo/config/application/api/xml/package-info.java @@ -1,7 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. @ExportPackage -@PublicApi package com.yahoo.config.application.api.xml; -import com.yahoo.api.annotations.PublicApi; import com.yahoo.osgi.annotation.ExportPackage; diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/package-info.java b/config-model-api/src/main/java/com/yahoo/config/model/api/package-info.java index 698fd1a7dbb..9560a9658d5 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/package-info.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/package-info.java @@ -3,4 +3,3 @@ package com.yahoo.config.model.api; import com.yahoo.osgi.annotation.ExportPackage; -import com.yahoo.api.annotations.PublicApi; diff --git a/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java b/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java index 2d98c8b35c0..9ee279c68d3 100644 --- a/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java +++ b/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java @@ -10,7 +10,6 @@ import com.yahoo.config.application.api.UnparsedConfigDefinition; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.InstanceName; -import com.yahoo.config.provision.Tags; import com.yahoo.config.provision.TenantName; import com.yahoo.io.IOUtils; import com.yahoo.io.reader.NamedReader; @@ -86,7 +85,6 @@ public class MockApplicationPackage implements ApplicationPackage { ApplicationId.from(TenantName.defaultName(), ApplicationName.from(APPLICATION_NAME), InstanceName.defaultName()), - Tags.empty(), "checksum", APPLICATION_GENERATION, 0L); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java index 9be0307e41d..db28ca0e172 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java @@ -122,17 +122,17 @@ public class VespaMetricSet { metrics.add(new Metric("handled.latency.sum")); metrics.add(new Metric("handled.latency.count")); - metrics.add(new Metric("serverRejectedRequests.rate")); - metrics.add(new Metric("serverRejectedRequests.count")); + metrics.add(new Metric("serverRejectedRequests.rate")); // TODO: Remove on Vespa 9. Use jdisc.thread_pool.rejected_tasks. + metrics.add(new Metric("serverRejectedRequests.count")); // TODO: Remove on Vespa 9. Use jdisc.thread_pool.rejected_tasks. - metrics.add(new Metric("serverThreadPoolSize.max")); - metrics.add(new Metric("serverThreadPoolSize.last")); + metrics.add(new Metric("serverThreadPoolSize.max")); // TODO: Remove on Vespa 9. Use jdisc.thread_pool.size. + metrics.add(new Metric("serverThreadPoolSize.last")); // TODO: Remove on Vespa 9. Use jdisc.thread_pool.size. - metrics.add(new Metric("serverActiveThreads.min")); - metrics.add(new Metric("serverActiveThreads.max")); - metrics.add(new Metric("serverActiveThreads.sum")); - metrics.add(new Metric("serverActiveThreads.count")); - metrics.add(new Metric("serverActiveThreads.last")); + metrics.add(new Metric("serverActiveThreads.min")); // TODO: Remove on Vespa 9. Use jdisc.thread_pool.active_threads. + metrics.add(new Metric("serverActiveThreads.max")); // TODO: Remove on Vespa 9. Use jdisc.thread_pool.active_threads. + metrics.add(new Metric("serverActiveThreads.sum")); // TODO: Remove on Vespa 9. Use jdisc.thread_pool.active_threads. + metrics.add(new Metric("serverActiveThreads.count")); // TODO: Remove on Vespa 9. Use jdisc.thread_pool.active_threads. + metrics.add(new Metric("serverActiveThreads.last")); // TODO: Remove on Vespa 9. Use jdisc.thread_pool.active_threads. metrics.add(new Metric("serverNumOpenConnections.average")); metrics.add(new Metric("serverNumOpenConnections.max")); diff --git a/config-model/src/test/java/com/yahoo/config/model/ApplicationDeployTest.java b/config-model/src/test/java/com/yahoo/config/model/ApplicationDeployTest.java index 6507341670f..47e1b35291d 100644 --- a/config-model/src/test/java/com/yahoo/config/model/ApplicationDeployTest.java +++ b/config-model/src/test/java/com/yahoo/config/model/ApplicationDeployTest.java @@ -10,7 +10,6 @@ import com.yahoo.config.model.application.provider.DeployData; import com.yahoo.config.model.application.provider.FilesApplicationPackage; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.Tags; import com.yahoo.document.DataType; import com.yahoo.io.IOUtils; import com.yahoo.path.Path; @@ -227,7 +226,6 @@ public class ApplicationDeployTest { ApplicationId applicationId = ApplicationId.from("tenant1", "application1", "instance1"); DeployData deployData = new DeployData("bar", applicationId, - Tags.fromString("tag1 tag2"), 13L, false, 1337L, @@ -238,7 +236,6 @@ public class ApplicationDeployTest { ApplicationMetaData meta = newApp.getMetaData(); assertEquals("bar", meta.getDeployPath()); assertEquals(applicationId, meta.getApplicationId()); - assertEquals(Tags.fromString("tag1 tag2"), meta.getTags()); assertEquals(13L, (long) meta.getDeployTimestamp()); assertEquals(1337L, (long) meta.getGeneration()); assertEquals(3L, meta.getPreviousActiveGeneration()); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java index 4dccebb98b9..b11355df689 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java @@ -23,7 +23,6 @@ import com.yahoo.config.provision.InfraDeployer; import com.yahoo.config.provision.Provisioner; import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.SystemName; -import com.yahoo.config.provision.Tags; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.Zone; import com.yahoo.config.provision.exception.ActivationConflictException; @@ -364,7 +363,6 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye private PrepareResult deploy(File applicationDir, PrepareParams prepareParams, DeployHandlerLogger logger) { long sessionId = createSession(prepareParams.getApplicationId(), - prepareParams.tags(), prepareParams.getTimeoutBudget(), applicationDir, logger); @@ -861,21 +859,21 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye return sessionRepository.createSessionFromExisting(fromSession, internalRedeploy, timeoutBudget, deployLogger).getSessionId(); } - public long createSession(ApplicationId applicationId, Tags tags, TimeoutBudget timeoutBudget, InputStream in, + public long createSession(ApplicationId applicationId, TimeoutBudget timeoutBudget, InputStream in, String contentType, DeployLogger logger) { File tempDir = uncheck(() -> Files.createTempDirectory("deploy")).toFile(); long sessionId; try { - sessionId = createSession(applicationId, tags, timeoutBudget, decompressApplication(in, contentType, tempDir), logger); + sessionId = createSession(applicationId, timeoutBudget, decompressApplication(in, contentType, tempDir), logger); } finally { cleanupTempDirectory(tempDir, logger); } return sessionId; } - public long createSession(ApplicationId applicationId, Tags tags, TimeoutBudget timeoutBudget, File applicationDirectory, DeployLogger deployLogger) { + public long createSession(ApplicationId applicationId, TimeoutBudget timeoutBudget, File applicationDirectory, DeployLogger deployLogger) { SessionRepository sessionRepository = getTenant(applicationId).getSessionRepository(); - Session session = sessionRepository.createSessionFromApplicationPackage(applicationDirectory, applicationId, tags, timeoutBudget, deployLogger); + Session session = sessionRepository.createSessionFromApplicationPackage(applicationDirectory, applicationId, timeoutBudget, deployLogger); return session.getSessionId(); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionCreateHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionCreateHandler.java index 71702e2926c..72f22aff7e2 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionCreateHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionCreateHandler.java @@ -6,7 +6,6 @@ import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.InstanceName; -import com.yahoo.config.provision.Tags; import com.yahoo.config.provision.TenantName; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; @@ -64,7 +63,6 @@ public class SessionCreateHandler extends SessionHandler { // TODO: Avoid using application id here at all ApplicationId applicationId = ApplicationId.from(tenantName, ApplicationName.defaultName(), InstanceName.defaultName()); sessionId = applicationRepository.createSession(applicationId, - Tags.empty(), timeoutBudget, request.getData(), request.getHeader(ApplicationApiHandler.contentTypeHeader), diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/PrepareParams.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/PrepareParams.java index a7a22393771..239026249e5 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/PrepareParams.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/PrepareParams.java @@ -10,7 +10,6 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.AthenzDomain; import com.yahoo.config.provision.CloudAccount; import com.yahoo.config.provision.DockerImage; -import com.yahoo.config.provision.Tags; import com.yahoo.config.provision.TenantName; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.security.X509CertificateUtils; @@ -42,7 +41,6 @@ public final class PrepareParams { static final String APPLICATION_NAME_PARAM_NAME = "applicationName"; static final String INSTANCE_PARAM_NAME = "instance"; - static final String TAGS_PARAM_NAME = "tags"; static final String IGNORE_VALIDATION_PARAM_NAME = "ignoreValidationErrors"; static final String DRY_RUN_PARAM_NAME = "dryRun"; static final String VERBOSE_PARAM_NAME = "verbose"; @@ -59,7 +57,6 @@ public final class PrepareParams { static final String CLOUD_ACCOUNT = "cloudAccount"; private final ApplicationId applicationId; - private final Tags tags; private final TimeoutBudget timeoutBudget; private final boolean ignoreValidationErrors; private final boolean dryRun; @@ -78,7 +75,6 @@ public final class PrepareParams { private final Optional<CloudAccount> cloudAccount; private PrepareParams(ApplicationId applicationId, - Tags tags, TimeoutBudget timeoutBudget, boolean ignoreValidationErrors, boolean dryRun, @@ -97,7 +93,6 @@ public final class PrepareParams { Optional<CloudAccount> cloudAccount) { this.timeoutBudget = timeoutBudget; this.applicationId = Objects.requireNonNull(applicationId); - this.tags = tags; this.ignoreValidationErrors = ignoreValidationErrors; this.dryRun = dryRun; this.verbose = verbose; @@ -124,7 +119,6 @@ public final class PrepareParams { private boolean force = false; private boolean waitForResourcesInPrepare = false; private ApplicationId applicationId = null; - private Tags tags = Tags.empty(); private TimeoutBudget timeoutBudget = new TimeoutBudget(Clock.systemUTC(), Duration.ofSeconds(60)); private Optional<Version> vespaVersion = Optional.empty(); private List<ContainerEndpoint> containerEndpoints = null; @@ -143,11 +137,6 @@ public final class PrepareParams { return this; } - public Builder tags(Tags tags) { - this.tags = tags; - return this; - } - public Builder ignoreValidationErrors(boolean ignoreValidationErrors) { this.ignoreValidationErrors = ignoreValidationErrors; return this; @@ -279,7 +268,6 @@ public final class PrepareParams { public PrepareParams build() { return new PrepareParams(applicationId, - tags, timeoutBudget, ignoreValidationErrors, dryRun, @@ -306,7 +294,6 @@ public final class PrepareParams { .verbose(request.getBooleanProperty(VERBOSE_PARAM_NAME)) .timeoutBudget(SessionHandler.getTimeoutBudget(request, barrierTimeout)) .applicationId(createApplicationId(request, tenant)) - .tags(Tags.fromString(request.getProperty(TAGS_PARAM_NAME))) .vespaVersion(request.getProperty(VESPA_VERSION_PARAM_NAME)) .containerEndpoints(request.getProperty(CONTAINER_ENDPOINTS_PARAM_NAME)) .endpointCertificateMetadata(request.getProperty(ENDPOINT_CERTIFICATE_METADATA_PARAM_NAME)) @@ -329,7 +316,6 @@ public final class PrepareParams { .verbose(booleanValue(params, VERBOSE_PARAM_NAME)) .timeoutBudget(SessionHandler.getTimeoutBudget(getTimeout(params, barrierTimeout))) .applicationId(createApplicationId(params, tenant)) - .tags(Tags.fromString(params.field(TAGS_PARAM_NAME).asString())) .vespaVersion(SlimeUtils.optionalString(params.field(VESPA_VERSION_PARAM_NAME)).orElse(null)) .containerEndpointList(deserialize(params.field(CONTAINER_ENDPOINTS_PARAM_NAME), ContainerEndpointSerializer::endpointListFromSlime, List.of())) .endpointCertificateMetadata(deserialize(params.field(ENDPOINT_CERTIFICATE_METADATA_PARAM_NAME), EndpointCertificateMetadataSerializer::fromSlime)) @@ -404,8 +390,6 @@ public final class PrepareParams { public ApplicationId getApplicationId() { return applicationId; } - public Tags tags() { return tags; } - /** Returns the Vespa version the nodes running the prepared system should have, or empty to use the system version */ public Optional<Version> vespaVersion() { return vespaVersion; } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/Session.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/Session.java index 835381b316d..94b3bd96620 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/Session.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/Session.java @@ -13,7 +13,6 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.AthenzDomain; import com.yahoo.config.provision.CloudAccount; import com.yahoo.config.provision.DockerImage; -import com.yahoo.config.provision.Tags; import com.yahoo.config.provision.TenantName; import com.yahoo.path.Path; import com.yahoo.transaction.Transaction; @@ -120,10 +119,6 @@ public abstract class Session implements Comparable<Session> { sessionZooKeeperClient.writeApplicationId(applicationId); } - public void setTags(Tags tags) { - sessionZooKeeperClient.writeTags(tags); - } - void setApplicationPackageReference(FileReference applicationPackageReference) { sessionZooKeeperClient.writeApplicationPackageReference(Optional.ofNullable(applicationPackageReference)); } @@ -159,10 +154,6 @@ public abstract class Session implements Comparable<Session> { /** Returns application id read from ZooKeeper. Will throw RuntimeException if not found */ public ApplicationId getApplicationId() { return sessionZooKeeperClient.readApplicationId(); } - public Tags getTags() { - return sessionZooKeeperClient.readTags(); - } - /** Returns application id read from ZooKeeper. Will return Optional.empty() if not found */ public Optional<ApplicationId> getOptionalApplicationId() { try { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java index 7629fdd8bfd..17a019067a3 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java @@ -159,7 +159,6 @@ public class SessionPreparer { final PrepareParams params; final ApplicationId applicationId; - final Tags tags; /** The repository part of docker image to be used for this deployment */ final Optional<DockerImage> dockerImageRepository; @@ -191,7 +190,6 @@ public class SessionPreparer { this.applicationPackage = applicationPackage; this.sessionZooKeeperClient = sessionZooKeeperClient; this.applicationId = params.getApplicationId(); - this.tags = params.tags(); this.dockerImageRepository = params.dockerImageRepository(); this.vespaVersion = params.vespaVersion().orElse(Vtag.currentVersion); this.containerEndpointsCache = new ContainerEndpointsCache(tenantPath, curator); @@ -357,7 +355,6 @@ public class SessionPreparer { writeStateToZooKeeper(sessionZooKeeperClient, preprocessedApplicationPackage, applicationId, - tags, filereference, dockerImageRepository, vespaVersion, @@ -399,7 +396,6 @@ public class SessionPreparer { private void writeStateToZooKeeper(SessionZooKeeperClient zooKeeperClient, ApplicationPackage applicationPackage, ApplicationId applicationId, - Tags tags, FileReference fileReference, Optional<DockerImage> dockerImageRepository, Version vespaVersion, @@ -416,7 +412,6 @@ public class SessionPreparer { zkDeployer.deploy(applicationPackage, fileRegistryMap, allocatedHosts); // Note: When changing the below you need to also change similar calls in SessionRepository.createSessionFromExisting() zooKeeperClient.writeApplicationId(applicationId); - zooKeeperClient.writeTags(tags); zooKeeperClient.writeApplicationPackageReference(Optional.of(fileReference)); zooKeeperClient.writeVespaVersion(vespaVersion); zooKeeperClient.writeDockerImageRepository(dockerImageRepository); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java index d26173b4dd7..306b24a3c69 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java @@ -265,17 +265,14 @@ public class SessionRepository { TimeoutBudget timeoutBudget, DeployLogger deployLogger) { ApplicationId existingApplicationId = existingSession.getApplicationId(); - Tags existingTags = existingSession.getTags(); File existingApp = getSessionAppDir(existingSession.getSessionId()); LocalSession session = createSessionFromApplication(existingApp, existingApplicationId, - existingTags, internalRedeploy, timeoutBudget, deployLogger); // Note: Setters below need to be kept in sync with calls in SessionPreparer.writeStateToZooKeeper() session.setApplicationId(existingApplicationId); - session.setTags(existingTags); session.setApplicationPackageReference(existingSession.getApplicationPackageReference()); session.setVespaVersion(existingSession.getVespaVersion()); session.setDockerImageRepository(existingSession.getDockerImageRepository()); @@ -297,20 +294,19 @@ public class SessionRepository { */ public LocalSession createSessionFromApplicationPackage(File applicationDirectory, ApplicationId applicationId, - Tags tags, TimeoutBudget timeoutBudget, DeployLogger deployLogger) { applicationRepo.createApplication(applicationId); - return createSessionFromApplication(applicationDirectory, applicationId, tags, false, timeoutBudget, deployLogger); + return createSessionFromApplication(applicationDirectory, applicationId, false, timeoutBudget, deployLogger); } /** * Creates a local session based on a remote session and the distributed application package. * Does not wait for session being created on other servers. */ - private void createLocalSession(File applicationFile, ApplicationId applicationId, Tags tags, long sessionId) { + private void createLocalSession(File applicationFile, ApplicationId applicationId, long sessionId) { try { - ApplicationPackage applicationPackage = createApplicationPackage(applicationFile, applicationId, tags, sessionId, false, Optional.empty()); + ApplicationPackage applicationPackage = createApplicationPackage(applicationFile, applicationId, sessionId, false, Optional.empty()); createLocalSession(sessionId, applicationPackage); } catch (Exception e) { throw new RuntimeException("Error creating session " + sessionId, e); @@ -686,13 +682,12 @@ public class SessionRepository { private ApplicationPackage createApplication(File userDir, File configApplicationDir, ApplicationId applicationId, - Tags tags, long sessionId, Optional<Long> currentlyActiveSessionId, boolean internalRedeploy, Optional<DeployLogger> deployLogger) { long deployTimestamp = System.currentTimeMillis(); - DeployData deployData = new DeployData(userDir.getAbsolutePath(), applicationId, tags, deployTimestamp, internalRedeploy, + DeployData deployData = new DeployData(userDir.getAbsolutePath(), applicationId, deployTimestamp, internalRedeploy, sessionId, currentlyActiveSessionId.orElse(nonExistingActiveSessionId)); FilesApplicationPackage app = FilesApplicationPackage.fromFileWithDeployData(configApplicationDir, deployData); validateFileExtensions(applicationId, deployLogger, app); @@ -720,14 +715,13 @@ public class SessionRepository { private LocalSession createSessionFromApplication(File applicationDirectory, ApplicationId applicationId, - Tags tags, boolean internalRedeploy, TimeoutBudget timeoutBudget, DeployLogger deployLogger) { long sessionId = getNextSessionId(); try { ensureSessionPathDoesNotExist(sessionId); - ApplicationPackage app = createApplicationPackage(applicationDirectory, applicationId, tags, sessionId, internalRedeploy, Optional.of(deployLogger)); + ApplicationPackage app = createApplicationPackage(applicationDirectory, applicationId, sessionId, internalRedeploy, Optional.of(deployLogger)); log.log(Level.FINE, () -> TenantRepository.logPre(tenantName) + "Creating session " + sessionId + " in ZooKeeper"); SessionZooKeeperClient sessionZKClient = createSessionZooKeeperClient(sessionId); sessionZKClient.createNewSession(clock.instant()); @@ -743,7 +737,6 @@ public class SessionRepository { private ApplicationPackage createApplicationPackage(File applicationDirectory, ApplicationId applicationId, - Tags tags, long sessionId, boolean internalRedeploy, Optional<DeployLogger> deployLogger) throws IOException { @@ -756,7 +749,6 @@ public class SessionRepository { ApplicationPackage applicationPackage = createApplication(applicationDirectory, userApplicationDir, applicationId, - tags, sessionId, activeSessionId, internalRedeploy, @@ -875,7 +867,7 @@ public class SessionRepository { } ApplicationId applicationId = sessionZKClient.readApplicationId(); log.log(Level.FINE, () -> "Creating local session for tenant '" + tenantName + "' with session id " + sessionId); - createLocalSession(sessionDir, applicationId, sessionZKClient.readTags(), sessionId); + createLocalSession(sessionDir, applicationId, sessionId); } private Optional<Long> getActiveSessionId(ApplicationId applicationId) { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClient.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClient.java index 2353c480a34..4bbb8a63974 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClient.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClient.java @@ -14,7 +14,6 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.AthenzDomain; import com.yahoo.config.provision.CloudAccount; import com.yahoo.config.provision.DockerImage; -import com.yahoo.config.provision.Tags; import com.yahoo.config.provision.TenantName; import com.yahoo.path.Path; import com.yahoo.slime.SlimeUtils; @@ -59,7 +58,6 @@ public class SessionZooKeeperClient { // NOTE: Any state added here MUST also be propagated in com.yahoo.vespa.config.server.deploy.Deployment.prepare() static final String APPLICATION_ID_PATH = "applicationId"; - static final String TAGS_PATH = "tags"; static final String APPLICATION_PACKAGE_REFERENCE_PATH = "applicationPackageReference"; private static final String VERSION_PATH = "version"; private static final String CREATE_TIME_PATH = "createTime"; @@ -175,20 +173,6 @@ public class SessionZooKeeperClient { .orElseThrow(() -> new NotFoundException("Could not find application id for session " + sessionId)); } - private Path tagsPath() { - return sessionPath.append(TAGS_PATH); - } - - public void writeTags(Tags tags) { - curator.set(tagsPath(), Utf8.toBytes(tags.asString())); - } - - public Tags readTags() { - Optional<byte[]> data = curator.getData(tagsPath()); - if (data.isEmpty()) return Tags.empty(); - return Tags.fromString(Utf8.toString(data.get())); - } - void writeApplicationPackageReference(Optional<FileReference> applicationPackageReference) { applicationPackageReference.ifPresent( reference -> curator.set(applicationPackageReferencePath(), Utf8.toBytes(reference.value()))); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java index 8f104a27000..76eb61ed141 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java @@ -17,7 +17,6 @@ import com.yahoo.config.provision.HostFilter; import com.yahoo.config.provision.HostSpec; import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.NetworkPorts; -import com.yahoo.config.provision.Tags; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.exception.ActivationConflictException; import com.yahoo.container.jdisc.HttpResponse; @@ -846,7 +845,7 @@ public class ApplicationRepositoryTest { } private long createSession(ApplicationId applicationId, TimeoutBudget timeoutBudget, File app) { - return applicationRepository.createSession(applicationId, Tags.empty(), timeoutBudget, app, new BaseDeployLogger()); + return applicationRepository.createSession(applicationId, timeoutBudget, app, new BaseDeployLogger()); } private long createSessionFromExisting(ApplicationId applicationId, TimeoutBudget timeoutBudget) { diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClientTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClientTest.java index 653753c97e7..fd6440a9632 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClientTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClientTest.java @@ -13,7 +13,6 @@ import com.yahoo.config.model.application.provider.MockFileRegistry; import com.yahoo.config.provision.AllocatedHosts; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.HostSpec; -import com.yahoo.config.provision.Tags; import com.yahoo.path.Path; import com.yahoo.text.Utf8; import com.yahoo.vespa.config.server.zookeeper.ZKApplicationPackage; @@ -60,7 +59,6 @@ public class ZooKeeperClientTest { ApplicationPackage app = FilesApplicationPackage.fromFileWithDeployData(new File("src/test/apps/zkfeed"), new DeployData("/bar/baz", ApplicationId.from("default", "appName", "default"), - Tags.fromString("tag1 tag2"), 1345L, true, 3L, @@ -123,7 +121,6 @@ public class ZooKeeperClientTest { assertTrue(metaData.getChecksum().length() > 0); assertTrue(metaData.isInternalRedeploy()); assertEquals("/bar/baz", metaData.getDeployPath()); - assertEquals(Tags.fromString("tag1 tag2"), metaData.getTags()); assertEquals(1345, metaData.getDeployTimestamp().longValue()); assertEquals(3, metaData.getGeneration().longValue()); assertEquals(2, metaData.getPreviousActiveGeneration()); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandlerTest.java index 816f7e3dcec..1c71ef0b7fb 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandlerTest.java @@ -6,7 +6,6 @@ import com.yahoo.component.Version; import com.yahoo.config.application.api.ApplicationMetaData; import com.yahoo.config.model.application.provider.BaseDeployLogger; import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.Tags; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.Zone; import com.yahoo.container.jdisc.HttpResponse; @@ -132,7 +131,6 @@ public class SessionActiveHandlerTest { void invoke() { long sessionId = applicationRepository.createSession(applicationId(), - Tags.empty(), new TimeoutBudget(clock, Duration.ofSeconds(10)), testApp, new BaseDeployLogger()); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareHandlerTest.java index 525a969ed1e..de6073bb1ea 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareHandlerTest.java @@ -10,7 +10,6 @@ import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.DockerImage; import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.NodeAllocationException; -import com.yahoo.config.provision.Tags; import com.yahoo.config.provision.TenantName; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.jdisc.http.HttpRequest; @@ -325,7 +324,7 @@ public class SessionPrepareHandlerTest extends SessionHandlerTest { } private long createSession(ApplicationId applicationId) { - return applicationRepository.createSession(applicationId, Tags.empty(), timeoutBudget, app, new BaseDeployLogger()); + return applicationRepository.createSession(applicationId, timeoutBudget, app, new BaseDeployLogger()); } private static class FailingSessionPrepareHandler extends SessionPrepareHandler { diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/session/PrepareParamsTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/session/PrepareParamsTest.java index 82d6f183761..2c898b8bf7d 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/session/PrepareParamsTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/session/PrepareParamsTest.java @@ -7,7 +7,6 @@ import com.yahoo.config.model.api.EndpointCertificateMetadata; import com.yahoo.config.model.api.TenantSecretStore; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.CloudAccount; -import com.yahoo.config.provision.Tags; import com.yahoo.config.provision.TenantName; import com.yahoo.container.jdisc.HttpRequest; @@ -63,7 +62,6 @@ public class PrepareParamsTest { PrepareParams prepareParams = createParams("http://foo:19071/application/v2/", TenantName.defaultName()); assertEquals(ApplicationId.defaultId(), prepareParams.getApplicationId()); - assertTrue(prepareParams.tags().isEmpty()); assertFalse(prepareParams.isDryRun()); assertFalse(prepareParams.isVerbose()); assertFalse(prepareParams.ignoreValidationErrors()); @@ -74,18 +72,6 @@ public class PrepareParamsTest { } @Test - public void testTagsParsing() throws IOException { - var prepareParams = createParams(request + "&" + PrepareParams.TAGS_PARAM_NAME + "=tag1%20tag2", TenantName.from("foo")); - assertEquals(Tags.fromString("tag1 tag2"), prepareParams.tags()); - - // Verify using json object - var slime = SlimeUtils.jsonToSlime(json); - slime.get().setString(PrepareParams.TAGS_PARAM_NAME, "tag1 tag2"); - PrepareParams prepareParamsJson = PrepareParams.fromJson(SlimeUtils.toJsonBytes(slime), TenantName.from("foo"), Duration.ofSeconds(60)); - assertPrepareParamsEqual(prepareParams, prepareParamsJson); - } - - @Test public void testCorrectParsingWithContainerEndpoints() throws IOException { var endpoints = List.of(new ContainerEndpoint("qrs1", ApplicationClusterEndpoint.Scope.global, List.of("c1.example.com", @@ -221,7 +207,6 @@ public class PrepareParamsTest { assertEquals(urlParams.force(), jsonParams.force()); assertEquals(urlParams.waitForResourcesInPrepare(), jsonParams.waitForResourcesInPrepare()); assertEquals(urlParams.getApplicationId(), jsonParams.getApplicationId()); - assertEquals(urlParams.tags(), jsonParams.tags()); assertEquals(urlParams.getTimeoutBudget().timeout(), jsonParams.getTimeoutBudget().timeout()); assertEquals(urlParams.vespaVersion(), jsonParams.vespaVersion()); assertEquals(urlParams.containerEndpoints(), jsonParams.containerEndpoints()); diff --git a/container-core/src/main/java/com/yahoo/osgi/OsgiImpl.java b/container-core/src/main/java/com/yahoo/osgi/OsgiImpl.java index 4acdc3d544b..9d023e2a7ca 100644 --- a/container-core/src/main/java/com/yahoo/osgi/OsgiImpl.java +++ b/container-core/src/main/java/com/yahoo/osgi/OsgiImpl.java @@ -62,6 +62,11 @@ public class OsgiImpl implements Osgi { if (bundle != null) { return resolveFromBundle(spec, bundle); } else { + if (jdiscOsgi.isFelixFramework() && ! spec.bundle.equals(spec.classId)) { + // Bundle was explicitly specified, but not found. + throw new IllegalArgumentException("Could not find bundle " + spec.bundle + " to create a component with class '" + + spec.classId.getName() + ". " + bundleResolutionErrorMessage(spec.bundle)); + } return resolveFromThisBundleOrSystemBundle(spec); } } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/DeploymentData.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/DeploymentData.java index 63c744c385d..5a55768ad2c 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/DeploymentData.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/DeploymentData.java @@ -5,7 +5,6 @@ import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.CloudAccount; import com.yahoo.config.provision.DockerImage; -import com.yahoo.config.provision.Tags; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.athenz.api.AthenzDomain; import com.yahoo.vespa.hosted.controller.api.integration.billing.Quota; @@ -32,7 +31,6 @@ import static java.util.Objects.requireNonNull; public class DeploymentData { private final ApplicationId instance; - private final Tags tags; private final ZoneId zone; private final Supplier<InputStream> applicationPackage; private final Version platform; @@ -46,7 +44,7 @@ public class DeploymentData { private final Supplier<Optional<CloudAccount>> cloudAccount; private final boolean dryRun; - public DeploymentData(ApplicationId instance, Tags tags, ZoneId zone, Supplier<InputStream> applicationPackage, Version platform, + public DeploymentData(ApplicationId instance, ZoneId zone, Supplier<InputStream> applicationPackage, Version platform, Set<ContainerEndpoint> containerEndpoints, Supplier<Optional<EndpointCertificateMetadata>> endpointCertificateMetadata, Optional<DockerImage> dockerImageRepo, @@ -57,7 +55,6 @@ public class DeploymentData { Supplier<Optional<CloudAccount>> cloudAccount, boolean dryRun) { this.instance = requireNonNull(instance); - this.tags = requireNonNull(tags); this.zone = requireNonNull(zone); this.applicationPackage = requireNonNull(applicationPackage); this.platform = requireNonNull(platform); @@ -76,8 +73,6 @@ public class DeploymentData { return instance; } - public Tags tags() { return tags; } - public ZoneId zone() { return zone; } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java index 39d01b16beb..db188ea3e7e 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java @@ -12,7 +12,6 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.CloudAccount; import com.yahoo.config.provision.DockerImage; import com.yahoo.config.provision.InstanceName; -import com.yahoo.config.provision.Tags; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.text.Text; @@ -175,9 +174,7 @@ public class ApplicationController { lockApplicationIfPresent(id, application -> { for (var declaredInstance : application.get().deploymentSpec().instances()) if ( ! application.get().instances().containsKey(declaredInstance.name())) - application = withNewInstance(application, - id.instance(declaredInstance.name()), - declaredInstance.tags()); + application = withNewInstance(application, id.instance(declaredInstance.name())); store(application); }); count++; @@ -459,14 +456,14 @@ public class ApplicationController { * * @throws IllegalArgumentException if the instance already exists, or has an invalid instance name. */ - public void createInstance(ApplicationId id, Tags tags) { + public void createInstance(ApplicationId id) { lockApplicationOrThrow(TenantAndApplicationId.from(id), application -> { - store(withNewInstance(application, id, tags)); + store(withNewInstance(application, id)); }); } /** Returns given application with a new instance */ - public LockedApplication withNewInstance(LockedApplication application, ApplicationId instance, Tags tags) { + public LockedApplication withNewInstance(LockedApplication application, ApplicationId instance) { if (instance.instance().isTester()) throw new IllegalArgumentException("'" + instance + "' is a tester application!"); InstanceId.validate(instance.instance().value()); @@ -477,7 +474,7 @@ public class ApplicationController { throw new IllegalArgumentException("Could not create '" + instance + "': Instance " + dashToUnderscore(instance) + " already exists"); log.info("Created " + instance); - return application.withNewInstance(instance.instance(), tags); + return application.withNewInstance(instance.instance()); } /** Deploys an application package for an existing application instance. */ @@ -520,7 +517,7 @@ public class ApplicationController { }; // Carry out deployment without holding the application lock. - DeploymentResult result = deploy(job.application(), instance.tags(), applicationPackage, zone, platform, containerEndpoints, + DeploymentResult result = deploy(job.application(), applicationPackage, zone, platform, containerEndpoints, endpointCertificateMetadata, run.isDryRun(), run.testerCertificate()); @@ -569,9 +566,7 @@ public class ApplicationController { var declaredInstances = applicationPackage.deploymentSpec().instances(); for (var declaredInstance : declaredInstances) { if ( ! existingInstances.containsKey(declaredInstance.name())) - application = withNewInstance(application, application.get().id().instance(declaredInstance.name()), declaredInstance.tags()); - else if ( ! existingInstances.get(declaredInstance.name()).tags().equals(declaredInstance.tags())) - application = application.with(declaredInstance.name(), instance -> instance.with(declaredInstance.tags())); + application = withNewInstance(application, application.get().id().instance(declaredInstance.name())); } // Delete zones not listed in DeploymentSpec, if allowed @@ -614,7 +609,7 @@ public class ApplicationController { ApplicationPackageStream applicationPackage = new ApplicationPackageStream( () -> new ByteArrayInputStream(artifactRepository.getSystemApplicationPackage(application.id(), zone, version)) ); - return deploy(application.id(), Tags.empty(), applicationPackage, zone, version, Set.of(), Optional::empty, false, Optional.empty()); + return deploy(application.id(), applicationPackage, zone, version, Set.of(), Optional::empty, false, Optional.empty()); } else { throw new RuntimeException("This system application does not have an application package: " + application.id().toShortString()); } @@ -622,10 +617,10 @@ public class ApplicationController { /** Deploys the given tester application to the given zone. */ public DeploymentResult deployTester(TesterId tester, ApplicationPackageStream applicationPackage, ZoneId zone, Version platform) { - return deploy(tester.id(), Tags.empty(), applicationPackage, zone, platform, Set.of(), Optional::empty, false, Optional.empty()); + return deploy(tester.id(), applicationPackage, zone, platform, Set.of(), Optional::empty, false, Optional.empty()); } - private DeploymentResult deploy(ApplicationId application, Tags tags, ApplicationPackageStream applicationPackage, + private DeploymentResult deploy(ApplicationId application, ApplicationPackageStream applicationPackage, ZoneId zone, Version platform, Set<ContainerEndpoint> endpoints, Supplier<Optional<EndpointCertificateMetadata>> endpointCertificateMetadata, boolean dryRun, Optional<X509Certificate> testerCertificate) { @@ -659,7 +654,7 @@ public class ApplicationController { } Supplier<Optional<CloudAccount>> cloudAccount = () -> decideCloudAccountOf(deployment, applicationPackage.truncatedPackage().deploymentSpec()); ConfigServer.PreparedApplication preparedApplication = - configServer.deploy(new DeploymentData(application, tags, zone, applicationPackage::zipStream, platform, + configServer.deploy(new DeploymentData(application, zone, applicationPackage::zipStream, platform, endpoints, endpointCertificateMetadata, dockerImageRepo, domain, deploymentQuota, tenantSecretStores, operatorCertificates, cloudAccount, dryRun)); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Instance.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Instance.java index 430bafe5c44..d66d1491f73 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Instance.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Instance.java @@ -5,7 +5,6 @@ import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.InstanceName; -import com.yahoo.config.provision.Tags; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RevisionId; @@ -41,7 +40,6 @@ import java.util.stream.Collectors; public class Instance { private final ApplicationId id; - private final Tags tags; private final Map<ZoneId, Deployment> deployments; private final List<AssignedRotation> rotations; private final RotationStatus rotationStatus; @@ -49,15 +47,14 @@ public class Instance { private final Change change; /** Creates an empty instance */ - public Instance(ApplicationId id, Tags tags) { - this(id, tags, Set.of(), Map.of(), List.of(), RotationStatus.EMPTY, Change.empty()); + public Instance(ApplicationId id) { + this(id, Set.of(), Map.of(), List.of(), RotationStatus.EMPTY, Change.empty()); } /** Creates an empty instance*/ - public Instance(ApplicationId id, Tags tags, Collection<Deployment> deployments, Map<JobType, Instant> jobPauses, + public Instance(ApplicationId id, Collection<Deployment> deployments, Map<JobType, Instant> jobPauses, List<AssignedRotation> rotations, RotationStatus rotationStatus, Change change) { this.id = Objects.requireNonNull(id, "id cannot be null"); - this.tags = Objects.requireNonNull(tags, "tags cannot be null"); this.deployments = Objects.requireNonNull(deployments, "deployments cannot be null").stream() .collect(Collectors.toUnmodifiableMap(Deployment::zone, Function.identity())); this.jobPauses = Map.copyOf(Objects.requireNonNull(jobPauses, "deploymentJobs cannot be null")); @@ -66,10 +63,6 @@ public class Instance { this.change = Objects.requireNonNull(change, "change cannot be null"); } - public Instance with(Tags tags) { - return new Instance(id, tags, deployments.values(), jobPauses, rotations, rotationStatus, change); - } - public Instance withNewDeployment(ZoneId zone, RevisionId revision, Version version, Instant instant, Map<DeploymentMetrics.Warning, Integer> warnings, QuotaUsage quotaUsage) { // Use info from previous deployment if available, otherwise create a new one. @@ -94,7 +87,7 @@ public class Instance { else jobPauses.remove(jobType); - return new Instance(id, tags, deployments.values(), jobPauses, rotations, rotationStatus, change); + return new Instance(id, deployments.values(), jobPauses, rotations, rotationStatus, change); } public Instance recordActivityAt(Instant instant, ZoneId zone) { @@ -125,15 +118,15 @@ public class Instance { } public Instance with(List<AssignedRotation> assignedRotations) { - return new Instance(id, tags, deployments.values(), jobPauses, assignedRotations, rotationStatus, change); + return new Instance(id, deployments.values(), jobPauses, assignedRotations, rotationStatus, change); } public Instance with(RotationStatus rotationStatus) { - return new Instance(id, tags, deployments.values(), jobPauses, rotations, rotationStatus, change); + return new Instance(id, deployments.values(), jobPauses, rotations, rotationStatus, change); } public Instance withChange(Change change) { - return new Instance(id, tags, deployments.values(), jobPauses, rotations, rotationStatus, change); + return new Instance(id, deployments.values(), jobPauses, rotations, rotationStatus, change); } private Instance with(Deployment deployment) { @@ -143,15 +136,13 @@ public class Instance { } private Instance with(Map<ZoneId, Deployment> deployments) { - return new Instance(id, tags, deployments.values(), jobPauses, rotations, rotationStatus, change); + return new Instance(id, deployments.values(), jobPauses, rotations, rotationStatus, change); } public ApplicationId id() { return id; } public InstanceName name() { return id.instance(); } - public Tags tags() { return tags; } - /** Returns an immutable map of the current deployments of this */ public Map<ZoneId, Deployment> deployments() { return deployments; } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java index fa702a166d2..b99c52d1533 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java @@ -4,9 +4,7 @@ package com.yahoo.vespa.hosted.controller; import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.application.api.ValidationOverrides; import com.yahoo.config.provision.InstanceName; -import com.yahoo.config.provision.Tags; import com.yahoo.transaction.Mutex; -import com.yahoo.vespa.curator.Lock; import com.yahoo.vespa.hosted.controller.api.integration.organization.IssueId; import com.yahoo.vespa.hosted.controller.api.integration.organization.User; import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; @@ -89,9 +87,9 @@ public class LockedApplication { projectId, revisions, instances.values()); } - LockedApplication withNewInstance(InstanceName instance, Tags tags) { + LockedApplication withNewInstance(InstanceName instance) { var instances = new HashMap<>(this.instances); - instances.put(instance, new Instance(id.instance(instance), tags)); + instances.put(instance, new Instance(id.instance(instance))); return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, deploymentIssueId, ownershipIssueId, owner, majorVersion, metrics, deployKeys, projectId, instances, revisions); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java index d41f8e6beb5..24bab28c520 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java @@ -8,7 +8,6 @@ import com.yahoo.concurrent.UncheckedTimeoutException; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.SystemName; -import com.yahoo.config.provision.Tags; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.transaction.Mutex; import com.yahoo.vespa.flags.FetchVector.Dimension; @@ -712,7 +711,7 @@ public class JobController { controller.applications().lockApplicationOrThrow(TenantAndApplicationId.from(id), application -> { if ( ! application.get().instances().containsKey(id.instance())) - application = controller.applications().withNewInstance(application, id, Tags.empty()); + application = controller.applications().withNewInstance(application, id); // TODO(mpolden): Enable for public CD once all tests have been updated if (controller.system() != SystemName.PublicCd) { controller.applications().validatePackage(applicationPackage, application.get()); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java index 37c45f38e36..2a9724bb911 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java @@ -8,8 +8,6 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.RegionName; -import com.yahoo.config.provision.SystemName; -import com.yahoo.config.provision.Tags; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.security.KeyUtils; import com.yahoo.slime.ArrayTraverser; @@ -98,7 +96,6 @@ public class ApplicationSerializer { // Instance fields private static final String instanceNameField = "instanceName"; - private static final String tagsField = "tags"; private static final String deploymentsField = "deployments"; private static final String deploymentJobsField = "deploymentJobs"; // TODO jonmv: clean up serialisation format private static final String assignedRotationsField = "assignedRotations"; @@ -186,7 +183,6 @@ public class ApplicationSerializer { for (Instance instance : application.instances().values()) { Cursor instanceObject = array.addObject(); instanceObject.setString(instanceNameField, instance.name().value()); - instanceObject.setString(tagsField, instance.tags().asString()); deploymentsToSlime(instance.deployments().values(), instanceObject.setArray(deploymentsField)); toSlime(instance.jobPauses(), instanceObject.setObject(deploymentJobsField)); assignedRotationsToSlime(instance.rotations(), instanceObject); @@ -383,14 +379,12 @@ public class ApplicationSerializer { List<Instance> instances = new ArrayList<>(); field.traverse((ArrayTraverser) (name, object) -> { InstanceName instanceName = InstanceName.from(object.field(instanceNameField).asString()); - Tags tags = Tags.fromString(object.field(tagsField).asString()); List < Deployment > deployments = deploymentsFromSlime(object.field(deploymentsField), id.instance(instanceName)); Map<JobType, Instant> jobPauses = jobPausesFromSlime(object.field(deploymentJobsField)); List<AssignedRotation> assignedRotations = assignedRotationsFromSlime(object); RotationStatus rotationStatus = rotationStatusFromSlime(object); Change change = changeFromSlime(object.field(deployingField)); instances.add(new Instance(id.instance(instanceName), - tags, deployments, jobPauses, assignedRotations, diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index 2d78667cb99..1505d7e2ca8 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -22,7 +22,6 @@ import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.NodeResources; -import com.yahoo.config.provision.Tags; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.zone.RoutingMethod; import com.yahoo.config.provision.zone.ZoneId; @@ -31,7 +30,6 @@ import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.container.jdisc.ThreadedHttpRequestHandler; import com.yahoo.io.IOUtils; -import com.yahoo.jdisc.Response; import com.yahoo.jdisc.http.filter.security.misc.User; import com.yahoo.restapi.ByteArrayResponse; import com.yahoo.restapi.ErrorResponse; @@ -1361,12 +1359,9 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { toSlime(cluster.min(), clusterObject.setObject("min")); toSlime(cluster.max(), clusterObject.setObject("max")); toSlime(cluster.current(), clusterObject.setObject("current")); - toSlime(cluster.target(), cluster, clusterObject.setObject("target")); - toSlime(cluster.suggested(), cluster, clusterObject.setObject("suggested")); - legacyUtilizationToSlime(cluster.target().peak(), cluster.target().ideal(), clusterObject.setObject("utilization")); // TODO: Remove after January 2023 + toSlime(cluster.target(), clusterObject.setObject("target")); + toSlime(cluster.suggested(), clusterObject.setObject("suggested")); scalingEventsToSlime(cluster.scalingEvents(), clusterObject.setArray("scalingEvents")); - clusterObject.setString("autoscalingStatusCode", cluster.target().status()); // TODO: Remove after January 2023 - clusterObject.setString("autoscalingStatus", cluster.target().description()); // TODO: Remove after January 2023 clusterObject.setLong("scalingDuration", cluster.scalingDuration().toMillis()); } return new SlimeJsonResponse(slime); @@ -2129,7 +2124,7 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { if (controller.applications().getApplication(applicationId).isEmpty()) createApplication(tenantName, applicationName, request); - controller.applications().createInstance(applicationId.instance(instanceName), Tags.empty()); + controller.applications().createInstance(applicationId.instance(instanceName)); Slime slime = new Slime(); toSlime(applicationId.instance(instanceName), slime.setObject(), request); @@ -2713,12 +2708,7 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { object.setDouble("cost", cost); } - private void toSlime(Cluster.Autoscaling autoscaling, Cluster cluster, Cursor autoscalingObject) { - // TODO: Remove after January 2023 - if (autoscaling.resources().isPresent() - && ! autoscaling.resources().get().justNumbers().equals(cluster.current().justNumbers())) - toSlime(autoscaling.resources().get(), autoscalingObject); - + private void toSlime(Cluster.Autoscaling autoscaling, Cursor autoscalingObject) { autoscalingObject.setString("status", autoscaling.status()); autoscalingObject.setString("description", autoscaling.description()); autoscaling.resources().ifPresent(resources -> toSlime(resources, autoscalingObject.setObject("resources"))); @@ -2733,17 +2723,6 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { loadObject.setDouble("disk", load.disk()); } - private void legacyUtilizationToSlime(Load peak, Load ideal, Cursor utilizationObject) { - utilizationObject.setDouble("idealCpu", ideal.cpu()); - utilizationObject.setDouble("peakCpu", peak.cpu()); - - utilizationObject.setDouble("idealMemory", ideal.memory()); - utilizationObject.setDouble("peakMemory", peak.memory()); - - utilizationObject.setDouble("idealDisk", ideal.disk()); - utilizationObject.setDouble("peakDisk", peak.disk()); - } - private void scalingEventsToSlime(List<Cluster.ScalingEvent> scalingEvents, Cursor scalingEventsArray) { for (Cluster.ScalingEvent scalingEvent : scalingEvents) { Cursor scalingEventObject = scalingEventsArray.addObject(); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java index 8fdff787420..95b81dffaed 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java @@ -15,7 +15,6 @@ import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.RegionName; -import com.yahoo.config.provision.Tags; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.zone.RoutingMethod; import com.yahoo.config.provision.zone.ZoneId; @@ -1406,7 +1405,7 @@ public class ControllerTest { void testDeactivateDeploymentUnknownByController() { DeploymentContext context = tester.newDeploymentContext(); DeploymentId deployment = context.deploymentIdIn(ZoneId.from("prod", "us-west-1")); - DeploymentData deploymentData = new DeploymentData(deployment.applicationId(), Tags.empty(), deployment.zoneId(), InputStream::nullInputStream, Version.fromString("6.1"), + DeploymentData deploymentData = new DeploymentData(deployment.applicationId(), deployment.zoneId(), InputStream::nullInputStream, Version.fromString("6.1"), Set.of(), Optional::empty, Optional.empty(), Optional.empty(), Quota::unlimited, List.of(), List.of(), Optional::empty, false); tester.configServer().deploy(deploymentData); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java index 38a4945e354..43a85aa6aca 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java @@ -7,7 +7,6 @@ import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.SystemName; -import com.yahoo.config.provision.Tags; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.zone.RoutingMethod; import com.yahoo.config.provision.zone.ZoneApi; @@ -371,7 +370,7 @@ public final class ControllerTester { public Application createApplication(String tenant, String applicationName, String instanceName) { Application application = createApplication(tenant, applicationName); - controller().applications().createInstance(application.id().instance(instanceName), Tags.empty()); + controller().applications().createInstance(application.id().instance(instanceName)); return application; } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/DeploymentQuotaCalculatorTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/DeploymentQuotaCalculatorTest.java index ff0bc4c8876..a199ef9e34e 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/DeploymentQuotaCalculatorTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/DeploymentQuotaCalculatorTest.java @@ -8,7 +8,6 @@ import com.yahoo.config.application.api.ValidationOverrides; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.RegionName; -import com.yahoo.config.provision.Tags; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Instance; @@ -38,23 +37,24 @@ public class DeploymentQuotaCalculatorTest { void quota_is_divided_among_prod_instances() { Quota calculated = DeploymentQuotaCalculator.calculate(Quota.unlimited().withBudget(10), List.of(), ApplicationId.defaultId(), ZoneId.defaultId(), DeploymentSpec.fromXml( - "<deployment version='1.0'>\n" + - " <instance id='instance1'> \n" + - " <test />\n" + - " <staging />\n" + - " <prod>\n" + - " <region active=\"true\">us-east-1</region>\n" + - " <region active=\"false\">us-west-1</region>\n" + - " </prod>\n" + - " </instance>\n" + - " <instance id='instance2'>\n" + - " <perf/>\n" + - " <dev/>\n" + - " <prod>\n" + - " <region active=\"true\">us-north-1</region>\n" + - " </prod>\n" + - " </instance>\n" + - "</deployment>")); + """ + <deployment version='1.0'> + <instance id='instance1'>\s + <test /> + <staging /> + <prod> + <region active="true">us-east-1</region> + <region active="false">us-west-1</region> + </prod> + </instance> + <instance id='instance2'> + <perf/> + <dev/> + <prod> + <region active="true">us-north-1</region> + </prod> + </instance> + </deployment>""")); assertEquals(10d / 3, calculated.budget().orElseThrow().doubleValue(), 1e-5); } @@ -63,23 +63,24 @@ public class DeploymentQuotaCalculatorTest { var existing_dev_deployment = new Application(TenantAndApplicationId.from(ApplicationId.defaultId()), Instant.EPOCH, DeploymentSpec.empty, ValidationOverrides.empty, Optional.empty(), Optional.empty(), Optional.empty(), OptionalInt.empty(), new ApplicationMetrics(1, 1), Set.of(), OptionalLong.empty(), RevisionHistory.empty(), - List.of(new Instance(ApplicationId.defaultId(), Tags.empty()).withNewDeployment(ZoneId.from(Environment.dev, RegionName.defaultName()), + List.of(new Instance(ApplicationId.defaultId()).withNewDeployment(ZoneId.from(Environment.dev, RegionName.defaultName()), RevisionId.forProduction(1), Version.emptyVersion, Instant.EPOCH, Map.of(), QuotaUsage.create(0.53d)))); Quota calculated = DeploymentQuotaCalculator.calculate(Quota.unlimited().withBudget(2), List.of(existing_dev_deployment), ApplicationId.defaultId(), ZoneId.defaultId(), DeploymentSpec.fromXml( - "<deployment version='1.0'>\n" + - " <instance id='default'> \n" + - " <test />\n" + - " <staging />\n" + - " <prod>\n" + - " <region active=\"true\">us-east-1</region>\n" + - " <region active=\"false\">us-west-1</region>\n" + - " <region active=\"true\">us-north-1</region>\n" + - " <region active=\"true\">us-south-1</region>\n" + - " </prod>\n" + - " </instance>\n" + - "</deployment>")); + """ + <deployment version='1.0'> + <instance id='default'>\s + <test /> + <staging /> + <prod> + <region active="true">us-east-1</region> + <region active="false">us-west-1</region> + <region active="true">us-north-1</region> + <region active="true">us-south-1</region> + </prod> + </instance> + </deployment>""")); assertEquals((2d - 0.53d) / 4d, calculated.budget().orElseThrow().doubleValue(), 1e-5); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificatesTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificatesTest.java index d79a81c7746..8a8500448d8 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificatesTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificatesTest.java @@ -9,7 +9,6 @@ import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.SystemName; -import com.yahoo.config.provision.Tags; import com.yahoo.config.provision.zone.RoutingMethod; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.security.KeyAlgorithm; @@ -105,7 +104,7 @@ public class EndpointCertificatesTest { return x509CertificateBuilder.build(); } - private final Instance testInstance = new Instance(ApplicationId.defaultId(), Tags.empty()); + private final Instance testInstance = new Instance(ApplicationId.defaultId()); private final String testKeyName = "testKeyName"; private final String testCertName = "testCertName"; private ZoneId testZone; @@ -245,7 +244,7 @@ public class EndpointCertificatesTest { @Test void includes_application_endpoint_when_declared() { - Instance instance = new Instance(ApplicationId.from("t1", "a1", "default"), Tags.empty()); + Instance instance = new Instance(ApplicationId.from("t1", "a1", "default")); ZoneId zone1 = ZoneId.from(Environment.prod, RegionName.from("aws-us-east-1c")); ZoneId zone2 = ZoneId.from(Environment.prod, RegionName.from("aws-us-west-2a")); ControllerTester tester = publicTester(); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java index c3ba7cf1009..9895cd68004 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java @@ -996,7 +996,7 @@ public class DeploymentTriggerTest { @Test void testUserInstancesNotInDeploymentSpec() { var app = tester.newDeploymentContext(); - tester.controller().applications().createInstance(app.application().id().instance("user"), Tags.empty()); + tester.controller().applications().createInstance(app.application().id().instance("user")); app.submit().deploy(); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java index 04a623f819b..faef6de94ca 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java @@ -7,7 +7,6 @@ import com.yahoo.config.application.api.ValidationOverrides; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.RegionName; -import com.yahoo.config.provision.Tags; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.security.KeyUtils; import com.yahoo.slime.SlimeUtils; @@ -133,14 +132,12 @@ public class ApplicationSerializerTest { Map.of(new JobId(id1, DeploymentContext.productionUsEast3), List.of(applicationVersion2))); List<Instance> instances = List.of(new Instance(id1, - Tags.fromString("tag1 tag2"), deployments, Map.of(DeploymentContext.systemTest, Instant.ofEpochMilli(333)), List.of(rotation("foo", "default", "my-rotation", Set.of("us-west-1"))), rotationStatus, Change.of(new Version("6.1"))), new Instance(id3, - Tags.empty(), List.of(), Map.of(), List.of(), @@ -184,9 +181,6 @@ public class ApplicationSerializerTest { assertEquals(original.revisions().production(), serialized.revisions().production()); assertEquals(original.revisions().development(), serialized.revisions().development()); - assertEquals(original.require(id1.instance()).tags(), serialized.require(id1.instance()).tags()); - assertEquals(original.require(id3.instance()).tags(), serialized.require(id3.instance()).tags()); - assertEquals(original.deploymentSpec().xmlForm(), serialized.deploymentSpec().xmlForm()); assertEquals(original.validationOverrides().xmlForm(), serialized.validationOverrides().xmlForm()); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-clusters.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-clusters.json index c013ccb00fe..d9021b1d894 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-clusters.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-clusters.json @@ -42,17 +42,6 @@ "cost": 0.22 }, "target": { - "nodes": 2, - "groups": 1, - "nodeResources": { - "vcpu": 3.0, - "memoryGb": 8.0, - "diskGb": 50.0, - "bandwidthGbps": 1.0, - "diskSpeed": "slow", - "storageType": "remote" - }, - "cost": 0.29, "status": "ideal", "description": "Cluster is ideally scaled", "resources": { @@ -95,14 +84,6 @@ "disk": 0.0 } }, - "utilization": { - "idealCpu": 0.2, - "peakCpu": 0.35, - "idealMemory": 0.5, - "peakMemory": 0.65, - "idealDisk": 0.8, - "peakDisk": 1.0 - }, "scalingEvents": [ { "from": { @@ -135,8 +116,6 @@ "completion": 2234 } ], - "autoscalingStatusCode": "ideal", - "autoscalingStatus": "Cluster is ideally scaled", "scalingDuration": 360000 } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/athenz/AthenzApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/athenz/AthenzApiTest.java index 2f4b4154c08..3a539987443 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/athenz/AthenzApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/athenz/AthenzApiTest.java @@ -3,7 +3,6 @@ package com.yahoo.vespa.hosted.controller.restapi.athenz; import com.yahoo.application.container.handler.Request; import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.Tags; import com.yahoo.vespa.athenz.api.AthenzDomain; import com.yahoo.vespa.hosted.controller.ControllerTester; import com.yahoo.vespa.hosted.controller.restapi.ContainerTester; @@ -26,8 +25,8 @@ public class AthenzApiTest extends ControllerContainerTest { controllerTester.createTenant("sandbox", AthenzApiHandler.sandboxDomainIn(tester.controller().system()), 123L); controllerTester.createApplication("sandbox", "app", "default"); - tester.controller().applications().createInstance(ApplicationId.from("sandbox", "app", hostedOperator.getName()), Tags.empty()); - tester.controller().applications().createInstance(ApplicationId.from("sandbox", "app", defaultUser.getName()), Tags.empty()); + tester.controller().applications().createInstance(ApplicationId.from("sandbox", "app", hostedOperator.getName())); + tester.controller().applications().createInstance(ApplicationId.from("sandbox", "app", defaultUser.getName())); controllerTester.createApplication("sandbox", "opp", "default"); controllerTester.createTenant("tenant1", "domain1", 123L); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java index 4aa54b7f6fa..f792c511adb 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java @@ -56,30 +56,23 @@ public class AutoscalingMaintainer extends NodeRepositoryMaintainer { private void autoscale(ApplicationId application, NodeList applicationNodes) { try { - nodesByCluster(applicationNodes).forEach((clusterId, clusterNodes) -> autoscale(application, clusterId, clusterNodes)); + nodesByCluster(applicationNodes).forEach((clusterId, clusterNodes) -> autoscale(application, clusterId)); } catch (IllegalArgumentException e) { throw new IllegalArgumentException("Illegal arguments for " + application, e); } } - private void autoscale(ApplicationId applicationId, ClusterSpec.Id clusterId, NodeList clusterNodes) { - Optional<Application> application = nodeRepository().applications().get(applicationId); - if (application.isEmpty()) return; - Optional<Cluster> cluster = application.get().cluster(clusterId); - if (cluster.isEmpty()) return; - - Cluster updatedCluster = updateCompletion(cluster.get(), clusterNodes); - var autoscaling = autoscaler.autoscale(application.get(), updatedCluster, clusterNodes); - - if ( ! anyChanges(autoscaling, cluster.get(), updatedCluster, clusterNodes)) return; - + private void autoscale(ApplicationId applicationId, ClusterSpec.Id clusterId) { try (var lock = nodeRepository().applications().lock(applicationId)) { - application = nodeRepository().applications().get(applicationId); + Optional<Application> application = nodeRepository().applications().get(applicationId); if (application.isEmpty()) return; - cluster = application.get().cluster(clusterId); + Optional<Cluster> cluster = application.get().cluster(clusterId); if (cluster.isEmpty()) return; - clusterNodes = nodeRepository().nodes().list(Node.State.active).owner(applicationId).cluster(clusterId); + + NodeList clusterNodes = nodeRepository().nodes().list(Node.State.active).owner(applicationId).cluster(clusterId); + Cluster updatedCluster = updateCompletion(cluster.get(), clusterNodes); + var autoscaling = autoscaler.autoscale(application.get(), updatedCluster, clusterNodes); // 1. Update cluster info updatedCluster = updateCompletion(cluster.get(), clusterNodes); @@ -100,15 +93,6 @@ public class AutoscalingMaintainer extends NodeRepositoryMaintainer { } } - private boolean anyChanges(Autoscaling autoscaling, Cluster cluster, Cluster updatedCluster, NodeList clusterNodes) { - if (updatedCluster != cluster) return true; - if ( ! cluster.target().resources().equals(autoscaling.resources())) return true; - if ( ! cluster.target().status().equals(autoscaling.status())) return true; - if (autoscaling.resources().isPresent() && - !autoscaling.resources().get().equals(new AllocatableClusterResources(clusterNodes, nodeRepository()).advertisedResources())) return true; - return false; - } - private Applications applications() { return nodeRepository().applications(); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainerTest.java index 6374bf52687..2da65fc1a2f 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainerTest.java @@ -78,10 +78,20 @@ public class AutoscalingMaintainerTest { tester.maintainer().maintain(); assertTrue(tester.deployer().lastDeployTime(app1).isEmpty()); // since autoscaling is off assertTrue(tester.deployer().lastDeployTime(app2).isPresent()); - assertNotEquals(Load.zero(), - tester.nodeRepository().applications().require(app1).cluster(cluster1.id()).get().target().peak()); - assertNotEquals(Load.zero(), - tester.nodeRepository().applications().require(app1).cluster(cluster1.id()).get().target().ideal()); + Load peakAt90 = tester.nodeRepository().applications().require(app1).cluster(cluster1.id()).get().target().peak(); + Load idealAt90 = tester.nodeRepository().applications().require(app1).cluster(cluster1.id()).get().target().ideal(); + assertNotEquals(Load.zero(), peakAt90); + assertNotEquals(Load.zero(), idealAt90); + + // Verify that load is updated even when there's no other change + tester.clock().advance(Duration.ofMinutes(10)); + tester.addMeasurements(0.8f, 0.8f, 0.8f, 0, 500, app1, cluster1.id()); + tester.maintainer().maintain(); + + Load peakAt80 = tester.nodeRepository().applications().require(app1).cluster(cluster1.id()).get().target().peak(); + Load idealAt80 = tester.nodeRepository().applications().require(app1).cluster(cluster1.id()).get().target().ideal(); + assertNotEquals(peakAt90, peakAt80); + assertNotEquals(idealAt90, idealAt80); } @Test diff --git a/screwdriver/build-vespa.sh b/screwdriver/build-vespa.sh index 6a93474c620..1a9288c007f 100755 --- a/screwdriver/build-vespa.sh +++ b/screwdriver/build-vespa.sh @@ -18,7 +18,7 @@ ccache -p if ! source $SOURCE_DIR/screwdriver/detect-what-to-build.sh; then echo "Could not detect what to build." - SHOULD_BUILD=all + SHOULD_BUILD=systemtest fi echo "Building: $SHOULD_BUILD" diff --git a/screwdriver/detect-what-to-build.sh b/screwdriver/detect-what-to-build.sh index 358c855f7bd..9666f058942 100755 --- a/screwdriver/detect-what-to-build.sh +++ b/screwdriver/detect-what-to-build.sh @@ -19,7 +19,7 @@ COMMITS=$(jq -re '.[].sha' <<< "$JSON") FILES=$(for C in $COMMITS; do JSON=$(curl -sLf https://api.github.com/repos/vespa-engine/vespa/commits/$C); jq -re '.files[].filename' <<< "$JSON"; done) -if [[ $PR_TITLE =~ \[run-systemtest\] ]]; then +if ! [[ $PR_TITLE =~ \[skip-systemtest\] ]]; then SHOULD_BUILD=systemtest elif [[ -z $FILES ]]; then SHOULD_BUILD=all diff --git a/screwdriver/release-container-image.sh b/screwdriver/release-container-image.sh index 6a9db770c82..6d8babe3dcc 100755 --- a/screwdriver/release-container-image.sh +++ b/screwdriver/release-container-image.sh @@ -49,7 +49,10 @@ docker context use vespa-context docker buildx create --name vespa-builder --driver docker-container --use docker buildx inspect --bootstrap -for data in "Dockerfile vespa" "Dockerfile.minimal vespa-minimal"; do +#The minimal image seem to have issues building on cd.screwdriver.cd. Needs investigation. +#for data in "Dockerfile vespa" "Dockerfile.minimal vespa-minimal"; do + +for data in "Dockerfile vespa"; do set -- $data DOCKER_FILE=$1 IMAGE_NAME=$2 diff --git a/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp b/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp index 584335803a8..d00342d0e51 100644 --- a/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp +++ b/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp @@ -976,8 +976,8 @@ TEST_F("Populates address space usage", DenseTensorAttributeHnswIndex) const auto& all = usage.get_all(); EXPECT_EQUAL(3u, all.size()); EXPECT_EQUAL(1u, all.count("tensor-store")); - EXPECT_EQUAL(1u, all.count("hnsw-node-store")); - EXPECT_EQUAL(1u, all.count("hnsw-link-store")); + EXPECT_EQUAL(1u, all.count("hnsw-levels-store")); + EXPECT_EQUAL(1u, all.count("hnsw-links-store")); } diff --git a/searchlib/src/tests/tensor/hnsw_index/hnsw_index_test.cpp b/searchlib/src/tests/tensor/hnsw_index/hnsw_index_test.cpp index d8e189717c3..7fbfe19b42b 100644 --- a/searchlib/src/tests/tensor/hnsw_index/hnsw_index_test.cpp +++ b/searchlib/src/tests/tensor/hnsw_index/hnsw_index_test.cpp @@ -522,7 +522,7 @@ TYPED_TEST(HnswIndexTest, memory_is_reclaimed_when_doing_changes_to_graph) EXPECT_EQ(0, mem_2.allocatedBytesOnHold()); this->remove_document(2); - size_t node_ref_growth = 0; + size_t nodes_growth = 0; if constexpr (TestFixture::is_single) { this->expect_level_0(1, {3}); this->expect_empty_level_0(2); @@ -532,11 +532,11 @@ TYPED_TEST(HnswIndexTest, memory_is_reclaimed_when_doing_changes_to_graph) this->expect_level_0(1, {2}); this->expect_empty_level_0(3); this->expect_level_0(2, {1}); - node_ref_growth = sizeof(HnswNode); // Entry for nodeid 3 added when adding doc 2 + nodes_growth = sizeof(HnswNode); // Entry for nodeid 3 added when adding doc 2 } auto mem_3 = this->memory_usage(); // We end up in the same state as before document 2 was added and effectively use the same amount of memory. - EXPECT_EQ((mem_1.usedBytes() - mem_1.deadBytes() + node_ref_growth), (mem_3.usedBytes() - mem_3.deadBytes())); + EXPECT_EQ((mem_1.usedBytes() - mem_1.deadBytes() + nodes_growth), (mem_3.usedBytes() - mem_3.deadBytes())); EXPECT_EQ(0, mem_3.allocatedBytesOnHold()); } @@ -642,14 +642,14 @@ make_graph_helper(HnswIndex<type>& index) using LinkArrayRef = typename HnswGraph<type>::LinkArrayRef; auto& graph = index.get_graph(); ResultGraph result(graph.size()); - assert(!graph.get_node_ref(0).valid()); + assert(!graph.get_levels_ref(0).valid()); for (uint32_t doc_id = 1; doc_id < graph.size(); ++doc_id) { auto& node = result[doc_id]; - auto node_ref = graph.get_node_ref(doc_id); + auto levels_ref = graph.get_levels_ref(doc_id); if constexpr (std::is_same_v<std::remove_reference_t<decltype(node)>, uint32_t>) { - node = node_ref.ref(); + node = levels_ref.ref(); } else { - LevelArrayRef level_array(graph.get_level_array(node_ref)); + LevelArrayRef level_array(graph.get_level_array(levels_ref)); for (uint32_t level = 0; level < level_array.size(); ++level) { if constexpr (std::is_same_v<std::remove_reference_t<decltype(node)>, std::vector<uint32_t>>) { node.emplace_back(level_array[level].load_relaxed().ref()); diff --git a/searchlib/src/tests/tensor/hnsw_nodeid_mapping/hnsw_nodeid_mapping_test.cpp b/searchlib/src/tests/tensor/hnsw_nodeid_mapping/hnsw_nodeid_mapping_test.cpp index ac8b21d6136..3dba1aa44ac 100644 --- a/searchlib/src/tests/tensor/hnsw_nodeid_mapping/hnsw_nodeid_mapping_test.cpp +++ b/searchlib/src/tests/tensor/hnsw_nodeid_mapping/hnsw_nodeid_mapping_test.cpp @@ -79,13 +79,13 @@ TEST_F(HnswNodeidMappingTest, free_ids_puts_nodeids_on_hold_list_and_then_free_l TEST_F(HnswNodeidMappingTest, on_load_populates_mapping) { std::vector<HnswNode> nodes(10); - nodes[1].ref().store_relaxed(EntryRef(1)); + nodes[1].levels_ref().store_relaxed(EntryRef(1)); nodes[1].store_docid(7); nodes[1].store_subspace(0); - nodes[2].ref().store_relaxed(EntryRef(2)); + nodes[2].levels_ref().store_relaxed(EntryRef(2)); nodes[2].store_docid(4); nodes[2].store_subspace(0); - nodes[7].ref().store_relaxed(EntryRef(3)); + nodes[7].levels_ref().store_relaxed(EntryRef(3)); nodes[7].store_docid(4); nodes[7].store_subspace(1); mapping.on_load(vespalib::ConstArrayRef(nodes.data(), nodes.size())); diff --git a/searchlib/src/tests/tensor/hnsw_saver/hnsw_save_load_test.cpp b/searchlib/src/tests/tensor/hnsw_saver/hnsw_save_load_test.cpp index bf4abdd7cf8..21ee88a46fe 100644 --- a/searchlib/src/tests/tensor/hnsw_saver/hnsw_save_load_test.cpp +++ b/searchlib/src/tests/tensor/hnsw_saver/hnsw_save_load_test.cpp @@ -113,7 +113,7 @@ void modify(HnswGraph<type> &graph) { graph.set_link_array(4, 1, V{7}); graph.set_link_array(7, 1, V{4}); - graph.set_entry_node({4, graph.get_node_ref(4), 1}); + graph.set_entry_node({4, graph.get_levels_ref(4), 1}); } @@ -124,7 +124,7 @@ public: GraphType copy; void expect_empty_d(uint32_t nodeid) const { - EXPECT_FALSE(copy.acquire_node_ref(nodeid).valid()); + EXPECT_FALSE(copy.acquire_levels_ref(nodeid).valid()); } void expect_level_0(uint32_t nodeid, const V& exp_links) const { @@ -160,7 +160,7 @@ public: } void expect_docid_and_subspace(uint32_t nodeid) const { - auto& node = copy.node_refs.get_elem_ref(nodeid); + auto& node = copy.nodes.get_elem_ref(nodeid); EXPECT_EQ(fake_docid<GraphType::index_type>(nodeid), fake_get_docid(node, nodeid)); EXPECT_EQ(fake_subspace<GraphType::index_type>(nodeid), node.acquire_subspace()); } diff --git a/searchlib/src/vespa/searchlib/attribute/address_space_components.cpp b/searchlib/src/vespa/searchlib/attribute/address_space_components.cpp index c8f2f93832a..244e01a3874 100644 --- a/searchlib/src/vespa/searchlib/attribute/address_space_components.cpp +++ b/searchlib/src/vespa/searchlib/attribute/address_space_components.cpp @@ -19,7 +19,7 @@ const vespalib::string AddressSpaceComponents::enum_store = "enum-store"; const vespalib::string AddressSpaceComponents::multi_value = "multi-value"; const vespalib::string AddressSpaceComponents::tensor_store = "tensor-store"; const vespalib::string AddressSpaceComponents::shared_string_repo = "shared-string-repo"; -const vespalib::string AddressSpaceComponents::hnsw_node_store = "hnsw-node-store"; -const vespalib::string AddressSpaceComponents::hnsw_link_store = "hnsw-link-store"; +const vespalib::string AddressSpaceComponents::hnsw_levels_store = "hnsw-levels-store"; +const vespalib::string AddressSpaceComponents::hnsw_links_store = "hnsw-links-store"; } diff --git a/searchlib/src/vespa/searchlib/attribute/address_space_components.h b/searchlib/src/vespa/searchlib/attribute/address_space_components.h index 3d409ece973..a8a41a024f8 100644 --- a/searchlib/src/vespa/searchlib/attribute/address_space_components.h +++ b/searchlib/src/vespa/searchlib/attribute/address_space_components.h @@ -18,8 +18,8 @@ public: static const vespalib::string multi_value; static const vespalib::string tensor_store; static const vespalib::string shared_string_repo; - static const vespalib::string hnsw_node_store; - static const vespalib::string hnsw_link_store; + static const vespalib::string hnsw_levels_store; + static const vespalib::string hnsw_links_store; }; } diff --git a/searchlib/src/vespa/searchlib/tensor/hnsw_graph.cpp b/searchlib/src/vespa/searchlib/tensor/hnsw_graph.cpp index 4be7a3e135c..e707b464049 100644 --- a/searchlib/src/vespa/searchlib/tensor/hnsw_graph.cpp +++ b/searchlib/src/vespa/searchlib/tensor/hnsw_graph.cpp @@ -9,13 +9,13 @@ namespace search::tensor { template <HnswIndexType type> HnswGraph<type>::HnswGraph() - : node_refs(), - node_refs_size(1u), - nodes(HnswIndex<type>::make_default_node_store_config(), {}), - links(HnswIndex<type>::make_default_link_store_config(), {}), + : nodes(), + nodes_size(1u), + levels_store(HnswIndex<type>::make_default_level_array_store_config(), {}), + links_store(HnswIndex<type>::make_default_link_array_store_config(), {}), entry_nodeid_and_level() { - node_refs.ensure_size(1, NodeType()); + nodes.ensure_size(1, NodeType()); EntryNode entry; set_entry_node(entry); } @@ -24,68 +24,68 @@ template <HnswIndexType type> HnswGraph<type>::~HnswGraph() = default; template <HnswIndexType type> -typename HnswGraph<type>::NodeRef +typename HnswGraph<type>::LevelsRef HnswGraph<type>::make_node(uint32_t nodeid, uint32_t docid, uint32_t subspace, uint32_t num_levels) { - node_refs.ensure_size(nodeid + 1, NodeType()); + nodes.ensure_size(nodeid + 1, NodeType()); // A document cannot be added twice. - assert(!get_node_ref(nodeid).valid()); + assert(!get_levels_ref(nodeid).valid()); // Note: The level array instance lives as long as the document is present in the index. std::vector<AtomicEntryRef> levels(num_levels, AtomicEntryRef()); - auto node_ref = nodes.add(levels); - auto& node = node_refs[nodeid]; - node.ref().store_release(node_ref); + auto levels_ref = levels_store.add(levels); + auto& node = nodes[nodeid]; + node.levels_ref().store_release(levels_ref); node.store_docid(docid); node.store_subspace(subspace); - if (nodeid >= node_refs_size.load(std::memory_order_relaxed)) { - node_refs_size.store(nodeid + 1, std::memory_order_release); + if (nodeid >= nodes_size.load(std::memory_order_relaxed)) { + nodes_size.store(nodeid + 1, std::memory_order_release); } - return node_ref; + return levels_ref; } template <HnswIndexType type> void HnswGraph<type>::remove_node(uint32_t nodeid) { - auto node_ref = get_node_ref(nodeid); - assert(node_ref.valid()); - auto levels = nodes.get(node_ref); + auto levels_ref = get_levels_ref(nodeid); + assert(levels_ref.valid()); + auto levels = levels_store.get(levels_ref); vespalib::datastore::EntryRef invalid; - node_refs[nodeid].ref().store_release(invalid); + nodes[nodeid].levels_ref().store_release(invalid); // Ensure data referenced through the old ref can be recycled: - nodes.remove(node_ref); + levels_store.remove(levels_ref); for (size_t i = 0; i < levels.size(); ++i) { auto old_links_ref = levels[i].load_relaxed(); - links.remove(old_links_ref); + links_store.remove(old_links_ref); } - if (nodeid + 1 == node_refs_size.load(std::memory_order_relaxed)) { - trim_node_refs_size(); + if (nodeid + 1 == nodes_size.load(std::memory_order_relaxed)) { + trim_nodes_size(); } } template <HnswIndexType type> void -HnswGraph<type>::trim_node_refs_size() +HnswGraph<type>::trim_nodes_size() { - uint32_t check_nodeid = node_refs_size.load(std::memory_order_relaxed) - 1; - while (check_nodeid > 0u && !get_node_ref(check_nodeid).valid()) { + uint32_t check_nodeid = nodes_size.load(std::memory_order_relaxed) - 1; + while (check_nodeid > 0u && !get_levels_ref(check_nodeid).valid()) { --check_nodeid; } - node_refs_size.store(check_nodeid + 1, std::memory_order_release); + nodes_size.store(check_nodeid + 1, std::memory_order_release); } template <HnswIndexType type> void HnswGraph<type>::set_link_array(uint32_t nodeid, uint32_t level, const LinkArrayRef& new_links) { - auto new_links_ref = links.add(new_links); - auto node_ref = get_node_ref(nodeid); - assert(node_ref.valid()); - auto levels = nodes.get_writable(node_ref); + auto new_links_ref = links_store.add(new_links); + auto levels_ref = get_levels_ref(nodeid); + assert(levels_ref.valid()); + auto levels = levels_store.get_writable(levels_ref); assert(level < levels.size()); auto old_links_ref = levels[level].load_relaxed(); levels[level].store_release(new_links_ref); - links.remove(old_links_ref); + links_store.remove(old_links_ref); } template <HnswIndexType type> @@ -93,17 +93,17 @@ typename HnswGraph<type>::Histograms HnswGraph<type>::histograms() const { Histograms result; - size_t num_nodes = node_refs_size.load(std::memory_order_acquire); + size_t num_nodes = nodes_size.load(std::memory_order_acquire); for (size_t i = 0; i < num_nodes; ++i) { - auto node_ref = acquire_node_ref(i); - if (node_ref.valid()) { + auto levels_ref = acquire_levels_ref(i); + if (levels_ref.valid()) { uint32_t levels = 0; uint32_t l0links = 0; - auto level_array = nodes.get(node_ref); + auto level_array = levels_store.get(levels_ref); levels = level_array.size(); if (levels > 0) { auto links_ref = level_array[0].load_acquire(); - auto link_array = links.get(links_ref); + auto link_array = links_store.get(links_ref); l0links = link_array.size(); } while (result.level_histogram.size() <= levels) { @@ -125,7 +125,7 @@ HnswGraph<type>::set_entry_node(EntryNode node) { uint64_t value = node.level; value <<= 32; value |= node.nodeid; - if (node.node_ref.valid()) { + if (node.levels_ref.valid()) { assert(node.level >= 0); assert(node.nodeid > 0); } else { diff --git a/searchlib/src/vespa/searchlib/tensor/hnsw_graph.h b/searchlib/src/vespa/searchlib/tensor/hnsw_graph.h index 3fcb1f09eeb..f56d6e60145 100644 --- a/searchlib/src/vespa/searchlib/tensor/hnsw_graph.h +++ b/searchlib/src/vespa/searchlib/tensor/hnsw_graph.h @@ -31,74 +31,74 @@ struct HnswGraph { using NodeType = typename HnswIndexTraits<type>::NodeType; // Provides mapping from document id -> node reference. - // The reference is used to lookup the node data in NodeStore. - using NodeRefVector = vespalib::RcuVector<NodeType>; - using NodeRef = vespalib::datastore::EntryRef; + // The reference is used to lookup the node data in LevelArrayStore. + using NodeVector = vespalib::RcuVector<NodeType>; + using LevelsRef = vespalib::datastore::EntryRef; // This stores the level arrays for all nodes. // Each node consists of an array of levels (from level 0 to n) where each entry is a reference to the link array at that level. - using NodeStore = vespalib::datastore::ArrayStore<AtomicEntryRef, LevelArrayEntryRefType>; - using LevelArrayRef = NodeStore::ConstArrayRef; + using LevelArrayStore = vespalib::datastore::ArrayStore<AtomicEntryRef, LevelArrayEntryRefType>; + using LevelArrayRef = LevelArrayStore::ConstArrayRef; // This stores all link arrays. // A link array consists of the document ids of the nodes a particular node is linked to. - using LinkStore = vespalib::datastore::ArrayStore<uint32_t, LinkArrayEntryRefType>; - using LinkArrayRef = LinkStore::ConstArrayRef; + using LinkArrayStore = vespalib::datastore::ArrayStore<uint32_t, LinkArrayEntryRefType>; + using LinkArrayRef = LinkArrayStore::ConstArrayRef; - NodeRefVector node_refs; - std::atomic<uint32_t> node_refs_size; - NodeStore nodes; - LinkStore links; + NodeVector nodes; + std::atomic<uint32_t> nodes_size; + LevelArrayStore levels_store; + LinkArrayStore links_store; std::atomic<uint64_t> entry_nodeid_and_level; HnswGraph(); ~HnswGraph(); - NodeRef make_node(uint32_t nodeid, uint32_t docid, uint32_t subspace, uint32_t num_levels); + LevelsRef make_node(uint32_t nodeid, uint32_t docid, uint32_t subspace, uint32_t num_levels); void remove_node(uint32_t nodeid); - void trim_node_refs_size(); + void trim_nodes_size(); - NodeRef get_node_ref(uint32_t nodeid) const { - return node_refs.get_elem_ref(nodeid).ref().load_relaxed(); // Called from writer only + LevelsRef get_levels_ref(uint32_t nodeid) const { + return nodes.get_elem_ref(nodeid).levels_ref().load_relaxed(); // Called from writer only } - const NodeType& acquire_node_refs_elem_ref(uint32_t nodeid) const { - return node_refs.acquire_elem_ref(nodeid); + const NodeType& acquire_node(uint32_t nodeid) const { + return nodes.acquire_elem_ref(nodeid); } - NodeRef acquire_node_ref(uint32_t nodeid) const { - return node_refs.acquire_elem_ref(nodeid).ref().load_acquire(); + LevelsRef acquire_levels_ref(uint32_t nodeid) const { + return nodes.acquire_elem_ref(nodeid).levels_ref().load_acquire(); } - bool still_valid(uint32_t nodeid, NodeRef node_ref) const { - return node_ref.valid() && (acquire_node_ref(nodeid) == node_ref); + bool still_valid(uint32_t nodeid, LevelsRef levels_ref) const { + return levels_ref.valid() && (acquire_levels_ref(nodeid) == levels_ref); } - LevelArrayRef get_level_array(NodeRef node_ref) const { - if (node_ref.valid()) { - return nodes.get(node_ref); + LevelArrayRef get_level_array(LevelsRef levels_ref) const { + if (levels_ref.valid()) { + return levels_store.get(levels_ref); } return LevelArrayRef(); } LevelArrayRef get_level_array(uint32_t nodeid) const { - auto node_ref = get_node_ref(nodeid); - return get_level_array(node_ref); + auto levels_ref = get_levels_ref(nodeid); + return get_level_array(levels_ref); } LevelArrayRef acquire_level_array(uint32_t nodeid) const { - auto node_ref = acquire_node_ref(nodeid); - return get_level_array(node_ref); + auto levels_ref = acquire_levels_ref(nodeid); + return get_level_array(levels_ref); } LinkArrayRef get_link_array(LevelArrayRef levels, uint32_t level) const { if (level < levels.size()) { auto links_ref = levels[level].load_acquire(); if (links_ref.valid()) { - return links.get(links_ref); + return links_store.get(links_ref); } } return LinkArrayRef(); @@ -114,8 +114,8 @@ struct HnswGraph { return get_link_array(levels, level); } - LinkArrayRef get_link_array(NodeRef node_ref, uint32_t level) const { - auto levels = get_level_array(node_ref); + LinkArrayRef get_link_array(LevelsRef levels_ref, uint32_t level) const { + auto levels = get_level_array(levels_ref); return get_link_array(levels, level); } @@ -123,16 +123,16 @@ struct HnswGraph { struct EntryNode { uint32_t nodeid; - NodeRef node_ref; + LevelsRef levels_ref; int32_t level; EntryNode() : nodeid(0), // Note that nodeid 0 is reserved and never used - node_ref(), + levels_ref(), level(-1) {} - EntryNode(uint32_t nodeid_in, NodeRef node_ref_in, int32_t level_in) + EntryNode(uint32_t nodeid_in, LevelsRef levels_ref_in, int32_t level_in) : nodeid(nodeid_in), - node_ref(node_ref_in), + levels_ref(levels_ref_in), level(level_in) {} }; @@ -148,18 +148,18 @@ struct HnswGraph { while (true) { uint64_t value = get_entry_atomic(); entry.nodeid = (uint32_t)value; - entry.node_ref = acquire_node_ref(entry.nodeid); + entry.levels_ref = acquire_levels_ref(entry.nodeid); entry.level = (int32_t)(value >> 32); if ((entry.nodeid == 0) && (entry.level == -1) - && ! entry.node_ref.valid()) + && ! entry.levels_ref.valid()) { // invalid in every way return entry; } if ((entry.nodeid > 0) && (entry.level > -1) - && entry.node_ref.valid() + && entry.levels_ref.valid() && (get_entry_atomic() == value)) { // valid in every way @@ -168,7 +168,7 @@ struct HnswGraph { } } - size_t size() const { return node_refs_size.load(std::memory_order_acquire); } + size_t size() const { return nodes_size.load(std::memory_order_acquire); } struct Histograms { std::vector<uint32_t> level_histogram; diff --git a/searchlib/src/vespa/searchlib/tensor/hnsw_index.cpp b/searchlib/src/vespa/searchlib/tensor/hnsw_index.cpp index 32087e0e731..abec443e13e 100644 --- a/searchlib/src/vespa/searchlib/tensor/hnsw_index.cpp +++ b/searchlib/src/vespa/searchlib/tensor/hnsw_index.cpp @@ -63,9 +63,9 @@ bool operator< (const PairDist &a, const PairDist &b) { template <HnswIndexType type> vespalib::datastore::ArrayStoreConfig -HnswIndex<type>::make_default_node_store_config() +HnswIndex<type>::make_default_level_array_store_config() { - return NodeStore::optimizedConfigForHugePage(max_level_array_size, + return LevelArrayStore::optimizedConfigForHugePage(max_level_array_size, vespalib::alloc::MemoryAllocator::HUGEPAGE_SIZE, vespalib::alloc::MemoryAllocator::PAGE_SIZE, min_num_arrays_for_new_buffer, @@ -74,9 +74,9 @@ HnswIndex<type>::make_default_node_store_config() template <HnswIndexType type> vespalib::datastore::ArrayStoreConfig -HnswIndex<type>::make_default_link_store_config() +HnswIndex<type>::make_default_link_array_store_config() { - return LinkStore::optimizedConfigForHugePage(max_link_array_size, + return LinkArrayStore::optimizedConfigForHugePage(max_link_array_size, vespalib::alloc::MemoryAllocator::HUGEPAGE_SIZE, vespalib::alloc::MemoryAllocator::PAGE_SIZE, min_num_arrays_for_new_buffer, @@ -272,9 +272,9 @@ HnswIndex<type>::find_nearest_in_layer(const TypedCells& input, const HnswCandid bool keep_searching = true; while (keep_searching) { keep_searching = false; - for (uint32_t neighbor_nodeid : _graph.get_link_array(nearest.node_ref, level)) { - auto& neighbor_node = _graph.acquire_node_refs_elem_ref(neighbor_nodeid); - auto neighbor_ref = neighbor_node.ref().load_acquire(); + for (uint32_t neighbor_nodeid : _graph.get_link_array(nearest.levels_ref, level)) { + auto& neighbor_node = _graph.acquire_node(neighbor_nodeid); + auto neighbor_ref = neighbor_node.levels_ref().load_acquire(); uint32_t neighbor_docid = acquire_docid(neighbor_node, neighbor_nodeid); uint32_t neighbor_subspace = neighbor_node.acquire_subspace(); double dist = calc_distance(input, neighbor_docid, neighbor_subspace); @@ -317,12 +317,12 @@ HnswIndex<type>::search_layer_helper(const TypedCells& input, uint32_t neighbors break; } candidates.pop(); - for (uint32_t neighbor_nodeid : _graph.get_link_array(cand.node_ref, level)) { + for (uint32_t neighbor_nodeid : _graph.get_link_array(cand.levels_ref, level)) { if (neighbor_nodeid >= nodeid_limit) { continue; } - auto& neighbor_node = _graph.acquire_node_refs_elem_ref(neighbor_nodeid); - auto neighbor_ref = neighbor_node.ref().load_acquire(); + auto& neighbor_node = _graph.acquire_node(neighbor_nodeid); + auto neighbor_ref = neighbor_node.levels_ref().load_acquire(); if ((! neighbor_ref.valid()) || ! visited.try_mark(neighbor_nodeid)) { @@ -351,7 +351,7 @@ void HnswIndex<type>::search_layer(const TypedCells& input, uint32_t neighbors_to_find, BestNeighbors& best_neighbors, uint32_t level, const GlobalFilter *filter) const { - uint32_t nodeid_limit = _graph.node_refs_size.load(std::memory_order_acquire); + uint32_t nodeid_limit = _graph.nodes_size.load(std::memory_order_acquire); if (filter) { nodeid_limit = std::min(filter->size(), nodeid_limit); } @@ -386,7 +386,7 @@ HnswIndex<type>::add_document(uint32_t docid) { vespalib::GenerationHandler::Guard no_guard_needed; PreparedAddDoc op(docid, std::move(no_guard_needed)); - auto input_vectors = get_vector_by_docid(docid); + auto input_vectors = get_vectors(docid); auto subspaces = input_vectors.subspaces(); op.nodes.reserve(subspaces); auto nodeids = _id_mapping.allocate_ids(docid, subspaces); @@ -427,8 +427,8 @@ HnswIndex<type>::internal_prepare_add_node(typename HnswIndex::PreparedAddDoc& o int search_level = entry.level; double entry_dist = calc_distance(input_vector, entry.nodeid); uint32_t entry_docid = get_docid(entry.nodeid); - // TODO: check if entry nodeid/node_ref is still valid here - HnswCandidate entry_point(entry.nodeid, entry_docid, entry.node_ref, entry_dist); + // TODO: check if entry nodeid/levels_ref is still valid here + HnswCandidate entry_point(entry.nodeid, entry_docid, entry.levels_ref, entry_dist); while (search_level > node_max_level) { entry_point = find_nearest_in_layer(input_vector, entry_point, search_level); --search_level; @@ -444,9 +444,9 @@ HnswIndex<type>::internal_prepare_add_node(typename HnswIndex::PreparedAddDoc& o auto& links = connections[search_level]; links.reserve(neighbors.used.size()); for (const auto & neighbor : neighbors.used) { - auto neighbor_levels = _graph.get_level_array(neighbor.node_ref); + auto neighbor_levels = _graph.get_level_array(neighbor.levels_ref); if (size_t(search_level) < neighbor_levels.size()) { - links.emplace_back(neighbor.nodeid, neighbor.node_ref); + links.emplace_back(neighbor.nodeid, neighbor.levels_ref); } else { LOG(warning, "in prepare_add(%u), selected neighbor %u is missing level %d (has %zu levels)", op.docid, neighbor.nodeid, search_level, neighbor_levels.size()); @@ -465,10 +465,10 @@ HnswIndex<type>::filter_valid_nodeids(uint32_t level, const typename PreparedAdd valid.reserve(neighbors.size()); for (const auto & neighbor : neighbors) { uint32_t nodeid = neighbor.first; - vespalib::datastore::EntryRef node_ref = neighbor.second; - if (_graph.still_valid(nodeid, node_ref)) { + vespalib::datastore::EntryRef levels_ref = neighbor.second; + if (_graph.still_valid(nodeid, levels_ref)) { assert(nodeid != self_nodeid); - auto levels = _graph.get_level_array(node_ref); + auto levels = _graph.get_level_array(levels_ref); if (level < levels.size()) { valid.push_back(nodeid); } @@ -495,13 +495,13 @@ void HnswIndex<type>::internal_complete_add_node(uint32_t nodeid, uint32_t docid, uint32_t subspace, PreparedAddNode &prepared_node) { int32_t num_levels = prepared_node.connections.size(); - auto node_ref = _graph.make_node(nodeid, docid, subspace, num_levels); + auto levels_ref = _graph.make_node(nodeid, docid, subspace, num_levels); for (int level = 0; level < num_levels; ++level) { auto neighbors = filter_valid_nodeids(level, prepared_node.connections[level], nodeid); connect_new_node(nodeid, neighbors, level); } if (num_levels - 1 > get_entry_level()) { - _graph.set_entry_node({nodeid, node_ref, num_levels - 1}); + _graph.set_entry_node({nodeid, levels_ref, num_levels - 1}); } } @@ -511,7 +511,7 @@ HnswIndex<type>::prepare_add_document(uint32_t docid, VectorBundle vectors, vespalib::GenerationHandler::Guard read_guard) const { - uint32_t max_nodes = _graph.node_refs_size.load(std::memory_order_acquire); + uint32_t max_nodes = _graph.nodes_size.load(std::memory_order_acquire); if (max_nodes < _cfg.min_size_before_two_phase()) { // the first documents added will do all work in write thread // to ensure they are linked together: @@ -530,7 +530,7 @@ HnswIndex<type>::complete_add_document(uint32_t docid, std::unique_ptr<PrepareRe internal_complete_add(docid, *prepared); } else { // we expect this for the first documents added, so no warning for them - if (_graph.node_refs.size() > 1.25 * _cfg.min_size_before_two_phase()) { + if (_graph.nodes.size() > 1.25 * _cfg.min_size_before_two_phase()) { LOG(warning, "complete_add_document(%u) called with invalid prepare_result %s/%u", docid, (prepared ? "valid ptr" : "nullptr"), (prepared ? prepared->docid : 0u)); } @@ -576,8 +576,8 @@ HnswIndex<type>::remove_node(uint32_t nodeid) LinkArrayRef my_links = _graph.get_link_array(nodeid, level); for (uint32_t neighbor_id : my_links) { if (need_new_entrypoint) { - auto entry_node_ref = _graph.get_node_ref(neighbor_id); - _graph.set_entry_node({neighbor_id, entry_node_ref, level}); + auto entry_levels_ref = _graph.get_levels_ref(neighbor_id); + _graph.set_entry_node({neighbor_id, entry_levels_ref, level}); need_new_entrypoint = false; } remove_link_to(neighbor_id, nodeid, level); @@ -608,9 +608,9 @@ HnswIndex<type>::assign_generation(generation_t current_gen) { // Note: RcuVector transfers hold lists as part of reallocation based on current generation. // We need to set the next generation here, as it is incremented on a higher level right after this call. - _graph.node_refs.setGeneration(current_gen + 1); - _graph.nodes.assign_generation(current_gen); - _graph.links.assign_generation(current_gen); + _graph.nodes.setGeneration(current_gen + 1); + _graph.levels_store.assign_generation(current_gen); + _graph.links_store.assign_generation(current_gen); _id_mapping.assign_generation(current_gen); } @@ -618,9 +618,9 @@ template <HnswIndexType type> void HnswIndex<type>::reclaim_memory(generation_t oldest_used_gen) { - _graph.node_refs.reclaim_memory(oldest_used_gen); _graph.nodes.reclaim_memory(oldest_used_gen); - _graph.links.reclaim_memory(oldest_used_gen); + _graph.levels_store.reclaim_memory(oldest_used_gen); + _graph.links_store.reclaim_memory(oldest_used_gen); _id_mapping.reclaim_memory(oldest_used_gen); } @@ -628,15 +628,15 @@ template <HnswIndexType type> void HnswIndex<type>::compact_level_arrays(CompactionSpec compaction_spec, const CompactionStrategy& compaction_strategy) { - auto compacting_buffers = _graph.nodes.start_compact_worst_buffers(compaction_spec, compaction_strategy); - uint32_t nodeid_limit = _graph.node_refs.size(); + auto compacting_buffers = _graph.levels_store.start_compact_worst_buffers(compaction_spec, compaction_strategy); + uint32_t nodeid_limit = _graph.nodes.size(); auto filter = compacting_buffers->make_entry_ref_filter(); - vespalib::ArrayRef<NodeType> refs(&_graph.node_refs[0], nodeid_limit); - for (auto& ref : refs) { - auto node_ref = ref.ref().load_relaxed(); - if (node_ref.valid() && filter.has(node_ref)) { - EntryRef new_node_ref = _graph.nodes.move_on_compact(node_ref); - ref.ref().store_release(new_node_ref); + vespalib::ArrayRef<NodeType> nodes(&_graph.nodes[0], nodeid_limit); + for (auto& node : nodes) { + auto levels_ref = node.levels_ref().load_relaxed(); + if (levels_ref.valid() && filter.has(levels_ref)) { + EntryRef new_levels_ref = _graph.levels_store.move_on_compact(levels_ref); + node.levels_ref().store_release(new_levels_ref); } } compacting_buffers->finish(); @@ -646,12 +646,12 @@ template <HnswIndexType type> void HnswIndex<type>::compact_link_arrays(CompactionSpec compaction_spec, const CompactionStrategy& compaction_strategy) { - auto context = _graph.links.compactWorst(compaction_spec, compaction_strategy); - uint32_t nodeid_limit = _graph.node_refs.size(); + auto context = _graph.links_store.compactWorst(compaction_spec, compaction_strategy); + uint32_t nodeid_limit = _graph.nodes.size(); for (uint32_t nodeid = 1; nodeid < nodeid_limit; ++nodeid) { - EntryRef level_ref = _graph.get_node_ref(nodeid); - if (level_ref.valid()) { - vespalib::ArrayRef<AtomicEntryRef> refs(_graph.nodes.get_writable(level_ref)); + EntryRef levels_ref = _graph.get_levels_ref(nodeid); + if (levels_ref.valid()) { + vespalib::ArrayRef<AtomicEntryRef> refs(_graph.levels_store.get_writable(levels_ref)); context->compact(refs); } } @@ -661,7 +661,7 @@ template <HnswIndexType type> bool HnswIndex<type>::consider_compact_level_arrays(const CompactionStrategy& compaction_strategy) { - if (!_graph.nodes.has_held_buffers() && _compaction_spec.level_arrays().compact()) { + if (!_graph.levels_store.has_held_buffers() && _compaction_spec.level_arrays().compact()) { compact_level_arrays(_compaction_spec.level_arrays(), compaction_strategy); return true; } @@ -672,7 +672,7 @@ template <HnswIndexType type> bool HnswIndex<type>::consider_compact_link_arrays(const CompactionStrategy& compaction_strategy) { - if (!_graph.links.has_held_buffers() && _compaction_spec.link_arrays().compact()) { + if (!_graph.links_store.has_held_buffers() && _compaction_spec.link_arrays().compact()) { compact_link_arrays(_compaction_spec.link_arrays(), compaction_strategy); return true; } @@ -698,12 +698,12 @@ vespalib::MemoryUsage HnswIndex<type>::update_stat(const CompactionStrategy& compaction_strategy) { vespalib::MemoryUsage result; - result.merge(_graph.node_refs.getMemoryUsage()); - auto level_arrays_memory_usage = _graph.nodes.getMemoryUsage(); - auto level_arrays_address_space_usage = _graph.nodes.addressSpaceUsage(); + result.merge(_graph.nodes.getMemoryUsage()); + auto level_arrays_memory_usage = _graph.levels_store.getMemoryUsage(); + auto level_arrays_address_space_usage = _graph.levels_store.addressSpaceUsage(); result.merge(level_arrays_memory_usage); - auto link_arrays_memory_usage = _graph.links.getMemoryUsage(); - auto link_arrays_address_space_usage = _graph.links.addressSpaceUsage(); + auto link_arrays_memory_usage = _graph.links_store.getMemoryUsage(); + auto link_arrays_address_space_usage = _graph.links_store.addressSpaceUsage(); _compaction_spec = HnswIndexCompactionSpec(compaction_strategy.should_compact(level_arrays_memory_usage, level_arrays_address_space_usage), compaction_strategy.should_compact(link_arrays_memory_usage, link_arrays_address_space_usage)); result.merge(link_arrays_memory_usage); @@ -715,9 +715,9 @@ vespalib::MemoryUsage HnswIndex<type>::memory_usage() const { vespalib::MemoryUsage result; - result.merge(_graph.node_refs.getMemoryUsage()); result.merge(_graph.nodes.getMemoryUsage()); - result.merge(_graph.links.getMemoryUsage()); + result.merge(_graph.levels_store.getMemoryUsage()); + result.merge(_graph.links_store.getMemoryUsage()); result.merge(_id_mapping.memory_usage()); return result; } @@ -726,8 +726,8 @@ template <HnswIndexType type> void HnswIndex<type>::populate_address_space_usage(search::AddressSpaceUsage& usage) const { - usage.set(AddressSpaceComponents::hnsw_node_store, _graph.nodes.addressSpaceUsage()); - usage.set(AddressSpaceComponents::hnsw_link_store, _graph.links.addressSpaceUsage()); + usage.set(AddressSpaceComponents::hnsw_levels_store, _graph.levels_store.addressSpaceUsage()); + usage.set(AddressSpaceComponents::hnsw_links_store, _graph.links_store.addressSpaceUsage()); } template <HnswIndexType type> @@ -737,9 +737,9 @@ HnswIndex<type>::get_state(const vespalib::slime::Inserter& inserter) const auto& object = inserter.insertObject(); auto& memUsageObj = object.setObject("memory_usage"); StateExplorerUtils::memory_usage_to_slime(memory_usage(), memUsageObj.setObject("all")); - StateExplorerUtils::memory_usage_to_slime(_graph.node_refs.getMemoryUsage(), memUsageObj.setObject("node_refs")); StateExplorerUtils::memory_usage_to_slime(_graph.nodes.getMemoryUsage(), memUsageObj.setObject("nodes")); - StateExplorerUtils::memory_usage_to_slime(_graph.links.getMemoryUsage(), memUsageObj.setObject("links")); + StateExplorerUtils::memory_usage_to_slime(_graph.levels_store.getMemoryUsage(), memUsageObj.setObject("levels")); + StateExplorerUtils::memory_usage_to_slime(_graph.links_store.getMemoryUsage(), memUsageObj.setObject("links")); object.setLong("nodes", _graph.size()); auto& histogram_array = object.setArray("level_histogram"); auto& links_hst_array = object.setArray("level_0_links_histogram"); @@ -775,12 +775,12 @@ void HnswIndex<type>::shrink_lid_space(uint32_t doc_id_limit) { assert(doc_id_limit >= 1u); - assert(doc_id_limit >= _graph.node_refs_size.load(std::memory_order_relaxed)); - uint32_t old_doc_id_limit = _graph.node_refs.size(); + assert(doc_id_limit >= _graph.nodes_size.load(std::memory_order_relaxed)); + uint32_t old_doc_id_limit = _graph.nodes.size(); if (doc_id_limit >= old_doc_id_limit) { return; } - _graph.node_refs.shrink(doc_id_limit); + _graph.nodes.shrink(doc_id_limit); } template <HnswIndexType type> @@ -850,8 +850,8 @@ HnswIndex<type>::top_k_candidates(const TypedCells &vector, uint32_t k, const Gl int search_level = entry.level; double entry_dist = calc_distance(vector, entry.nodeid); uint32_t entry_docid = get_docid(entry.nodeid); - // TODO: check if entry docid/node_ref is still valid here - HnswCandidate entry_point(entry.nodeid, entry_docid, entry.node_ref, entry_dist); + // TODO: check if entry docid/levels_ref is still valid here + HnswCandidate entry_point(entry.nodeid, entry_docid, entry.levels_ref, entry_dist); while (search_level > 0) { entry_point = find_nearest_in_layer(vector, entry_point, search_level); --search_level; @@ -865,14 +865,14 @@ template <HnswIndexType type> HnswTestNode HnswIndex<type>::get_node(uint32_t nodeid) const { - auto node_ref = _graph.acquire_node_ref(nodeid); - if (!node_ref.valid()) { + auto levels_ref = _graph.acquire_levels_ref(nodeid); + if (!levels_ref.valid()) { return HnswTestNode(); } - auto levels = _graph.nodes.get(node_ref); + auto levels = _graph.levels_store.get(levels_ref); HnswTestNode::LevelArray result; for (const auto& links_ref : levels) { - auto links = _graph.links.get(links_ref.load_acquire()); + auto links = _graph.links_store.get(links_ref.load_acquire()); HnswTestNode::LinkArray result_links(links.begin(), links.end()); std::sort(result_links.begin(), result_links.end()); result.push_back(result_links); @@ -886,13 +886,13 @@ HnswIndex<type>::set_node(uint32_t nodeid, const HnswTestNode &node) { size_t num_levels = node.size(); assert(num_levels > 0); - auto node_ref = _graph.make_node(nodeid, nodeid, 0, num_levels); + auto levels_ref = _graph.make_node(nodeid, nodeid, 0, num_levels); for (size_t level = 0; level < num_levels; ++level) { connect_new_node(nodeid, node.level(level), level); } int max_level = num_levels - 1; if (get_entry_level() < max_level) { - _graph.set_entry_node({nodeid, node_ref, max_level}); + _graph.set_entry_node({nodeid, levels_ref, max_level}); } } @@ -903,12 +903,12 @@ HnswIndex<type>::check_link_symmetry() const bool all_sym = true; size_t nodeid_limit = _graph.size(); for (size_t nodeid = 0; nodeid < nodeid_limit; ++nodeid) { - auto node_ref = _graph.acquire_node_ref(nodeid); - if (node_ref.valid()) { - auto levels = _graph.nodes.get(node_ref); + auto levels_ref = _graph.acquire_levels_ref(nodeid); + if (levels_ref.valid()) { + auto levels = _graph.levels_store.get(levels_ref); uint32_t level = 0; for (const auto& links_ref : levels) { - auto links = _graph.links.get(links_ref.load_acquire()); + auto links = _graph.links_store.get(links_ref.load_acquire()); for (auto neighbor_nodeid : links) { auto neighbor_links = _graph.acquire_link_array(neighbor_nodeid, level); if (! has_link_to(neighbor_links, nodeid)) { diff --git a/searchlib/src/vespa/searchlib/tensor/hnsw_index.h b/searchlib/src/vespa/searchlib/tensor/hnsw_index.h index 32058e9ac44..38b2c69faf2 100644 --- a/searchlib/src/vespa/searchlib/tensor/hnsw_index.h +++ b/searchlib/src/vespa/searchlib/tensor/hnsw_index.h @@ -65,7 +65,7 @@ public: if constexpr (NodeType::identity_mapping) { return nodeid; } else { - return _graph.node_refs.acquire_elem_ref(nodeid).acquire_docid(); + return _graph.nodes.acquire_elem_ref(nodeid).acquire_docid(); } } @@ -76,9 +76,9 @@ protected: using GraphType = HnswGraph<type>; using NodeType = typename GraphType::NodeType; using AtomicEntryRef = vespalib::datastore::AtomicEntryRef; - using NodeStore = typename GraphType::NodeStore; + using LevelArrayStore = typename GraphType::LevelArrayStore; - using LinkStore = typename GraphType::LinkStore; + using LinkArrayStore = typename GraphType::LinkArrayStore; using LinkArrayRef = typename GraphType::LinkArrayRef; using LinkArray = std::vector<uint32_t, vespalib::allocator_large<uint32_t>>; @@ -137,7 +137,7 @@ protected: if constexpr (NodeType::identity_mapping) { return _vectors.get_vector(nodeid, 0); } else { - auto& ref = _graph.node_refs.acquire_elem_ref(nodeid); + auto& ref = _graph.nodes.acquire_elem_ref(nodeid); uint32_t docid = ref.acquire_docid(); uint32_t subspace = ref.acquire_subspace(); return _vectors.get_vector(docid, subspace); @@ -146,7 +146,7 @@ protected: inline TypedCells get_vector(uint32_t docid, uint32_t subspace) const { return _vectors.get_vector(docid, subspace); } - inline VectorBundle get_vector_by_docid(uint32_t docid) const { + inline VectorBundle get_vectors(uint32_t docid) const { return _vectors.get_vectors(docid); } @@ -259,8 +259,8 @@ public: GraphType& get_graph() { return _graph; } IdMapping& get_id_mapping() { return _id_mapping; } - static vespalib::datastore::ArrayStoreConfig make_default_node_store_config(); - static vespalib::datastore::ArrayStoreConfig make_default_link_store_config(); + static vespalib::datastore::ArrayStoreConfig make_default_level_array_store_config(); + static vespalib::datastore::ArrayStoreConfig make_default_link_array_store_config(); }; } diff --git a/searchlib/src/vespa/searchlib/tensor/hnsw_index_loader.hpp b/searchlib/src/vespa/searchlib/tensor/hnsw_index_loader.hpp index de9cc760fec..279adbac559 100644 --- a/searchlib/src/vespa/searchlib/tensor/hnsw_index_loader.hpp +++ b/searchlib/src/vespa/searchlib/tensor/hnsw_index_loader.hpp @@ -61,12 +61,12 @@ HnswIndexLoader<ReaderType, type>::load_next() if (++_nodeid < _num_nodes) { return true; } else { - _graph.node_refs.ensure_size(std::max(_num_nodes, 1u)); - _graph.node_refs_size.store(std::max(_num_nodes, 1u), std::memory_order_release); - _graph.trim_node_refs_size(); - auto entry_node_ref = _graph.get_node_ref(_entry_nodeid); - _graph.set_entry_node({_entry_nodeid, entry_node_ref, _entry_level}); - _id_mapping.on_load(_graph.node_refs.make_read_view(_graph.node_refs.size())); + _graph.nodes.ensure_size(std::max(_num_nodes, 1u)); + _graph.nodes_size.store(std::max(_num_nodes, 1u), std::memory_order_release); + _graph.trim_nodes_size(); + auto entry_levels_ref = _graph.get_levels_ref(_entry_nodeid); + _graph.set_entry_node({_entry_nodeid, entry_levels_ref, _entry_level}); + _id_mapping.on_load(_graph.nodes.make_read_view(_graph.nodes.size())); _complete = true; return false; } diff --git a/searchlib/src/vespa/searchlib/tensor/hnsw_index_saver.cpp b/searchlib/src/vespa/searchlib/tensor/hnsw_index_saver.cpp index 370e2ddf92a..70daeb34f5c 100644 --- a/searchlib/src/vespa/searchlib/tensor/hnsw_index_saver.cpp +++ b/searchlib/src/vespa/searchlib/tensor/hnsw_index_saver.cpp @@ -14,11 +14,11 @@ template <HnswIndexType type> size_t count_valid_link_arrays(const HnswGraph<type> & graph) { size_t count(0); - size_t num_nodes = graph.node_refs.get_size(); // Called from writer only + size_t num_nodes = graph.nodes.get_size(); // Called from writer only for (size_t i = 0; i < num_nodes; ++i) { - auto node_ref = graph.get_node_ref(i); - if (node_ref.valid()) { - count += graph.nodes.get(node_ref).size(); + auto levels_ref = graph.get_levels_ref(i); + if (levels_ref.valid()) { + count += graph.levels_store.get(levels_ref).size(); } } return count; @@ -42,23 +42,23 @@ HnswIndexSaver<type>::~HnswIndexSaver() = default; template <HnswIndexType type> HnswIndexSaver<type>::HnswIndexSaver(const HnswGraph<type> &graph) - : _graph_links(graph.links), _meta_data() + : _graph_links(graph.links_store), _meta_data() { auto entry = graph.get_entry_node(); _meta_data.entry_nodeid = entry.nodeid; _meta_data.entry_level = entry.level; - size_t num_nodes = graph.node_refs.get_size(); // Called from writer only + size_t num_nodes = graph.nodes.get_size(); // Called from writer only assert (num_nodes <= (std::numeric_limits<uint32_t>::max() - 1)); size_t link_array_count = count_valid_link_arrays(graph); assert (link_array_count <= std::numeric_limits<uint32_t>::max()); _meta_data.refs.reserve(link_array_count); _meta_data.nodes.reserve(num_nodes+1); for (size_t i = 0; i < num_nodes; ++i) { - auto& node = graph.node_refs.get_elem_ref(i); + auto& node = graph.nodes.get_elem_ref(i); _meta_data.nodes.emplace_back(_meta_data.refs.size(), node); - auto node_ref = node.ref().load_relaxed(); - if (node_ref.valid()) { - auto levels = graph.nodes.get(node_ref); + auto levels_ref = node.levels_ref().load_relaxed(); + if (levels_ref.valid()) { + auto levels = graph.levels_store.get(levels_ref); for (const auto& links_ref : levels) { _meta_data.refs.push_back(links_ref.load_relaxed()); } diff --git a/searchlib/src/vespa/searchlib/tensor/hnsw_index_saver.h b/searchlib/src/vespa/searchlib/tensor/hnsw_index_saver.h index 2884ee9b494..76a97274cc6 100644 --- a/searchlib/src/vespa/searchlib/tensor/hnsw_index_saver.h +++ b/searchlib/src/vespa/searchlib/tensor/hnsw_index_saver.h @@ -34,7 +34,7 @@ private: MetaData(); ~MetaData(); }; - const typename HnswGraph<type>::LinkStore &_graph_links; + const typename HnswGraph<type>::LinkArrayStore &_graph_links; MetaData _meta_data; }; diff --git a/searchlib/src/vespa/searchlib/tensor/hnsw_index_utils.h b/searchlib/src/vespa/searchlib/tensor/hnsw_index_utils.h index a88b805f198..95aace19e69 100644 --- a/searchlib/src/vespa/searchlib/tensor/hnsw_index_utils.h +++ b/searchlib/src/vespa/searchlib/tensor/hnsw_index_utils.h @@ -15,14 +15,14 @@ namespace search::tensor { */ struct HnswTraversalCandidate { uint32_t nodeid; - vespalib::datastore::EntryRef node_ref; + vespalib::datastore::EntryRef levels_ref; double distance; HnswTraversalCandidate(uint32_t nodeid_in, double distance_in) noexcept - : nodeid(nodeid_in), node_ref(), distance(distance_in) {} - HnswTraversalCandidate(uint32_t nodeid_in, vespalib::datastore::EntryRef node_ref_in, double distance_in) noexcept - : nodeid(nodeid_in), node_ref(node_ref_in), distance(distance_in) {} - HnswTraversalCandidate(uint32_t nodeid_in, uint32_t docid_in, vespalib::datastore::EntryRef node_ref_in, double distance_in) noexcept - : nodeid(nodeid_in), node_ref(node_ref_in), distance(distance_in) + : nodeid(nodeid_in), levels_ref(), distance(distance_in) {} + HnswTraversalCandidate(uint32_t nodeid_in, vespalib::datastore::EntryRef levels_ref_in, double distance_in) noexcept + : nodeid(nodeid_in), levels_ref(levels_ref_in), distance(distance_in) {} + HnswTraversalCandidate(uint32_t nodeid_in, uint32_t docid_in, vespalib::datastore::EntryRef levels_ref_in, double distance_in) noexcept + : nodeid(nodeid_in), levels_ref(levels_ref_in), distance(distance_in) { (void) docid_in; } @@ -35,8 +35,8 @@ struct HnswTraversalCandidate { struct HnswCandidate : public HnswTraversalCandidate { uint32_t docid; - HnswCandidate(uint32_t nodeid_in, uint32_t docid_in, vespalib::datastore::EntryRef node_ref_in, double distance_in) noexcept - : HnswTraversalCandidate(nodeid_in, docid_in, node_ref_in, distance_in), + HnswCandidate(uint32_t nodeid_in, uint32_t docid_in, vespalib::datastore::EntryRef levels_ref_in, double distance_in) noexcept + : HnswTraversalCandidate(nodeid_in, docid_in, levels_ref_in, distance_in), docid(docid_in) { } diff --git a/searchlib/src/vespa/searchlib/tensor/hnsw_node.h b/searchlib/src/vespa/searchlib/tensor/hnsw_node.h index 2e14f363bba..e5e2910fe81 100644 --- a/searchlib/src/vespa/searchlib/tensor/hnsw_node.h +++ b/searchlib/src/vespa/searchlib/tensor/hnsw_node.h @@ -14,19 +14,19 @@ class HnswNode { using AtomicEntryRef = vespalib::datastore::AtomicEntryRef; using EntryRef = vespalib::datastore::EntryRef; - AtomicEntryRef _ref; + AtomicEntryRef _levels_ref; vespalib::datastore::AtomicValueWrapper<uint32_t> _docid; vespalib::datastore::AtomicValueWrapper<uint32_t> _subspace; public: HnswNode() noexcept - : _ref(), + : _levels_ref(), _docid(), _subspace() { } - AtomicEntryRef& ref() noexcept { return _ref; } - const AtomicEntryRef& ref() const noexcept { return _ref; } + AtomicEntryRef& levels_ref() noexcept { return _levels_ref; } + const AtomicEntryRef& levels_ref() const noexcept { return _levels_ref; } void store_docid(uint32_t docid) noexcept { _docid.store_release(docid); } void store_subspace(uint32_t subspace) noexcept { _subspace.store_release(subspace); } // Mapping from nodeid to docid and subspace. diff --git a/searchlib/src/vespa/searchlib/tensor/hnsw_nodeid_mapping.cpp b/searchlib/src/vespa/searchlib/tensor/hnsw_nodeid_mapping.cpp index 9983bf6d97b..787abf4ad14 100644 --- a/searchlib/src/vespa/searchlib/tensor/hnsw_nodeid_mapping.cpp +++ b/searchlib/src/vespa/searchlib/tensor/hnsw_nodeid_mapping.cpp @@ -122,7 +122,7 @@ get_docid_limit(vespalib::ConstArrayRef<HnswNode> nodes) { uint32_t max_docid = 0; for (auto& node : nodes) { - if (node.ref().load_relaxed().valid()) { + if (node.levels_ref().load_relaxed().valid()) { max_docid = std::max(node.acquire_docid(), max_docid); } } @@ -135,7 +135,7 @@ make_subspaces_histogram(vespalib::ConstArrayRef<HnswNode> nodes, uint32_t docid // Make histogram std::vector<uint32_t> histogram(docid_limit); for (auto& node : nodes) { - if (node.ref().load_relaxed().valid()) { + if (node.levels_ref().load_relaxed().valid()) { auto docid = node.acquire_docid(); auto subspace = node.acquire_subspace(); auto &num_subspaces = histogram[docid]; @@ -171,7 +171,7 @@ HnswNodeidMapping::populate_docid_to_nodeids_mapping_and_free_list(vespalib::Con { uint32_t nodeid = 0; for (auto& node : nodes) { - if (node.ref().load_relaxed().valid()) { + if (node.levels_ref().load_relaxed().valid()) { auto docid = node.acquire_docid(); auto subspace = node.acquire_subspace(); auto nodeids = _nodeids.get_writable(_refs[docid]); @@ -207,7 +207,7 @@ HnswNodeidMapping::on_load(vespalib::ConstArrayRef<HnswNode> nodes) return; } // Check that reserved nodeid is not used - assert(!nodes[0].ref().load_relaxed().valid()); + assert(!nodes[0].levels_ref().load_relaxed().valid()); auto docid_limit = get_docid_limit(nodes); auto histogram = make_subspaces_histogram(nodes, docid_limit); // Allocate mapping from docid to nodeids allocate_docid_to_nodeids_mapping(std::move(histogram)); diff --git a/searchlib/src/vespa/searchlib/tensor/hnsw_simple_node.h b/searchlib/src/vespa/searchlib/tensor/hnsw_simple_node.h index 9740cab5c31..4d775f61a8d 100644 --- a/searchlib/src/vespa/searchlib/tensor/hnsw_simple_node.h +++ b/searchlib/src/vespa/searchlib/tensor/hnsw_simple_node.h @@ -13,15 +13,15 @@ class HnswSimpleNode { using AtomicEntryRef = vespalib::datastore::AtomicEntryRef; using EntryRef = vespalib::datastore::EntryRef; - AtomicEntryRef _ref; + AtomicEntryRef _levels_ref; public: HnswSimpleNode() - : _ref() + : _levels_ref() { } - AtomicEntryRef& ref() noexcept { return _ref; } - const AtomicEntryRef& ref() const noexcept { return _ref; } + AtomicEntryRef& levels_ref() noexcept { return _levels_ref; } + const AtomicEntryRef& levels_ref() const noexcept { return _levels_ref; } void store_docid(uint32_t docid) noexcept { (void) docid; } void store_subspace(uint32_t subspace) noexcept { (void) subspace; } // Mapping from nodeid to docid and subspace. diff --git a/slobrok/src/vespa/slobrok/server/exchange_manager.cpp b/slobrok/src/vespa/slobrok/server/exchange_manager.cpp index b5460460a4a..4b8c1c02252 100644 --- a/slobrok/src/vespa/slobrok/server/exchange_manager.cpp +++ b/slobrok/src/vespa/slobrok/server/exchange_manager.cpp @@ -118,8 +118,8 @@ ExchangeManager::healthCheck() if (remoteList.size() != 0) { vespalib::string diff = diffLists(newWorldList, remoteList); if (! diff.empty()) { - LOG(warning, "Diff from consensus map to peer slobrok mirror: %s", - diff.c_str()); + LOG(warning, "Peer slobrok at %s may have problems, differences from consensus map: %s", + partner->getName().c_str(), diff.c_str()); someBad = true; } } diff --git a/slobrok/src/vespa/slobrok/server/remote_check.cpp b/slobrok/src/vespa/slobrok/server/remote_check.cpp index 40a43a8946b..dccf0934a91 100644 --- a/slobrok/src/vespa/slobrok/server/remote_check.cpp +++ b/slobrok/src/vespa/slobrok/server/remote_check.cpp @@ -15,7 +15,7 @@ RemoteCheck::RemoteCheck(FNET_Scheduler *sched, ExchangeManager& exch) : FNET_Task(sched), _exchanger(exch) { - double seconds = randomIn(15.3, 27.9); + double seconds = randomIn(5.3, 9.7); Schedule(seconds); } @@ -31,7 +31,7 @@ RemoteCheck::PerformTask() { LOG(debug, "asking exchanger to health check"); _exchanger.healthCheck(); - double seconds = randomIn(151.3, 179.7); + double seconds = randomIn(15.3, 17.7); Schedule(seconds); } diff --git a/vespa-dependencies-enforcer/allowed-maven-dependencies.txt b/vespa-dependencies-enforcer/allowed-maven-dependencies.txt index 9cd6ed787b3..1e38b6029ba 100644 --- a/vespa-dependencies-enforcer/allowed-maven-dependencies.txt +++ b/vespa-dependencies-enforcer/allowed-maven-dependencies.txt @@ -240,4 +240,3 @@ org.junit.vintage:junit-vintage-engine:5.8.1 org.mockito:mockito-core:4.0.0 org.mockito:mockito-junit-jupiter:4.0.0 org.objenesis:objenesis:3.2 -xmlunit:xmlunit:1.5 diff --git a/vespalib/CMakeLists.txt b/vespalib/CMakeLists.txt index 2ceb56bf226..8509d5fc382 100644 --- a/vespalib/CMakeLists.txt +++ b/vespalib/CMakeLists.txt @@ -194,6 +194,7 @@ vespa_define_module( src/tests/util/generationhandler_stress src/tests/util/hamming src/tests/util/md5 + src/tests/util/memory_trap src/tests/util/mmap_file_allocator src/tests/util/mmap_file_allocator_factory src/tests/util/rcuvector diff --git a/vespalib/src/tests/util/memory_trap/CMakeLists.txt b/vespalib/src/tests/util/memory_trap/CMakeLists.txt new file mode 100644 index 00000000000..c3241b0ad93 --- /dev/null +++ b/vespalib/src/tests/util/memory_trap/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(vespalib_util_memory_trap_test_app TEST + SOURCES + memory_trap_test.cpp + DEPENDS + vespalib + GTest::GTest +) +vespa_add_test(NAME vespalib_util_memory_trap_test_app COMMAND vespalib_util_memory_trap_test_app) diff --git a/vespalib/src/tests/util/memory_trap/memory_trap_test.cpp b/vespalib/src/tests/util/memory_trap/memory_trap_test.cpp new file mode 100644 index 00000000000..ee26231c546 --- /dev/null +++ b/vespalib/src/tests/util/memory_trap/memory_trap_test.cpp @@ -0,0 +1,61 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/vespalib/util/memory_trap.h> +#include <vespa/vespalib/gtest/gtest.h> +#include <cstdlib> + +using namespace vespalib; +using namespace ::testing; + +template <typename T> +void do_not_optimize_away(T&& t) noexcept { + asm volatile("" : : "m"(t) : "memory"); // Clobber the value to avoid losing it to compiler optimizations +} + +struct MemoryTrapTest : Test { + static void SetUpTestSuite() { + // Don't overwrite env var if already set; we'll assume it's done for a good reason. + setenv("VESPA_USE_MPROTECT_TRAP", "yes", 0); + } +}; + +TEST_F(MemoryTrapTest, untouched_memory_traps_do_not_trigger) { + InlineMemoryTrap<2> stack_trap; + HeapMemoryTrap heap_trap(4); + // No touching == no crashing. Good times. +} + +TEST_F(MemoryTrapTest, write_to_stack_trap_eventually_discovered) { + // We don't explicitly test death messages since the way the process dies depends on + // whether mprotect is enabled, whether ASAN instrumentation is enabled etc. + ASSERT_DEATH({ + InlineMemoryTrap<2> stack_trap; + // This may trigger immediately or on destruction. Either way it eventually kills the process. + stack_trap.trapper().buffer()[0] = 0x01; + },""); +} + +TEST_F(MemoryTrapTest, write_to_heap_trap_eventually_discovered) { + ASSERT_DEATH({ + HeapMemoryTrap heap_trap(4); + // This may trigger immediately or on destruction. Either way it eventually kills the process. + heap_trap.trapper().buffer()[heap_trap.trapper().size() - 1] = 0x01; + },""); +} + +TEST_F(MemoryTrapTest, read_from_hw_backed_trap_crashes_process) { + if (!MemoryRangeTrapper::hw_trapping_enabled()) { + return; + } + ASSERT_DEATH({ + HeapMemoryTrap heap_trap(4); // Entire buffer should always be covered + // Clobber trap just in case the compiler is clever enough to look into the trap implementation + // and see that we memset everything to zero and `dummy` can thus be constant-promoted to 0 + // (probably won't dare to do this anyway due to opaque mprotect() that touches buffer pointer). + do_not_optimize_away(heap_trap); + char dummy = heap_trap.trapper().buffer()[0]; + do_not_optimize_away(dummy); // never reached + },""); +} + +GTEST_MAIN_RUN_ALL_TESTS() diff --git a/vespalib/src/vespa/vespalib/net/tls/maybe_tls_crypto_engine.cpp b/vespalib/src/vespa/vespalib/net/tls/maybe_tls_crypto_engine.cpp index 832d52c0383..971256ff402 100644 --- a/vespalib/src/vespa/vespalib/net/tls/maybe_tls_crypto_engine.cpp +++ b/vespalib/src/vespa/vespalib/net/tls/maybe_tls_crypto_engine.cpp @@ -2,11 +2,22 @@ #include "maybe_tls_crypto_engine.h" #include "maybe_tls_crypto_socket.h" +#include "crypto_codec.h" namespace vespalib { MaybeTlsCryptoEngine::~MaybeTlsCryptoEngine() = default; +std::unique_ptr<net::tls::CryptoCodec> +MaybeTlsCryptoEngine::create_tls_client_crypto_codec(const SocketHandle &socket, const SocketSpec &spec) { + return _tls_engine->create_tls_client_crypto_codec(socket, spec); +} + +std::unique_ptr<net::tls::CryptoCodec> +MaybeTlsCryptoEngine::create_tls_server_crypto_codec(const SocketHandle &socket) { + return _tls_engine->create_tls_server_crypto_codec(socket); +} + CryptoSocket::UP MaybeTlsCryptoEngine::create_client_crypto_socket(SocketHandle socket, const SocketSpec &spec) { diff --git a/vespalib/src/vespa/vespalib/net/tls/maybe_tls_crypto_engine.h b/vespalib/src/vespa/vespalib/net/tls/maybe_tls_crypto_engine.h index 2b82d6eb8bc..e69d5858eab 100644 --- a/vespalib/src/vespa/vespalib/net/tls/maybe_tls_crypto_engine.h +++ b/vespalib/src/vespa/vespalib/net/tls/maybe_tls_crypto_engine.h @@ -15,7 +15,7 @@ namespace vespalib { * connections is controlled by the use_tls_when_client flag given to * the constructor. **/ -class MaybeTlsCryptoEngine : public CryptoEngine +class MaybeTlsCryptoEngine : public AbstractTlsCryptoEngine { private: std::shared_ptr<NullCryptoEngine> _null_engine; @@ -29,6 +29,8 @@ public: _tls_engine(std::move(tls_engine)), _use_tls_when_client(use_tls_when_client) {} ~MaybeTlsCryptoEngine() override; + std::unique_ptr<CryptoCodec> create_tls_client_crypto_codec(const SocketHandle &socket, const SocketSpec &spec) override; + std::unique_ptr<CryptoCodec> create_tls_server_crypto_codec(const SocketHandle &socket) override; bool use_tls_when_client() const override { return _use_tls_when_client; } bool always_use_tls_when_server() const override { return false; } CryptoSocket::UP create_client_crypto_socket(SocketHandle socket, const SocketSpec &spec) override; diff --git a/vespalib/src/vespa/vespalib/util/CMakeLists.txt b/vespalib/src/vespa/vespalib/util/CMakeLists.txt index 3812fda4bdf..ad2db89288c 100644 --- a/vespalib/src/vespa/vespalib/util/CMakeLists.txt +++ b/vespalib/src/vespa/vespalib/util/CMakeLists.txt @@ -54,6 +54,7 @@ vespa_add_library(vespalib_vespalib_util OBJECT lz4compressor.cpp malloc_mmap_guard.cpp md5.c + memory_trap.cpp memoryusage.cpp mmap_file_allocator.cpp mmap_file_allocator_factory.cpp diff --git a/vespalib/src/vespa/vespalib/util/memory_trap.cpp b/vespalib/src/vespa/vespalib/util/memory_trap.cpp new file mode 100644 index 00000000000..d3b666d9a6e --- /dev/null +++ b/vespalib/src/vespa/vespalib/util/memory_trap.cpp @@ -0,0 +1,166 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "memory_trap.h" +#include <string_view> +#include <cassert> +#include <cerrno> +#include <cstdint> +#include <cstdlib> +#include <cstring> +#include <malloc.h> +#include <unistd.h> +#include <sys/mman.h> + +#include <vespa/log/log.h> +LOG_SETUP(".vespalib.util.memory_trap"); + +using namespace std::string_view_literals; + +namespace vespalib { + +namespace { + +// Have some symbols that provide immediate context in a crash backtrace +[[noreturn]] void abort_due_to_guard_bits_tampered_with() __attribute__((noinline)); +[[noreturn]] void abort_due_to_guard_bits_tampered_with() { + abort(); +} + +[[noreturn]] void abort_due_to_PROTECTED_guard_bits_tampered_with() __attribute__((noinline)); +[[noreturn]] void abort_due_to_PROTECTED_guard_bits_tampered_with() { + abort(); +} + +} // anon ns + +MemoryRangeTrapper::MemoryRangeTrapper(char* trap_buf, size_t buf_len) noexcept + : _trap_buf(trap_buf), + _buf_len(buf_len), + _trap_offset(0), + _trap_len(0) +{ + if (_buf_len > 0) { + memset(trap_buf, 0, _buf_len); + } + rw_protect_buffer_if_possible(); +} + +MemoryRangeTrapper::~MemoryRangeTrapper() { + check_and_release(); +} + +void MemoryRangeTrapper::check_and_release() noexcept { + unprotect_buffer_to_read_only(); // Make sure sanity check can't race with writes + verify_buffer_is_all_zeros(); + unprotect_buffer_to_read_and_write(); + _trap_len = _buf_len = 0; +} + +void MemoryRangeTrapper::verify_buffer_is_all_zeros() { + for (size_t i = 0; i < _buf_len; ++i) { + if (_trap_buf[i] != 0) { + const bool in_protected_area = ((i >= _trap_offset) && (i < _trap_offset + _trap_len)); + LOG(error, "Memory corruption detected! Offset %zu into buffer %p: 0x%.2x != 0x00%s", + i, _trap_buf, static_cast<unsigned int>(_trap_buf[i]), + in_protected_area ? ". CORRUPTION IN R/W PROTECTED MEMORY!" : ""); + if (in_protected_area) { + abort_due_to_PROTECTED_guard_bits_tampered_with(); + } else { + abort_due_to_guard_bits_tampered_with(); + } + } + } +} + +#ifdef __linux__ + +namespace { + +bool has_4k_pages() noexcept { + return (sysconf(_SC_PAGESIZE) == 4096); +} + +constexpr bool is_4k_aligned(size_t v) noexcept { + return (v % 4096) == 0; +} + +constexpr size_t align_up_4k(size_t v) noexcept { + return (v + 4095) & ~4095ULL; +} + +constexpr size_t align_down_4k(size_t v) noexcept { + return v & ~4095ULL; +} + +bool env_var_is_yes(const char *env_var) noexcept { + const char *ev = getenv(env_var); + return ((ev != nullptr) && ("yes"sv == ev)); +} + +bool mprotect_trapping_is_enabled() noexcept { + static const bool enabled = (has_4k_pages() && env_var_is_yes("VESPA_USE_MPROTECT_TRAP")); + return enabled; +} + +} // anon ns + +void MemoryRangeTrapper::rw_protect_buffer_if_possible() { + static_assert(std::is_same_v<size_t, uintptr_t>); + const auto aligned_start = align_up_4k(reinterpret_cast<uintptr_t>(_trap_buf)); + const auto aligned_end = align_down_4k(reinterpret_cast<uintptr_t>(_trap_buf + _buf_len)); + if ((aligned_end > aligned_start) && mprotect_trapping_is_enabled()) { + _trap_offset = aligned_start - reinterpret_cast<uintptr_t>(_trap_buf); + _trap_len = aligned_end - aligned_start; + assert(is_4k_aligned(_trap_len)); + + LOG(info, "attempting mprotect(%p + %zu = %p, %zu, PROT_NONE)", + _trap_buf, _trap_offset, _trap_buf + _trap_offset, _trap_len); + int ret = mprotect(_trap_buf + _trap_offset, _trap_len, PROT_NONE); + if (ret != 0) { + LOG(warning, "Failed to mprotect(%p + %zu, %zu, PROT_NONE). errno = %d. " + "Falling back to unprotected mode.", + _trap_buf, _trap_offset, _trap_len, errno); + _trap_offset = _trap_len = 0; + } + } +} + +bool MemoryRangeTrapper::hw_trapping_enabled() noexcept { + return mprotect_trapping_is_enabled(); +} + +void MemoryRangeTrapper::unprotect_buffer_to_read_only() { + if (_trap_len > 0) { + int ret = mprotect(_trap_buf + _trap_offset, _trap_len, PROT_READ); + assert(ret == 0 && "failed to un-protect memory region to PROT_READ"); + } +} + +void MemoryRangeTrapper::unprotect_buffer_to_read_and_write() { + if (_trap_len > 0) { + int ret = mprotect(_trap_buf + _trap_offset, _trap_len, PROT_READ | PROT_WRITE); + assert(ret == 0 && "failed to un-protect memory region to PROT_READ | PROT_WRITE"); + } +} + +#else // Not on Linux, fall back to no-ops + +void MemoryRangeTrapper::rw_protect_buffer_if_possible() { /* no-op */ } +bool MemoryRangeTrapper::hw_trapping_enabled() noexcept { return false; } +void MemoryRangeTrapper::unprotect_buffer_to_read_only() { /* no-op */ } +void MemoryRangeTrapper::unprotect_buffer_to_read_and_write() { /* no-op */ } + +#endif + +HeapMemoryTrap::HeapMemoryTrap(size_t trap_4k_pages) + : _trap_buf(static_cast<char*>(memalign(4096, trap_4k_pages * 4096))), + _trapper(_trap_buf, _trap_buf ? trap_4k_pages * 4096 : 0) +{ +} + +HeapMemoryTrap::~HeapMemoryTrap() { + _trapper.check_and_release(); + free(_trap_buf); +} + +} diff --git a/vespalib/src/vespa/vespalib/util/memory_trap.h b/vespalib/src/vespa/vespalib/util/memory_trap.h new file mode 100644 index 00000000000..f9c8e8458fd --- /dev/null +++ b/vespalib/src/vespa/vespalib/util/memory_trap.h @@ -0,0 +1,101 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <cstddef> + +namespace vespalib { + +/** + * Guard for attempting to detect spurious writes (and if possible; reads) to a memory region. + * + * If supported by the OS+HW, as much as possible of the buffer will be mapped + * as non-readable and non-writable. This immediately triggers a SIGSEGV for any + * spurious read or write to the mapped buffer sub-range. + * + * For memory map-backed trapping to be used, all of the following must hold: + * - The process must be running on Linux and on hardware with a page size of 4 KiB + * - The environment variable VESPA_USE_MPROTECT_TRAP must be set and have the value 'yes' + * - The trap buffer must be long enough to fit at least one whole 4 KiB-aligned page + * - The buffer passed to the trapper must originally have been allocated via mmap(). + * This should hold for any reasonable implementation of malloc(). + * + * Regardless of whether memory map-backed trapping is used, the buffer will always be + * filled with all zeroes upon construction. If any buffer byte is non-zero upon + * destruction, the process will be terminated with a corruption error in the logs. + * + * If buffer mapping fails during construction, the trapper falls back to just checking + * buffer contents. This may happen if the kernel has exhausted the bookkeeping-structures + * for keeping track of separate virtual memory ranges. + * + * Note that due to possible interference with things like hugepages etc, VESPA_USE_MPROTECT_TRAP + * should only be selectively enabled. + */ +class MemoryRangeTrapper { + char* _trap_buf; + size_t _buf_len; + size_t _trap_offset; + size_t _trap_len; +public: + MemoryRangeTrapper(char* trap_buf, size_t buf_len) noexcept; + ~MemoryRangeTrapper(); + + MemoryRangeTrapper(const MemoryRangeTrapper&) = delete; + MemoryRangeTrapper(MemoryRangeTrapper&&) noexcept = delete; + + // Exposed for testing only + char* buffer() const noexcept { return _trap_buf; } + size_t size() const noexcept { return _buf_len; } + + void check_and_release() noexcept; + + [[nodiscard]] static bool hw_trapping_enabled() noexcept; +private: + void rw_protect_buffer_if_possible(); + void unprotect_buffer_to_read_only(); + void unprotect_buffer_to_read_and_write(); + void verify_buffer_is_all_zeros(); +}; + +/** + * Places a memory trap "inline" with other variables in an object. I.e. the trap will + * be in a memory range that is a sub-range of that taken up by the owning object. + * + * Always takes up at least 8 KiB of space. + */ +template <size_t Guard4KPages> +class InlineMemoryTrap { + static_assert(Guard4KPages > 0); + constexpr static size_t BufSize = 4096 * (Guard4KPages + 1); + char _trap_buf[BufSize]; + MemoryRangeTrapper _trapper; +public: + InlineMemoryTrap() noexcept : _trap_buf(), _trapper(_trap_buf, BufSize) {} + ~InlineMemoryTrap() = default; + + InlineMemoryTrap(const InlineMemoryTrap&) = delete; + InlineMemoryTrap(InlineMemoryTrap&&) noexcept = delete; + + // Exposed for testing only + const MemoryRangeTrapper& trapper() const noexcept { return _trapper; } +}; + +/** + * Allocates a 4 KiB-aligned heap buffer and watches it for spurious access. + * Useful for distributing traps across various allocation size-classes. + */ +class HeapMemoryTrap { + char* _trap_buf; + MemoryRangeTrapper _trapper; +public: + explicit HeapMemoryTrap(size_t trap_4k_pages); + ~HeapMemoryTrap(); + + HeapMemoryTrap(const HeapMemoryTrap&) = delete; + HeapMemoryTrap(HeapMemoryTrap&&) noexcept = delete; + + // Exposed for testing only + const MemoryRangeTrapper& trapper() const noexcept { return _trapper; } +}; + +} |