diff options
159 files changed, 2893 insertions, 2219 deletions
diff --git a/application-model/src/main/java/com/yahoo/vespa/archive/ArchiveStreamReader.java b/application-model/src/main/java/com/yahoo/vespa/archive/ArchiveStreamReader.java index 87665efc1ef..acc18b4c13f 100644 --- a/application-model/src/main/java/com/yahoo/vespa/archive/ArchiveStreamReader.java +++ b/application-model/src/main/java/com/yahoo/vespa/archive/ArchiveStreamReader.java @@ -74,7 +74,7 @@ public class ArchiveStreamReader implements AutoCloseable { outputStream.write(buffer, 0, read); } } - return new ArchiveFile(path, crc32(entry), size); + return new ArchiveFile(path, size); } } catch (IOException e) { throw new UncheckedIOException(e); @@ -91,15 +91,10 @@ public class ArchiveStreamReader implements AutoCloseable { public static class ArchiveFile { private final Path path; - private final OptionalLong crc32; private final long size; - public ArchiveFile(Path name, OptionalLong crc32, long size) { + public ArchiveFile(Path name, long size) { this.path = Objects.requireNonNull(name); - this.crc32 = Objects.requireNonNull(crc32); - if (crc32.isPresent()) { - requireNonNegative("crc32", crc32.getAsLong()); - } this.size = requireNonNegative("size", size); } @@ -108,11 +103,6 @@ public class ArchiveStreamReader implements AutoCloseable { return path; } - /** The CRC-32 checksum of this file, if any */ - public OptionalLong crc32() { - return crc32; - } - /** The decompressed size of this file */ public long size() { return size; @@ -120,15 +110,6 @@ public class ArchiveStreamReader implements AutoCloseable { } - /** Get the CRC-32 checksum of given archive entry, if any */ - private static OptionalLong crc32(ArchiveEntry entry) { - long crc32 = -1; - if (entry instanceof ZipArchiveEntry) { - crc32 = ((ZipArchiveEntry) entry).getCrc(); - } - return crc32 > -1 ? OptionalLong.of(crc32) : OptionalLong.empty(); - } - private static boolean isSymlink(ArchiveEntry entry) { // Symlinks inside ZIP files are not part of the ZIP spec and are only supported by some implementations, such // as Info-ZIP. diff --git a/client/go/jvm/env.go b/client/go/jvm/env.go index dbc957e3ccb..d2f0b2e6616 100644 --- a/client/go/jvm/env.go +++ b/client/go/jvm/env.go @@ -19,7 +19,7 @@ func (opts *Options) exportEnvSettings(ps *prog.Spec) { vlt := fmt.Sprintf("file:%s/vespa.log", lvd) lcd := fmt.Sprintf("%s/var/db/vespa/logcontrol", vespaHome) lcf := fmt.Sprintf("%s/%s.logcontrol", lcd, c.ServiceName()) - dlp := fmt.Sprintf("%s/lib64", vespaHome) + dlp := fmt.Sprintf("%s/lib64:/opt/vespa-deps/lib64", vespaHome) opts.fixSpec.FixDir(lvd) opts.fixSpec.FixDir(lcd) ps.Setenv(envvars.VESPA_LOG_TARGET, vlt) diff --git a/client/go/prog/common_env.go b/client/go/prog/common_env.go index a970895bf84..150a4aaa38f 100644 --- a/client/go/prog/common_env.go +++ b/client/go/prog/common_env.go @@ -15,7 +15,7 @@ func (spec *Spec) configureCommonEnv() { os.Unsetenv(envvars.LD_PRELOAD) spec.Setenv(envvars.STD_THREAD_PREVENT_TRY_CATCH, "true") spec.Setenv(envvars.GLIBCXX_FORCE_NEW, "1") - spec.Setenv(envvars.LD_LIBRARY_PATH, vespa.FindHome()+"/lib64") + spec.Setenv(envvars.LD_LIBRARY_PATH, vespa.FindHome()+"/lib64:/opt/vespa-deps/lib64") spec.Setenv(envvars.MALLOC_ARENA_MAX, "1") // fallback from old env.vars: diff --git a/client/go/script-utils/startcbinary/common_env.go b/client/go/script-utils/startcbinary/common_env.go index 812f182a757..eadb0aafb1b 100644 --- a/client/go/script-utils/startcbinary/common_env.go +++ b/client/go/script-utils/startcbinary/common_env.go @@ -26,7 +26,7 @@ func (spec *ProgSpec) configureCommonEnv() { os.Unsetenv(envvars.LD_PRELOAD) spec.setenv(envvars.STD_THREAD_PREVENT_TRY_CATCH, "true") spec.setenv(envvars.GLIBCXX_FORCE_NEW, "1") - spec.setenv(envvars.LD_LIBRARY_PATH, vespa.FindHome()+"/lib64") + spec.setenv(envvars.LD_LIBRARY_PATH, vespa.FindHome()+"/lib64:/opt/vespa-deps/lib64") spec.setenv(envvars.MALLOC_ARENA_MAX, "1") // fallback from old env.vars: diff --git a/config-application-package/pom.xml b/config-application-package/pom.xml index 41e65f19b79..dc2c8d91db1 100644 --- a/config-application-package/pom.xml +++ b/config-application-package/pom.xml @@ -109,12 +109,6 @@ <artifactId>junit</artifactId> <scope>test</scope> </dependency> - <dependency> - <groupId>xmlunit</groupId> - <artifactId>xmlunit</artifactId> - <version>1.5</version> - <scope>test</scope> - </dependency> </dependencies> <build> <plugins> diff --git a/config-application-package/src/main/java/com/yahoo/config/application/OverrideProcessor.java b/config-application-package/src/main/java/com/yahoo/config/application/OverrideProcessor.java index 21bb193ef93..169b1c8557b 100644 --- a/config-application-package/src/main/java/com/yahoo/config/application/OverrideProcessor.java +++ b/config-application-package/src/main/java/com/yahoo/config/application/OverrideProcessor.java @@ -13,7 +13,6 @@ import org.w3c.dom.NamedNodeMap; import javax.xml.transform.TransformerException; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collections; import java.util.Iterator; import java.util.LinkedHashMap; import java.util.List; diff --git a/config-application-package/src/main/java/com/yahoo/config/application/Xml.java b/config-application-package/src/main/java/com/yahoo/config/application/Xml.java index 2cc5e1ac4e2..cf391f2dd0e 100644 --- a/config-application-package/src/main/java/com/yahoo/config/application/Xml.java +++ b/config-application-package/src/main/java/com/yahoo/config/application/Xml.java @@ -101,7 +101,12 @@ public class Xml { } StringWriter writer = new StringWriter(); transformer.transform(new DOMSource(document), new StreamResult(writer)); - return writer.toString(); + String[] lines = writer.toString().split("\n"); + var b = new StringBuilder(); + for (String line : lines) + if ( ! line.isBlank()) + b.append(line).append("\n"); + return b.toString(); } static String documentAsString(Document document) throws TransformerException { diff --git a/config-application-package/src/main/java/com/yahoo/config/application/XmlPreProcessor.java b/config-application-package/src/main/java/com/yahoo/config/application/XmlPreProcessor.java index 42333ea7662..7b99b19a9af 100644 --- a/config-application-package/src/main/java/com/yahoo/config/application/XmlPreProcessor.java +++ b/config-application-package/src/main/java/com/yahoo/config/application/XmlPreProcessor.java @@ -43,15 +43,6 @@ public class XmlPreProcessor { private final Tags tags; private final List<PreProcessor> chain; - // TODO: Remove after November 2022 - public XmlPreProcessor(File applicationDir, - File xmlInput, - InstanceName instance, - Environment environment, - RegionName region) throws IOException { - this(applicationDir, new FileReader(xmlInput), instance, environment, region, Tags.empty()); - } - public XmlPreProcessor(File applicationDir, File xmlInput, InstanceName instance, diff --git a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/DeployData.java b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/DeployData.java index c3e9b99f562..de4ea62f671 100644 --- a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/DeployData.java +++ b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/DeployData.java @@ -2,7 +2,6 @@ package com.yahoo.config.model.application.provider; import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.Tags; import java.util.Set; @@ -15,8 +14,6 @@ public class DeployData { private final ApplicationId applicationId; - private final Tags tags; - /** The absolute path to the directory holding the application */ private final String deployedFromDir; @@ -32,14 +29,12 @@ public class DeployData { public DeployData(String deployedFromDir, ApplicationId applicationId, - Tags tags, Long deployTimestamp, boolean internalRedeploy, Long generation, long currentlyActiveGeneration) { this.deployedFromDir = deployedFromDir; this.applicationId = applicationId; - this.tags = tags; this.deployTimestamp = deployTimestamp; this.internalRedeploy = internalRedeploy; this.generation = generation; @@ -58,6 +53,4 @@ public class DeployData { public ApplicationId getApplicationId() { return applicationId; } - public Tags getTags() { return tags; } - } diff --git a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java index e61ea01a99a..c095b9ad586 100644 --- a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java +++ b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java @@ -11,7 +11,10 @@ import com.yahoo.config.application.api.ApplicationMetaData; import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.application.api.ComponentInfo; import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.config.application.api.DeploymentInstanceSpec; +import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.application.api.UnparsedConfigDefinition; +import com.yahoo.config.application.api.xml.DeploymentSpecXmlReader; import com.yahoo.config.codegen.DefParser; import com.yahoo.config.model.application.AbstractApplicationPackage; import com.yahoo.config.provision.ApplicationId; @@ -139,7 +142,6 @@ public class FilesApplicationPackage extends AbstractApplicationPackage { deployData.getDeployTimestamp(), deployData.isInternalRedeploy(), deployData.getApplicationId(), - deployData.getTags(), computeCheckSum(appDir), deployData.getGeneration(), deployData.getCurrentlyActiveGeneration()); @@ -486,7 +488,6 @@ public class FilesApplicationPackage extends AbstractApplicationPackage { ApplicationId.from(TenantName.defaultName(), ApplicationName.from(originalAppDir), InstanceName.defaultName()), - Tags.empty(), "", 0L, 0L); @@ -582,12 +583,16 @@ public class FilesApplicationPackage extends AbstractApplicationPackage { private void preprocessXML(File destination, File inputXml, Zone zone) throws IOException { if ( ! inputXml.exists()) return; try { + InstanceName instance = metaData.getApplicationId().instance(); Document document = new XmlPreProcessor(appDir, inputXml, - metaData.getApplicationId().instance(), + instance, zone.environment(), zone.region(), - metaData.getTags()) + getDeployment().map(new DeploymentSpecXmlReader(false)::read) + .flatMap(spec -> spec.instance(instance)) + .map(DeploymentInstanceSpec::tags) + .orElse(Tags.empty())) .run(); try (FileOutputStream outputStream = new FileOutputStream(destination)) { diff --git a/config-application-package/src/test/java/com/yahoo/config/application/HostedOverrideProcessorComplexTest.java b/config-application-package/src/test/java/com/yahoo/config/application/HostedOverrideProcessorComplexTest.java new file mode 100644 index 00000000000..93e038c786a --- /dev/null +++ b/config-application-package/src/test/java/com/yahoo/config/application/HostedOverrideProcessorComplexTest.java @@ -0,0 +1,121 @@ +package com.yahoo.config.application; + +import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.config.application.api.DeploymentInstanceSpec; +import com.yahoo.config.application.api.xml.DeploymentSpecXmlReader; +import com.yahoo.config.model.application.provider.FilesApplicationPackage; +import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.InstanceName; +import com.yahoo.config.provision.RegionName; +import com.yahoo.config.provision.Tags; +import org.junit.Test; +import org.w3c.dom.Document; + +import javax.xml.transform.TransformerException; +import java.io.File; +import java.io.IOException; + +import static org.junit.Assert.assertEquals; + +public class HostedOverrideProcessorComplexTest { + + private static final String servicesFile = "src/test/resources/complex-app/services.xml"; + + @Test + public void testProdBetaUsWest2a() throws TransformerException, IOException { + String expected = + """ + <?xml version="1.0" encoding="UTF-8" standalone="no"?> + <services xmlns:deploy="vespa" xmlns:preprocess="properties" version="1.0"> + <container id="docsgateway" version="1.0"> + <nodes count="3"> + <resources disk="1800Gb" disk-speed="fast" memory="96Gb" storage-type="local" vcpu="48"/> + </nodes> + </container> + <container id="qrs" version="1.0"> + <nodes count="3" required="true"> + <resources disk="64Gb" memory="32Gb" vcpu="16"/> + </nodes> + <search/> + </container> + <container id="visitor" version="1.0"> + <nodes count="2"> + <resources disk="32Gb" memory="16Gb" vcpu="8"/> + </nodes> + <search/> + </container> + <content id="all" version="1.0"> + <nodes count="3" groups="3" required="true"> + <resources disk="1800Gb" disk-speed="fast" memory="96Gb" storage-type="local" vcpu="48"/> + </nodes> + <redundancy>1</redundancy> + </content> + <content id="filedocument" version="1.0"> + <nodes count="2" groups="2"> + <resources disk="32Gb" memory="8Gb" vcpu="4"/> + </nodes> + <redundancy>1</redundancy> + </content> + </services> + """; + assertOverride(InstanceName.from("beta1"), + Environment.prod, + RegionName.from("aws-us-west-2a"), + expected); + } + + @Test + public void testProdBetaUsEast1b() throws TransformerException, IOException { + String expected = + """ + <?xml version="1.0" encoding="UTF-8" standalone="no"?> + <services xmlns:deploy="vespa" xmlns:preprocess="properties" version="1.0"> + <container id="docsgateway" version="1.0"> + <nodes count="3"> + <resources disk="1800Gb" disk-speed="fast" memory="96Gb" storage-type="local" vcpu="48"/> + </nodes> + </container> + <container id="qrs" version="1.0"> + <nodes count="5" required="true"> + <resources disk="64Gb" memory="32Gb" vcpu="16"/> + </nodes> + <search/> + </container> + <container id="visitor" version="1.0"> + <nodes count="2"> + <resources disk="32Gb" memory="16Gb" vcpu="8"/> + </nodes> + <search/> + </container> + <content id="all" version="1.0"> + <nodes count="10" groups="10" required="true"> + <resources disk="1800Gb" disk-speed="fast" memory="96Gb" storage-type="local" vcpu="48"/> + </nodes> + <redundancy>1</redundancy> + </content> + <content id="filedocument" version="1.0"> + <nodes count="2" groups="2"> + <resources disk="32Gb" memory="8Gb" vcpu="4"/> + </nodes> + <redundancy>1</redundancy> + </content> + </services> + """; + assertOverride(InstanceName.from("beta1"), + Environment.prod, + RegionName.from("aws-us-east-1b"), + expected); + } + + private void assertOverride(InstanceName instance, Environment environment, RegionName region, String expected) throws TransformerException { + ApplicationPackage app = FilesApplicationPackage.fromFile(new File(servicesFile).getParentFile()); + Document inputDoc = Xml.getDocument(app.getServices()); + Tags tags = app.getDeployment() + .map(new DeploymentSpecXmlReader(false)::read) + .flatMap(spec -> spec.instance(instance).map(DeploymentInstanceSpec::tags)) + .orElse(Tags.empty()); + Document newDoc = new OverrideProcessor(instance, environment, region, tags).process(inputDoc); + assertEquals(expected, Xml.documentAsString(newDoc, true)); + } + +} diff --git a/config-application-package/src/test/java/com/yahoo/config/application/HostedOverrideProcessorTagsTest.java b/config-application-package/src/test/java/com/yahoo/config/application/HostedOverrideProcessorTagsTest.java index 8d7431d33b6..b8b09396fb4 100644 --- a/config-application-package/src/test/java/com/yahoo/config/application/HostedOverrideProcessorTagsTest.java +++ b/config-application-package/src/test/java/com/yahoo/config/application/HostedOverrideProcessorTagsTest.java @@ -5,7 +5,6 @@ import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.Tags; -import org.custommonkey.xmlunit.XMLUnit; import org.junit.Test; import org.w3c.dom.Document; @@ -17,10 +16,6 @@ import java.io.StringReader; */ public class HostedOverrideProcessorTagsTest { - static { - XMLUnit.setIgnoreWhitespace(true); - } - private static final String input = "<?xml version='1.0' encoding='UTF-8' standalone='no'?>" + "<services xmlns:deploy='vespa' xmlns:preprocess='?' version='1.0'>" + diff --git a/config-application-package/src/test/java/com/yahoo/config/application/HostedOverrideProcessorTest.java b/config-application-package/src/test/java/com/yahoo/config/application/HostedOverrideProcessorTest.java index 451c7a3c217..eb65c01522e 100644 --- a/config-application-package/src/test/java/com/yahoo/config/application/HostedOverrideProcessorTest.java +++ b/config-application-package/src/test/java/com/yahoo/config/application/HostedOverrideProcessorTest.java @@ -5,51 +5,42 @@ import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.Tags; -import org.custommonkey.xmlunit.XMLUnit; import org.junit.Test; import org.w3c.dom.Document; -import org.xml.sax.SAXException; -import javax.xml.parsers.ParserConfigurationException; -import javax.xml.stream.XMLStreamException; import javax.xml.transform.TransformerException; -import java.io.IOException; import java.io.StringReader; -import java.util.List; /** * @author bratseth */ public class HostedOverrideProcessorTest { - static { - XMLUnit.setIgnoreWhitespace(true); - } - private static final String input = - "<?xml version='1.0' encoding='UTF-8' standalone='no'?>" + - "<services xmlns:deploy='vespa' xmlns:preprocess='?' version='1.0'>" + - " <container id='foo' version='1.0'>" + - " <nodes count='1'/>" + - " <nodes count='3' deploy:environment='perf'/>" + - " <nodes deploy:environment='staging' count='2' required='true'/>" + - " <nodes deploy:environment='prod' count='3' flavor='v-4-8-100'/>" + - " <nodes deploy:environment='prod' deploy:region='us-west' count='4'/>" + - " <nodes deploy:environment='prod' deploy:region='us-east-3' flavor='v-8-8-100' count='5'/>" + - " <nodes deploy:instance='myinstance' deploy:environment='prod' deploy:region='us-west' count='1'/>" + - " </container>" + - "</services>"; + """ + <?xml version='1.0' encoding='UTF-8' standalone='no'?> + <services xmlns:deploy='vespa' xmlns:preprocess='?' version='1.0'> + <container id='foo' version='1.0'> + <nodes count='1'/> + <nodes count='3' deploy:environment='perf'/> + <nodes deploy:environment='staging' count='2' required='true'/> + <nodes deploy:environment='prod' count='3' flavor='v-4-8-100'/> + <nodes deploy:environment='prod' deploy:region='us-west' count='4'/> + <nodes deploy:environment='prod' deploy:region='us-east-3' flavor='v-8-8-100' count='5'/> + <nodes deploy:instance='myinstance' deploy:environment='prod' deploy:region='us-west' count='1'/> + </container></services>"""; @Test public void testParsingDefault() throws TransformerException { - String expected = - "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>" + - "<services xmlns:deploy=\"vespa\" xmlns:preprocess=\"?\" version=\"1.0\">" + - " <container id=\"foo\" version=\"1.0\">" + - " <nodes count='1'/>" + - " </container>" + - "</services>"; + String expected = + """ + <?xml version="1.0" encoding="UTF-8" standalone="no"?> + <services xmlns:deploy="vespa" xmlns:preprocess="?" version="1.0"> + <container id="foo" version="1.0"> + <nodes count='1'/> + </container> + </services>"""; assertOverride(InstanceName.defaultName(), Environment.test, RegionName.defaultName(), @@ -59,13 +50,14 @@ public class HostedOverrideProcessorTest { @Test public void testParsingEnvironmentAndRegion() throws TransformerException { - String expected = - "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>" + - "<services xmlns:deploy=\"vespa\" xmlns:preprocess=\"?\" version=\"1.0\">" + - " <container id=\"foo\" version=\"1.0\">" + - " <nodes count='4' required='true'/>" + - " </container>" + - "</services>"; + String expected = + """ + <?xml version="1.0" encoding="UTF-8" standalone="no"?> + <services xmlns:deploy="vespa" xmlns:preprocess="?" version="1.0"> + <container id="foo" version="1.0"> + <nodes count='4' required='true'/> + </container> + </services>"""; assertOverride(InstanceName.defaultName(), Environment.from("prod"), RegionName.from("us-west"), diff --git a/config-application-package/src/test/java/com/yahoo/config/application/IncludeProcessorTest.java b/config-application-package/src/test/java/com/yahoo/config/application/IncludeProcessorTest.java index 3de624c78ac..697d8c208d3 100644 --- a/config-application-package/src/test/java/com/yahoo/config/application/IncludeProcessorTest.java +++ b/config-application-package/src/test/java/com/yahoo/config/application/IncludeProcessorTest.java @@ -2,7 +2,6 @@ package com.yahoo.config.application; import com.yahoo.config.application.api.ApplicationPackage; -import org.junit.Assert; import org.junit.Test; import org.w3c.dom.Document; import org.xml.sax.SAXException; @@ -28,54 +27,58 @@ public class IncludeProcessorTest { DocumentBuilder docBuilder = Xml.getPreprocessDocumentBuilder(); String expected = - "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>" + - "<services xmlns:deploy=\"vespa\" xmlns:preprocess=\"properties\" version=\"1.0\">\n" + - " <preprocess:properties>\n" + - " <qrs.port>4099</qrs.port>\n" + - " <qrs.port>5000</qrs.port>\n" + - " </preprocess:properties>\n" + - " <preprocess:properties deploy:environment='prod'>\n" + - " <qrs.port deploy:region='us-west'>5001</qrs.port>" + - " <qrs.port deploy:region='us-east us-central'>5002</qrs.port>" + - " </preprocess:properties>\n" + - " <admin version=\"2.0\">\n" + - " <adminserver hostalias=\"node0\"/>\n" + - " </admin>\n" + - " <admin deploy:environment=\"staging prod\" deploy:region=\"us-east us-central\" version=\"2.0\">\n" + - " <adminserver hostalias=\"node1\"/>\n" + - " </admin>\n" + - " <content id=\"foo\" version=\"1.0\">\n" + - " <redundancy>1</redundancy><documents>\n" + - " <document mode=\"index\" type=\"music.sd\"/>\n" + - " </documents><nodes>\n" + - " <node distribution-key=\"0\" hostalias=\"node0\"/>\n" + - " </nodes>" + - " <nodes deploy:environment=\"prod\">\n" + - " <node distribution-key=\"0\" hostalias=\"node0\"/>\n" + - " <node distribution-key=\"1\" hostalias=\"node1\"/>\n" + - " </nodes>" + - " <nodes deploy:environment=\"prod\" deploy:region=\"us-west\">\n" + - " <node distribution-key=\"0\" hostalias=\"node0\"/>\n" + - " <node distribution-key=\"1\" hostalias=\"node1\"/>\n" + - " <node distribution-key=\"2\" hostalias=\"node2\"/>\n" + - " </nodes>" + - "</content>\n" + - "<container id=\"stateless\" version=\"1.0\">\n" + - " <search deploy:environment=\"prod\">\n" + - " <chain id=\"common\">\n" + - " <searcher id=\"MySearcher1\" />\n" + - " <searcher deploy:environment=\"prod\" id=\"MySearcher2\" />\n" + - " </chain>\n" + - " </search>\n" + - " <search/>\n" + - " <component id=\"foo\" class=\"MyFoo\" bundle=\"foobundle\" />\n" + - " <component id=\"bar\" class=\"TestBar\" bundle=\"foobundle\" deploy:environment=\"dev\" />\n" + - " <component id=\"bar\" class=\"ProdBar\" bundle=\"foobundle\" deploy:environment=\"prod\" />\n" + - " <component id=\"baz\" class=\"ProdBaz\" bundle=\"foobundle\" deploy:environment=\"prod\" />\n" + - " <nodes>\n" + - " <node baseport=\"${qrs.port}\" hostalias=\"node0\"/>\n" + - " </nodes>\n" + - "</container></services>"; + """ + <?xml version="1.0" encoding="UTF-8" standalone="no"?> + <!-- Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --><services xmlns:deploy="vespa" xmlns:preprocess="properties" version="1.0"> + <preprocess:properties> + <qrs.port>4099</qrs.port> + <qrs.port>5000</qrs.port> + </preprocess:properties> + <preprocess:properties deploy:environment="prod"> + <qrs.port deploy:region="us-west">5001</qrs.port> + <qrs.port deploy:region="us-east us-central">5002</qrs.port> + </preprocess:properties> + <admin version="2.0"> + <adminserver hostalias="node0"/> + </admin> + <admin deploy:environment="staging prod" deploy:region="us-east us-central" version="2.0"> + <adminserver hostalias="node1"/> + </admin> + <content id="foo" version="1.0"> + <redundancy>1</redundancy> + <documents> + <document mode="index" type="music.sd"/> + </documents> + <nodes> + <node distribution-key="0" hostalias="node0"/> + </nodes> + <nodes deploy:environment="prod"> + <node distribution-key="0" hostalias="node0"/> + <node distribution-key="1" hostalias="node1"/> + </nodes> + <nodes deploy:environment="prod" deploy:region="us-west"> + <node distribution-key="0" hostalias="node0"/> + <node distribution-key="1" hostalias="node1"/> + <node distribution-key="2" hostalias="node2"/> + </nodes> + </content> + <container id="stateless" version="1.0"> + <search deploy:environment="prod"> + <chain id="common"> + <searcher id="MySearcher1"/> + <searcher deploy:environment="prod" id="MySearcher2"/> + </chain> + </search> + <search/> + <component bundle="foobundle" class="MyFoo" id="foo"/> + <component bundle="foobundle" class="TestBar" deploy:environment="dev" id="bar"/> + <component bundle="foobundle" class="ProdBar" deploy:environment="prod" id="bar"/> + <component bundle="foobundle" class="ProdBaz" deploy:environment="prod" id="baz"/> + <nodes> + <node baseport="${qrs.port}" hostalias="node0"/> + </nodes> + </container> + </services>"""; Document doc = new IncludeProcessor(app).process(docBuilder.parse(getServices(app))); // System.out.println(Xml.documentAsString(doc)); diff --git a/config-application-package/src/test/java/com/yahoo/config/application/MultiOverrideProcessorTest.java b/config-application-package/src/test/java/com/yahoo/config/application/MultiOverrideProcessorTest.java index debde6c1438..8602b2955aa 100644 --- a/config-application-package/src/test/java/com/yahoo/config/application/MultiOverrideProcessorTest.java +++ b/config-application-package/src/test/java/com/yahoo/config/application/MultiOverrideProcessorTest.java @@ -5,7 +5,6 @@ import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.Tags; -import org.custommonkey.xmlunit.XMLUnit; import org.junit.Test; import org.w3c.dom.Document; @@ -19,107 +18,109 @@ import java.io.StringReader; */ public class MultiOverrideProcessorTest { - static { - XMLUnit.setIgnoreWhitespace(true); - } - private static final String input = - "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" + - "<services version=\"1.0\" xmlns:deploy=\"vespa\">\n" + - " <container id='default' version='1.0'>\n" + - " <component id=\"comp-B\" class=\"com.yahoo.ls.MyComponent\" bundle=\"lsbe-hv\">\n" + - " <config name=\"ls.config.resource-pool\">\n" + - " <resource>\n" + - " <item>\n" + - " <id>comp-B-item-0</id>\n" + - " <type></type>\n" + - " </item>\n" + - " <item deploy:environment=\"dev perf test staging prod\" deploy:region=\"us-west-1 us-east-3\">\n" + - " <id>comp-B-item-1</id>\n" + - " <type></type>\n" + - " </item>\n" + - " <item>\n" + - " <id>comp-B-item-2</id>\n" + - " <type></type>\n" + - " </item>\n" + - " </resource>\n" + - " </config>\n" + - " </component>\n" + - " </container>\n" + - "</services>\n"; + """ + <?xml version="1.0" encoding="UTF-8"?> + <services version="1.0" xmlns:deploy="vespa"> + <container id='default' version='1.0'> + <component id="comp-B" class="com.yahoo.ls.MyComponent" bundle="lsbe-hv"> + <config name="ls.config.resource-pool"> + <resource> + <item> + <id>comp-B-item-0</id> + <type></type> + </item> + <item deploy:environment="dev perf test staging prod" deploy:region="us-west-1 us-east-3"> + <id>comp-B-item-1</id> + <type></type> + </item> + <item> + <id>comp-B-item-2</id> + <type></type> + </item> + </resource> + </config> + </component> + </container> + </services> + """; private static final String inputWithIds = - "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" + - "<services version=\"1.0\" xmlns:deploy=\"vespa\">\n" + - " <container id='default' version='1.0'>\n" + - " <component id=\"comp-B\" class=\"com.yahoo.ls.MyComponent\" bundle=\"lsbe-hv\">\n" + - " <config name=\"ls.config.resource-pool\">\n" + - " <resource>\n" + - " <item id='1'>\n" + - " <id>comp-B-item-0</id>\n" + - " <type></type>\n" + - " </item>\n" + - " <item id='2' deploy:environment=\"dev perf test staging prod\" deploy:region=\"us-west-1 us-east-3\">\n" + - " <id>comp-B-item-1</id>\n" + - " <type></type>\n" + - " </item>\n" + - " <item id='3'>\n" + - " <id>comp-B-item-2</id>\n" + - " <type></type>\n" + - " </item>\n" + - " </resource>\n" + - " </config>\n" + - " </component>\n" + - " </container>\n" + - "</services>\n"; + """ + <?xml version="1.0" encoding="UTF-8"?> + <services version="1.0" xmlns:deploy="vespa"> + <container id='default' version='1.0'> + <component id="comp-B" class="com.yahoo.ls.MyComponent" bundle="lsbe-hv"> + <config name="ls.config.resource-pool"> + <resource> + <item id='1'> + <id>comp-B-item-0</id> + <type></type> + </item> + <item id='2' deploy:environment="dev perf test staging prod" deploy:region="us-west-1 us-east-3"> + <id>comp-B-item-1</id> + <type></type> + </item> + <item id='3'> + <id>comp-B-item-2</id> + <type></type> + </item> + </resource> + </config> + </component> + </container> + </services> + """; @Test public void testParsingDev() throws TransformerException { String expected = - "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" + - "<services version=\"1.0\" xmlns:deploy=\"vespa\">\n" + - " <container id='default' version='1.0'>\n" + - " <component id=\"comp-B\" class=\"com.yahoo.ls.MyComponent\" bundle=\"lsbe-hv\">\n" + - " <config name=\"ls.config.resource-pool\">\n" + - " <resource>\n" + - " <item>\n" + - " <id>comp-B-item-1</id>\n" + - " <type></type>\n" + - " </item>\n" + - " </resource>\n" + - " </config>\n" + - " </component>\n" + - " </container>\n" + - "</services>"; + """ + <?xml version="1.0" encoding="UTF-8"?> + <services version="1.0" xmlns:deploy="vespa"> + <container id='default' version='1.0'> + <component id="comp-B" class="com.yahoo.ls.MyComponent" bundle="lsbe-hv"> + <config name="ls.config.resource-pool"> + <resource> + <item> + <id>comp-B-item-1</id> + <type></type> + </item> + </resource> + </config> + </component> + </container> + </services>"""; assertOverride(Environment.dev, RegionName.from("us-east-3"), expected); } @Test public void testParsingDevWithIds() throws TransformerException { String expected = - "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" + - "<services version=\"1.0\" xmlns:deploy=\"vespa\">\n" + - " <container id='default' version='1.0'>\n" + - " <component id=\"comp-B\" class=\"com.yahoo.ls.MyComponent\" bundle=\"lsbe-hv\">\n" + - " <config name=\"ls.config.resource-pool\">\n" + - " <resource>\n" + - " <item id='1'>\n" + - " <id>comp-B-item-0</id>\n" + - " <type></type>\n" + - " </item>\n" + - " <item id='2'>\n" + - " <id>comp-B-item-1</id>\n" + - " <type></type>\n" + - " </item>\n" + - " <item id='3'>\n" + - " <id>comp-B-item-2</id>\n" + - " <type></type>\n" + - " </item>\n" + - " </resource>\n" + - " </config>\n" + - " </component>\n" + - " </container>\n" + - "</services>"; + """ + <?xml version="1.0" encoding="UTF-8"?> + <services version="1.0" xmlns:deploy="vespa"> + <container id='default' version='1.0'> + <component id="comp-B" class="com.yahoo.ls.MyComponent" bundle="lsbe-hv"> + <config name="ls.config.resource-pool"> + <resource> + <item id='1'> + <id>comp-B-item-0</id> + <type></type> + </item> + <item id='2'> + <id>comp-B-item-1</id> + <type></type> + </item> + <item id='3'> + <id>comp-B-item-2</id> + <type></type> + </item> + </resource> + </config> + </component> + </container> + </services>"""; assertOverrideWithIds(Environment.dev, RegionName.from("us-east-3"), expected); } diff --git a/config-application-package/src/test/java/com/yahoo/config/application/OverrideProcessorTest.java b/config-application-package/src/test/java/com/yahoo/config/application/OverrideProcessorTest.java index 150999390d8..ca074ef9704 100644 --- a/config-application-package/src/test/java/com/yahoo/config/application/OverrideProcessorTest.java +++ b/config-application-package/src/test/java/com/yahoo/config/application/OverrideProcessorTest.java @@ -5,7 +5,6 @@ import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.Tags; -import org.custommonkey.xmlunit.XMLUnit; import org.junit.Test; import org.w3c.dom.Document; @@ -17,10 +16,6 @@ import java.io.StringReader; */ public class OverrideProcessorTest { - static { - XMLUnit.setIgnoreWhitespace(true); - } - private static final String input = "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>" + "<services xmlns:deploy=\"vespa\" xmlns:preprocess=\"?\" version=\"1.0\">" + diff --git a/config-application-package/src/test/java/com/yahoo/config/application/PropertiesProcessorTest.java b/config-application-package/src/test/java/com/yahoo/config/application/PropertiesProcessorTest.java index 3bf0f25766c..b8896ca2b79 100644 --- a/config-application-package/src/test/java/com/yahoo/config/application/PropertiesProcessorTest.java +++ b/config-application-package/src/test/java/com/yahoo/config/application/PropertiesProcessorTest.java @@ -1,7 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.config.application; -import org.custommonkey.xmlunit.XMLUnit; import org.junit.Test; import org.w3c.dom.Document; @@ -16,25 +15,22 @@ import static org.junit.Assert.assertEquals; */ public class PropertiesProcessorTest { - static { - XMLUnit.setIgnoreWhitespace(true); - } - @Test public void testPropertyValues() throws TransformerException { - String input = "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>" + - "<services xmlns:deploy=\"vespa\" xmlns:preprocess=\"properties\" version=\"1.0\">" + - " <preprocess:properties>" + - " <slobrok.port>4099</slobrok.port>" + - " <redundancy>2</redundancy>" + - " </preprocess:properties>" + - " <admin version=\"2.0\">" + - " <adminserver hostalias=\"node0\"/>" + - " <slobroks>" + - " <slobrok hostalias=\"node1\" baseport=\"${slobrok.port}\"/>" + - " </slobroks>" + - " </admin>" + - "</services>"; + String input = """ + <?xml version="1.0" encoding="UTF-8" standalone="no"?> + <services xmlns:deploy="vespa" xmlns:preprocess="properties" version="1.0"> + <preprocess:properties> + <slobrok.port>4099</slobrok.port> + <redundancy>2</redundancy> + </preprocess:properties> + <admin version="2.0"> + <adminserver hostalias="node0"/> + <slobroks> + <slobrok hostalias="node1" baseport="${slobrok.port}"/> + </slobroks> + </admin> + </services>"""; PropertiesProcessor p = new PropertiesProcessor(); p.process(Xml.getDocument(new StringReader(input))); @@ -46,49 +42,51 @@ public class PropertiesProcessorTest { @Test public void testPropertyApplying() throws TransformerException { - String input = "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>" + - "<services xmlns:deploy=\"vespa\" xmlns:preprocess=\"properties\" version=\"1.0\">" + - " <preprocess:properties>" + - " <slobrok.port>4099</slobrok.port>" + - " <redundancy>2</redundancy>" + - " <doctype>music</doctype>" + - " <zero>0</zero>" + - " </preprocess:properties>" + - " <admin version=\"2.0\">" + - " <adminserver hostalias=\"node0\"/>" + - " <slobroks>" + - " <slobrok hostalias=\"node1\" baseport=\"${slobrok.port}\"/>" + - " </slobroks>" + - " </admin>" + - " <content id=\"foo\" version=\"1.0\">" + - " <redundancy>${redundancy}</redundancy>" + - " <documents>" + - " <document mode=\"index\" type=\"${doctype}.sd\"/>" + - " </documents>" + - " <nodes>" + - " <node distribution-key=\"${zero}\" hostalias=\"node${zero}\"/>" + - " </nodes>" + - " </content>" + - "</services>"; - - String expected = "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>" + - "<services xmlns:deploy=\"vespa\" xmlns:preprocess=\"properties\" version=\"1.0\">" + - " <admin version=\"2.0\">" + - " <adminserver hostalias=\"node0\"/>" + - " <slobroks>" + - " <slobrok hostalias=\"node1\" baseport=\"4099\"/>" + - " </slobroks>" + - " </admin>" + - " <content id=\"foo\" version=\"1.0\">" + - " <redundancy>2</redundancy>" + - " <documents>" + - " <document mode=\"index\" type=\"music.sd\"/>" + - " </documents>" + - " <nodes>" + - " <node distribution-key=\"0\" hostalias=\"node0\"/>" + - " </nodes>" + - " </content>" + - "</services>"; + String input = """ + <?xml version="1.0" encoding="UTF-8" standalone="no"?> + <services xmlns:deploy="vespa" xmlns:preprocess="properties" version="1.0"> + <preprocess:properties> + <slobrok.port>4099</slobrok.port> + <redundancy>2</redundancy> + <doctype>music</doctype> + <zero>0</zero> + </preprocess:properties> + <admin version="2.0"> + <adminserver hostalias="node0"/> + <slobroks> + <slobrok hostalias="node1" baseport="${slobrok.port}"/> + </slobroks> + </admin> + <content id="foo" version="1.0"> + <redundancy>${redundancy}</redundancy> + <documents> + <document mode="index" type="${doctype}.sd"/> + </documents> + <nodes> + <node distribution-key="${zero}" hostalias="node${zero}"/> + </nodes> + </content> + </services>"""; + + String expected = """ + <?xml version="1.0" encoding="UTF-8" standalone="no"?> + <services xmlns:deploy="vespa" xmlns:preprocess="properties" version="1.0"> + <admin version="2.0"> + <adminserver hostalias="node0"/> + <slobroks> + <slobrok hostalias="node1" baseport="4099"/> + </slobroks> + </admin> + <content id="foo" version="1.0"> + <redundancy>2</redundancy> + <documents> + <document mode="index" type="music.sd"/> + </documents> + <nodes> + <node distribution-key="0" hostalias="node0"/> + </nodes> + </content> + </services>"""; Document inputDoc = Xml.getDocument(new StringReader(input)); @@ -100,31 +98,34 @@ public class PropertiesProcessorTest { // TODO: Check that warning is actually logged @Test public void testWarnIfDuplicatePropertyForSameEnvironment() throws TransformerException { - String input = "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>" + - "<services xmlns:deploy=\"vespa\" xmlns:preprocess=\"properties\" version=\"1.0\">" + - " <preprocess:properties>" + - " <slobrok.port>4099</slobrok.port>" + - " <slobrok.port>5000</slobrok.port>" + - " <redundancy>2</redundancy>" + - " </preprocess:properties>" + - " <admin version=\"2.0\">" + - " <adminserver hostalias=\"node0\"/>" + - " <slobroks>" + - " <slobrok hostalias=\"node1\" baseport=\"${slobrok.port}\"/>" + - " </slobroks>" + - " </admin>" + - "</services>"; - - - String expected = "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>" + - "<services xmlns:deploy=\"vespa\" xmlns:preprocess=\"properties\" version=\"1.0\">" + - " <admin version=\"2.0\">" + - " <adminserver hostalias=\"node0\"/>" + - " <slobroks>" + - " <slobrok hostalias=\"node1\" baseport=\"5000\"/>" + // Should get the last defined value - " </slobroks>" + - " </admin>" + - "</services>"; + String input = """ + <?xml version="1.0" encoding="UTF-8" standalone="no"?> + <services xmlns:deploy="vespa" xmlns:preprocess="properties" version="1.0"> + <preprocess:properties> + <slobrok.port>4099</slobrok.port> + <slobrok.port>5000</slobrok.port> + <redundancy>2</redundancy> + </preprocess:properties> + <admin version="2.0"> + <adminserver hostalias="node0"/> + <slobroks> + <slobrok hostalias="node1" baseport="${slobrok.port}"/> + </slobroks> + </admin> + </services>"""; + + + // Should get the last defined value + String expected = """ + <?xml version="1.0" encoding="UTF-8" standalone="no"?> + <services xmlns:deploy="vespa" xmlns:preprocess="properties" version="1.0"> + <admin version="2.0"> + <adminserver hostalias="node0"/> + <slobroks> + <slobrok hostalias="node1" baseport="5000"/> + </slobroks> + </admin> + </services>"""; Document inputDoc = Xml.getDocument(new StringReader(input)); Document newDoc = new PropertiesProcessor().process(inputDoc); diff --git a/config-application-package/src/test/java/com/yahoo/config/application/TestBase.java b/config-application-package/src/test/java/com/yahoo/config/application/TestBase.java index 06172c2698f..7e3a4758f8e 100644 --- a/config-application-package/src/test/java/com/yahoo/config/application/TestBase.java +++ b/config-application-package/src/test/java/com/yahoo/config/application/TestBase.java @@ -1,14 +1,13 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.config.application; -import org.custommonkey.xmlunit.Diff; -import org.custommonkey.xmlunit.XMLUnit; import org.w3c.dom.Document; +import javax.xml.transform.TransformerException; import java.io.Reader; import java.io.StringReader; -import static org.junit.Assert.assertTrue; +import static org.junit.Assert.assertEquals; /** * Utilities for tests @@ -16,14 +15,15 @@ import static org.junit.Assert.assertTrue; * @author hmusum */ public class TestBase { - static { - XMLUnit.setIgnoreWhitespace(true); - } static void assertDocument(String expected, Document output) { - Document expectedDoc = Xml.getDocument(new StringReader(expected)); - Diff diff = new Diff(expectedDoc, output); - assertTrue(diff.toString(), diff.identical()); + try { + assertEquals(Xml.documentAsString(Xml.getDocument(new StringReader(expected)), true), + Xml.documentAsString(output, true)); + } + catch (TransformerException e) { + throw new AssertionError(e); + } } public static void assertDocument(String expectedDocument, Reader document) { diff --git a/config-application-package/src/test/java/com/yahoo/config/application/XmlPreprocessorTest.java b/config-application-package/src/test/java/com/yahoo/config/application/XmlPreprocessorTest.java index 0da94b69e58..49dbacbae3d 100644 --- a/config-application-package/src/test/java/com/yahoo/config/application/XmlPreprocessorTest.java +++ b/config-application-package/src/test/java/com/yahoo/config/application/XmlPreprocessorTest.java @@ -23,29 +23,31 @@ public class XmlPreprocessorTest { @Test public void testPreProcessing() throws Exception { String expectedDev = - "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>" + - "<services xmlns:deploy=\"vespa\" xmlns:preprocess=\"properties\" version=\"1.0\">\n" + - " <admin version=\"2.0\">\n" + - " <adminserver hostalias=\"node0\"/>\n" + - " </admin>\n" + - " <content id=\"foo\" version=\"1.0\">\n" + - " <redundancy>1</redundancy>\n" + - " <documents>\n" + - " <document mode=\"index\" type=\"music.sd\"/>\n" + - " </documents>\n" + - " <nodes>\n" + - " <node distribution-key=\"0\" hostalias=\"node0\"/>\n" + - " </nodes>\n" + - " </content>\n" + - " <container id=\"stateless\" version=\"1.0\">\n" + - " <search/>\n" + - " <component bundle=\"foobundle\" class=\"MyFoo\" id=\"foo\"/>\n" + - " <component bundle=\"foobundle\" class=\"TestBar\" id=\"bar\"/>\n" + - " <nodes>\n" + - " <node hostalias=\"node0\" baseport=\"5000\"/>\n" + - " </nodes>\n" + - " </container>\n" + - "</services>"; + """ + <?xml version="1.0" encoding="UTF-8" standalone="no"?> + <!-- Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> + <services xmlns:deploy="vespa" xmlns:preprocess="properties" version="1.0"> + <admin version="2.0"> + <adminserver hostalias="node0"/> + </admin> + <content id="foo" version="1.0"> + <redundancy>1</redundancy> + <documents> + <document mode="index" type="music.sd"/> + </documents> + <nodes> + <node distribution-key="0" hostalias="node0"/> + </nodes> + </content> + <container id="stateless" version="1.0"> + <search/> + <component bundle="foobundle" class="MyFoo" id="foo"/> + <component bundle="foobundle" class="TestBar" id="bar"/> + <nodes> + <node hostalias="node0" baseport="5000"/> + </nodes> + </container> + </services>"""; TestBase.assertDocument(expectedDev, new XmlPreProcessor(appDir, services, @@ -54,30 +56,33 @@ public class XmlPreprocessorTest { RegionName.defaultName(), Tags.empty()).run()); + // Difference from dev: node1 + // Difference from dev: no TestBar String expectedStaging = - "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>" + - "<services xmlns:deploy=\"vespa\" xmlns:preprocess=\"properties\" version=\"1.0\">\n" + - " <admin version=\"2.0\">\n" + - " <adminserver hostalias=\"node1\"/>\n" + // Difference from dev: node1 - " </admin>\n" + - " <content id=\"foo\" version=\"1.0\">\n" + - " <redundancy>1</redundancy>\n" + - " <documents>\n" + - " <document mode=\"index\" type=\"music.sd\"/>\n" + - " </documents>\n" + - " <nodes>\n" + - " <node distribution-key=\"0\" hostalias=\"node0\"/>\n" + - " </nodes>\n" + - " </content>\n" + - " <container id=\"stateless\" version=\"1.0\">\n" + - " <search/>\n" + - " <component bundle=\"foobundle\" class=\"MyFoo\" id=\"foo\"/>\n" + - "" + // Difference from dev: no TestBar - " <nodes>\n" + - " <node hostalias=\"node0\" baseport=\"5000\"/>\n" + - " </nodes>\n" + - " </container>\n" + - "</services>"; + """ + <?xml version="1.0" encoding="UTF-8" standalone="no"?> + <!-- Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> + <services xmlns:deploy="vespa" xmlns:preprocess="properties" version="1.0"> + <admin version="2.0"> + <adminserver hostalias="node1"/> + </admin> + <content id="foo" version="1.0"> + <redundancy>1</redundancy> + <documents> + <document mode="index" type="music.sd"/> + </documents> + <nodes> + <node distribution-key="0" hostalias="node0"/> + </nodes> + </content> + <container id="stateless" version="1.0"> + <search/> + <component bundle="foobundle" class="MyFoo" id="foo"/> + <nodes> + <node hostalias="node0" baseport="5000"/> + </nodes> + </container> + </services>"""; TestBase.assertDocument(expectedStaging, new XmlPreProcessor(appDir, services, @@ -87,37 +92,39 @@ public class XmlPreprocessorTest { Tags.empty()).run()); String expectedUsWest = - "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>" + - "<services xmlns:deploy=\"vespa\" xmlns:preprocess=\"properties\" version=\"1.0\">\n" + - " <admin version=\"2.0\">\n" + - " <adminserver hostalias=\"node0\"/>\n" + - " </admin>\n" + - " <content id=\"foo\" version=\"1.0\">\n" + - " <redundancy>1</redundancy>\n" + - " <documents>\n" + - " <document mode=\"index\" type=\"music.sd\"/>\n" + - " </documents>\n" + - " <nodes>\n" + - " <node distribution-key=\"0\" hostalias=\"node0\"/>\n" + - " <node distribution-key=\"1\" hostalias=\"node1\"/>\n" + - " <node distribution-key=\"2\" hostalias=\"node2\"/>\n" + - " </nodes>\n" + - " </content>\n" + - " <container id=\"stateless\" version=\"1.0\">\n" + - " <search>\n" + - " <chain id=\"common\">\n" + - " <searcher id=\"MySearcher1\"/>\n" + - " <searcher id=\"MySearcher2\"/>\n" + - " </chain>\n" + - " </search>\n" + - " <component bundle=\"foobundle\" class=\"MyFoo\" id=\"foo\"/>\n" + - " <component bundle=\"foobundle\" class=\"ProdBar\" id=\"bar\"/>\n" + - " <component bundle=\"foobundle\" class=\"ProdBaz\" id=\"baz\"/>\n" + - " <nodes>\n" + - " <node hostalias=\"node0\" baseport=\"5001\"/>\n" + - " </nodes>\n" + - " </container>\n" + - "</services>"; + """ + <?xml version="1.0" encoding="UTF-8" standalone="no"?> + <!-- Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> + <services xmlns:deploy="vespa" xmlns:preprocess="properties" version="1.0"> + <admin version="2.0"> + <adminserver hostalias="node0"/> + </admin> + <content id="foo" version="1.0"> + <redundancy>1</redundancy> + <documents> + <document mode="index" type="music.sd"/> + </documents> + <nodes> + <node distribution-key="0" hostalias="node0"/> + <node distribution-key="1" hostalias="node1"/> + <node distribution-key="2" hostalias="node2"/> + </nodes> + </content> + <container id="stateless" version="1.0"> + <search> + <chain id="common"> + <searcher id="MySearcher1"/> + <searcher id="MySearcher2"/> + </chain> + </search> + <component bundle="foobundle" class="MyFoo" id="foo"/> + <component bundle="foobundle" class="ProdBar" id="bar"/> + <component bundle="foobundle" class="ProdBaz" id="baz"/> + <nodes> + <node hostalias="node0" baseport="5001"/> + </nodes> + </container> + </services>"""; TestBase.assertDocument(expectedUsWest, new XmlPreProcessor(appDir, services, @@ -127,36 +134,38 @@ public class XmlPreprocessorTest { Tags.empty()).run()); String expectedUsEastAndCentral = - "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>" + - "<services xmlns:deploy=\"vespa\" xmlns:preprocess=\"properties\" version=\"1.0\">\n" + - " <admin version=\"2.0\">\n" + - " <adminserver hostalias=\"node1\"/>\n" + - " </admin>\n" + - " <content id=\"foo\" version=\"1.0\">\n" + - " <redundancy>1</redundancy>\n" + - " <documents>\n" + - " <document mode=\"index\" type=\"music.sd\"/>\n" + - " </documents>\n" + - " <nodes>\n" + - " <node distribution-key=\"0\" hostalias=\"node0\"/>\n" + - " <node distribution-key=\"1\" hostalias=\"node1\"/>\n" + - " </nodes>\n" + - " </content>\n" + - " <container id=\"stateless\" version=\"1.0\">\n" + - " <search>\n" + - " <chain id=\"common\">\n" + - " <searcher id=\"MySearcher1\"/>\n" + - " <searcher id=\"MySearcher2\"/>\n" + - " </chain>\n" + - " </search>\n" + - " <component bundle=\"foobundle\" class=\"MyFoo\" id=\"foo\"/>\n" + - " <component bundle=\"foobundle\" class=\"ProdBar\" id=\"bar\"/>\n" + - " <component bundle=\"foobundle\" class=\"ProdBaz\" id=\"baz\"/>\n" + - " <nodes>\n" + - " <node hostalias=\"node0\" baseport=\"5002\"/>\n" + - " </nodes>\n" + - " </container>\n" + - "</services>"; + """ + <?xml version="1.0" encoding="UTF-8" standalone="no"?> + <!-- Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> + <services xmlns:deploy="vespa" xmlns:preprocess="properties" version="1.0"> + <admin version="2.0"> + <adminserver hostalias="node1"/> + </admin> + <content id="foo" version="1.0"> + <redundancy>1</redundancy> + <documents> + <document mode="index" type="music.sd"/> + </documents> + <nodes> + <node distribution-key="0" hostalias="node0"/> + <node distribution-key="1" hostalias="node1"/> + </nodes> + </content> + <container id="stateless" version="1.0"> + <search> + <chain id="common"> + <searcher id="MySearcher1"/> + <searcher id="MySearcher2"/> + </chain> + </search> + <component bundle="foobundle" class="MyFoo" id="foo"/> + <component bundle="foobundle" class="ProdBar" id="bar"/> + <component bundle="foobundle" class="ProdBaz" id="baz"/> + <nodes> + <node hostalias="node0" baseport="5002"/> + </nodes> + </container> + </services>"""; TestBase.assertDocument(expectedUsEastAndCentral, new XmlPreProcessor(appDir, services, @@ -176,44 +185,46 @@ public class XmlPreprocessorTest { @Test public void testPropertiesWithOverlappingNames() throws Exception { String input = - "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>" + - "<services xmlns:deploy=\"vespa\" xmlns:preprocess=\"properties\" version=\"1.0\">" + - " <preprocess:properties>" + - " <sherpa.host>gamma-usnc1.dht.yahoo.com</sherpa.host>" + - " <sherpa.port>4080</sherpa.port>" + - " <lidspacecompaction_interval>3600</lidspacecompaction_interval>" + - " <lidspacecompaction_interval deploy:environment='prod'>36000</lidspacecompaction_interval>" + - " <lidspacecompaction_allowedlidbloat>50000</lidspacecompaction_allowedlidbloat>" + - " <lidspacecompaction_allowedlidbloat deploy:environment='prod'>50000000</lidspacecompaction_allowedlidbloat>" + - " <lidspacecompaction_allowedlidbloatfactor>0.01</lidspacecompaction_allowedlidbloatfactor>" + - " <lidspacecompaction_allowedlidbloatfactor deploy:environment='prod'>0.91</lidspacecompaction_allowedlidbloatfactor>" + - " </preprocess:properties>" + - " <config name='a'>" + - " <a>${lidspacecompaction_interval}</a>" + - " <b>${lidspacecompaction_allowedlidbloat}</b>" + - " <c>${lidspacecompaction_allowedlidbloatfactor}</c>" + - " <host>${sherpa.host}</host>" + - " <port>${sherpa.port}</port>" + - " </config>" + - " <admin version=\"2.0\">" + - " <adminserver hostalias=\"node0\"/>" + - " </admin>" + - "</services>"; + """ + <?xml version="1.0" encoding="UTF-8" standalone="no"?> + <services xmlns:deploy="vespa" xmlns:preprocess="properties" version="1.0"> + <preprocess:properties> + <sherpa.host>gamma-usnc1.dht.yahoo.com</sherpa.host> + <sherpa.port>4080</sherpa.port> + <lidspacecompaction_interval>3600</lidspacecompaction_interval> + <lidspacecompaction_interval deploy:environment='prod'>36000</lidspacecompaction_interval> + <lidspacecompaction_allowedlidbloat>50000</lidspacecompaction_allowedlidbloat> + <lidspacecompaction_allowedlidbloat deploy:environment='prod'>50000000</lidspacecompaction_allowedlidbloat> + <lidspacecompaction_allowedlidbloatfactor>0.01</lidspacecompaction_allowedlidbloatfactor> + <lidspacecompaction_allowedlidbloatfactor deploy:environment='prod'>0.91</lidspacecompaction_allowedlidbloatfactor> + </preprocess:properties> + <config name='a'> + <a>${lidspacecompaction_interval}</a> + <b>${lidspacecompaction_allowedlidbloat}</b> + <c>${lidspacecompaction_allowedlidbloatfactor}</c> + <host>${sherpa.host}</host> + <port>${sherpa.port}</port> + </config> + <admin version="2.0"> + <adminserver hostalias="node0"/> + </admin> + </services>"""; String expectedProd = - "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>" + - "<services xmlns:deploy=\"vespa\" xmlns:preprocess=\"properties\" version=\"1.0\">" + - " <config name='a'>" + - " <a>36000</a>" + - " <b>50000000</b>" + - " <c>0.91</c>" + - " <host>gamma-usnc1.dht.yahoo.com</host>" + - " <port>4080</port>" + - " </config>" + - " <admin version=\"2.0\">" + - " <adminserver hostalias=\"node0\"/>" + - " </admin>" + - "</services>"; + """ + <?xml version="1.0" encoding="UTF-8" standalone="no"?> + <services xmlns:deploy="vespa" xmlns:preprocess="properties" version="1.0"> + <config name='a'> + <a>36000</a> + <b>50000000</b> + <c>0.91</c> + <host>gamma-usnc1.dht.yahoo.com</host> + <port>4080</port> + </config> + <admin version="2.0"> + <adminserver hostalias="node0"/> + </admin> + </services>"""; Document docDev = (new XmlPreProcessor(appDir, new StringReader(input), InstanceName.defaultName(), diff --git a/config-application-package/src/test/java/com/yahoo/config/model/application/provider/FilesApplicationPackageTest.java b/config-application-package/src/test/java/com/yahoo/config/model/application/provider/FilesApplicationPackageTest.java index 29bc2b6fbd6..6c83b2029ad 100644 --- a/config-application-package/src/test/java/com/yahoo/config/model/application/provider/FilesApplicationPackageTest.java +++ b/config-application-package/src/test/java/com/yahoo/config/model/application/provider/FilesApplicationPackageTest.java @@ -41,34 +41,40 @@ public class FilesApplicationPackageTest { ApplicationPackage processed = app.preprocess(new Zone(Environment.dev, RegionName.defaultName()), new BaseDeployLogger()); assertTrue(new File(appDir, ".preprocessed").exists()); - String expectedServices = "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?><services xmlns:deploy=\"vespa\" xmlns:preprocess=\"properties\" version=\"1.0\">\n" + - " <admin version=\"2.0\">\n" + - " <adminserver hostalias=\"node0\"/>\n" + - " </admin>\n" + - " <content id=\"foo\" version=\"1.0\">\n" + - " <redundancy>1</redundancy>\n" + - " <documents>\n" + - " <document mode=\"index\" type=\"music.sd\"/>\n" + - " </documents>\n" + - " <nodes>\n" + - " <node distribution-key=\"0\" hostalias=\"node0\"/>\n" + - " </nodes>\n" + - " </content>\n" + - " <container id=\"stateless\" version=\"1.0\">\n" + - " <search/>\n" + - " <component bundle=\"foobundle\" class=\"MyFoo\" id=\"foo\"/>\n" + - " <component bundle=\"foobundle\" class=\"TestBar\" id=\"bar\"/>\n" + - " <nodes>\n" + - " <node hostalias=\"node0\" baseport=\"5000\"/>\n" + - " </nodes>\n" + - " </container>\n" + - "</services>"; + String expectedServices = """ + <?xml version="1.0" encoding="UTF-8" standalone="no"?> + <!-- Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> + <services xmlns:deploy="vespa" xmlns:preprocess="properties" version="1.0"> + <admin version="2.0"> + <adminserver hostalias="node0"/> + </admin> + <content id="foo" version="1.0"> + <redundancy>1</redundancy> + <documents> + <document mode="index" type="music.sd"/> + </documents> + <nodes> + <node distribution-key="0" hostalias="node0"/> + </nodes> + </content> + <container id="stateless" version="1.0"> + <search/> + <component bundle="foobundle" class="MyFoo" id="foo"/> + <component bundle="foobundle" class="TestBar" id="bar"/> + <nodes> + <node hostalias="node0" baseport="5000"/> + </nodes> + </container> + </services>"""; TestBase.assertDocument(expectedServices, processed.getServices()); - String expectedHosts = "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?><hosts xmlns:deploy=\"vespa\" xmlns:preprocess=\"properties\">\n" + - " <host name=\"bar.yahoo.com\">\n" + - " <alias>node1</alias>\n" + - " </host>\n" + - "</hosts>"; + String expectedHosts = """ + <?xml version="1.0" encoding="UTF-8" standalone="no"?> + <!-- Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> + <hosts xmlns:deploy="vespa" xmlns:preprocess="properties"> + <host name="bar.yahoo.com"> + <alias>node1</alias> + </host> + </hosts>"""; TestBase.assertDocument(expectedHosts, processed.getHosts()); } diff --git a/config-application-package/src/test/resources/complex-app/deployment.xml b/config-application-package/src/test/resources/complex-app/deployment.xml new file mode 100644 index 00000000000..1fa3a3a6f78 --- /dev/null +++ b/config-application-package/src/test/resources/complex-app/deployment.xml @@ -0,0 +1,140 @@ +<deployment version="1.0" major-version="8" + athenz-domain="gemini-native.aws-core-dev" athenz-service="csp" + cloud-account="337717828807"> + + <parallel> + <instance id="instance0" cloud-account="417744444827"> + <test tester-flavor="d-8-16-10" /> + <block-change version="true" revision="false" days="mon-fri,sun" hours="4-23" time-zone="UTC" /> + </instance> + + <instance id="instance1" cloud-account="417744444827"> + <test tester-flavor="d-8-16-10" /> + <block-change version="true" revision="false" days="mon-fri,sun" hours="4-23" time-zone="UTC" /> + </instance> + + <instance id="instance2" cloud-account="417744444827"> + <test tester-flavor="d-8-16-10" /> + <block-change version="true" revision="false" days="mon-fri,sun" hours="4-23" time-zone="UTC" /> + </instance> + + <instance id="instance3" cloud-account="417744444827"> + <test tester-flavor="d-8-16-10" /> + <block-change version="true" revision="false" days="mon-fri,sun" hours="4-23" time-zone="UTC" /> + </instance> + + <instance id="stress" cloud-account="417744444827"> + <staging /> + <block-change version="true" revision="false" days="mon-fri,sun" hours="4-23" time-zone="UTC" /> + </instance> + </parallel> + + <instance id="beta1" cloud-account="417744444827" tags="beta beta-prod beta-prod-cd"> + <block-change version="true" revision="false" days="mon-fri,sun" hours="4-23" time-zone="UTC" /> + <block-change version="true" revision="false" days="sat" hours="0-23" time-zone="UTC" /> + <upgrade revision-change='when-clear' rollout='separate' revision-target='next' policy='conservative'/> + <prod> + <parallel> + <steps> + <region>aws-us-west-2a</region> + <test>aws-us-west-2a</test> + </steps> + <steps> + <region>aws-us-east-1b</region> + <test>aws-us-east-1b</test> + </steps> + <steps> + <region>aws-eu-west-1a</region> + <test>aws-eu-west-1a</test> + </steps> + <steps> + <region>aws-ap-southeast-1a</region> + <test>aws-ap-southeast-1a</test> + </steps> + </parallel> + </prod> + </instance> + + <instance id="gamma5" cloud-account="417744444827" tags="gamma prod beta-prod beta-prod-cd"> + <block-change version="true" revision="false" days="mon-fri,sun" hours="4-23" time-zone="UTC" /> + <block-change version="true" revision="false" days="sat" hours="0-23" time-zone="UTC" /> + <upgrade revision-change='when-clear' rollout='separate' revision-target='next' policy='conservative'/> + <prod> + <parallel> + <steps> + <region>aws-us-west-2a</region> + <test>aws-us-west-2a</test> + </steps> + <steps> + <region>aws-us-east-1b</region> + <test>aws-us-east-1b</test> + </steps> + <steps> + <region>aws-eu-west-1a</region> + <test>aws-eu-west-1a</test> + </steps> + <steps> + <region>aws-ap-southeast-1a</region> + <test>aws-ap-southeast-1a</test> + </steps> + </parallel> + </prod> + </instance> + + <instance id="delta21" cloud-account="417744444827" tags="delta prod beta-prod beta-prod-cd"> + <block-change version="true" revision="false" days="mon-fri,sun" hours="4-23" time-zone="UTC" /> + <block-change version="true" revision="false" days="sat" hours="0-23" time-zone="UTC" /> + <upgrade revision-change='when-clear' rollout='separate' revision-target='next' policy='conservative'/> + <prod> + <parallel> + <steps> + <region>aws-us-west-2a</region> + <test>aws-us-west-2a</test> + </steps> + <steps> + <region>aws-us-east-1b</region> + <test>aws-us-east-1b</test> + </steps> + <steps> + <region>aws-eu-west-1a</region> + <test>aws-eu-west-1a</test> + </steps> + <steps> + <region>aws-ap-southeast-1a</region> + <test>aws-ap-southeast-1a</test> + </steps> + </parallel> + </prod> + </instance> + + <instance id="prod21a" cloud-account="417744444827" tags="prod-a prod beta-prod beta-prod-cd"> + <block-change version="true" revision="false" days="mon-fri,sun" hours="4-23" time-zone="UTC" /> + <block-change version="true" revision="false" days="sat" hours="0-23" time-zone="UTC" /> + <upgrade revision-change='when-clear' rollout='separate' revision-target='next' policy='conservative'/> + </instance> + + <instance id="prod21b" cloud-account="417744444827" tags="prod-b prod beta-prod beta-prod-cd"> + <block-change version="true" revision="false" days="mon-fri,sun" hours="4-23" time-zone="UTC" /> + <block-change version="true" revision="false" days="sat" hours="0-23" time-zone="UTC" /> + <upgrade revision-change='when-clear' rollout='separate' revision-target='next' policy='conservative'/> + </instance> + + <instance id="prod21c" cloud-account="417744444827" tags="prod-c prod beta-prod beta-prod-cd"> + <block-change version="true" revision="false" days="mon-fri,sun" hours="4-23" time-zone="UTC" /> + <block-change version="true" revision="false" days="sat" hours="0-23" time-zone="UTC" /> + <upgrade revision-change='when-clear' rollout='separate' revision-target='next' policy='conservative'/> + </instance> + + <instance id="cd10" cloud-account="417744444827" tags="cd beta-prod-cd"> + <block-change version="true" revision="false" days="mon-fri,sun" hours="4-23" time-zone="UTC" /> + <block-change version="true" revision="false" days="sat" hours="0-23" time-zone="UTC" /> + <upgrade revision-change='when-clear' rollout='separate' revision-target='next' policy='conservative'/> + </instance> + + <instance id="prod1" cloud-account="417744444827"> + <block-change version="true" revision="false" days="mon-fri,sun" hours="4-23" time-zone="UTC" /> + <block-change version="true" revision="false" days="sat" hours="0-23" time-zone="UTC" /> + <upgrade revision-change='when-clear' rollout='separate' revision-target='next' policy='conservative'/> + </instance> + +</deployment> diff --git a/config-application-package/src/test/resources/complex-app/services.xml b/config-application-package/src/test/resources/complex-app/services.xml new file mode 100644 index 00000000000..23b5a90e5a2 --- /dev/null +++ b/config-application-package/src/test/resources/complex-app/services.xml @@ -0,0 +1,268 @@ +<services version="1.0" xmlns:deploy="vespa" xmlns:preprocess="properties"> + + <container id="docsgateway" version="1.0"> + + <nodes count="1" deploy:environment="dev" deploy:instance="staging1"> + <resources vcpu="48" memory="96Gb" disk="1800Gb" disk-speed="fast" storage-type="local" /> + </nodes> + + <nodes count="1" deploy:environment="dev" deploy:instance="staging1-root"> + <resources vcpu="48" memory="96Gb" disk="1800Gb" disk-speed="fast" storage-type="local" /> + </nodes> + + <nodes count="3"> + <resources deploy:environment="prod perf" vcpu="48" memory="96Gb" disk="1800Gb" disk-speed="fast" storage-type="local" /> + <resources vcpu="8" memory="16Gb" disk="32Gb" /> + </nodes> + + <nodes deploy:environment="staging" count="3"> + <resources vcpu="8" memory="16Gb" disk="32Gb" /> + </nodes> + + </container> + + <container id="qrs" version="1.0"> + <nodes count="1" deploy:environment="dev" deploy:instance="staging1"> + <resources vcpu="16" memory="32Gb" disk="64Gb" /> + </nodes> + + <nodes count="1" deploy:environment="dev" deploy:instance="staging1-root"> + <resources vcpu="16" memory="32Gb" disk="64Gb" /> + </nodes> + + <nodes count="2"> + <resources vcpu="8" memory="16Gb" disk="10Gb" /> + <resources deploy:environment="prod perf" vcpu="16" memory="32Gb" disk="64Gb" /> + </nodes> + + <nodes count="25" deploy:tags="gamma" deploy:region="aws-us-east-1b"> + <resources vcpu="16" memory="32Gb" disk="64Gb" /> + </nodes> + + <nodes count="2" deploy:tags="cd" deploy:region="aws-ap-southeast-1a"> + <resources vcpu="16" memory="32Gb" disk="64Gb" /> + </nodes> + + <nodes count="2" deploy:tags="delta" deploy:region="aws-ap-southeast-1a"> + <resources vcpu="16" memory="32Gb" disk="64Gb" /> + </nodes> + + <nodes count="2" deploy:tags="prod-a" deploy:region="aws-ap-southeast-1a"> + <resources vcpu="16" memory="32Gb" disk="64Gb" /> + </nodes> + + <nodes count="2" deploy:tags="prod-b" deploy:region="aws-ap-southeast-1a"> + <resources vcpu="16" memory="32Gb" disk="64Gb" /> + </nodes> + + <nodes count="2" deploy:tags="prod-c" deploy:region="aws-ap-southeast-1a"> + <resources vcpu="16" memory="32Gb" disk="64Gb" /> + </nodes> + + <nodes count="2" deploy:tags="cd" deploy:region="aws-us-east-1b"> + <resources vcpu="16" memory="32Gb" disk="64Gb" /> + </nodes> + + <nodes count="2" deploy:tags="delta" deploy:region="aws-us-east-1b"> + <resources vcpu="16" memory="32Gb" disk="64Gb" /> + </nodes> + + <nodes count="2" deploy:tags="prod-a" deploy:region="aws-us-east-1b"> + <resources vcpu="16" memory="32Gb" disk="64Gb" /> + </nodes> + + <nodes count="2" deploy:tags="prod-b" deploy:region="aws-us-east-1b"> + <resources vcpu="16" memory="32Gb" disk="64Gb" /> + </nodes> + + <nodes count="2" deploy:tags="prod-c" deploy:region="aws-us-east-1b"> + <resources vcpu="16" memory="32Gb" disk="64Gb" /> + </nodes> + + <nodes count="2" deploy:tags="cd" deploy:region="aws-us-west-2a aws-eu-west-1a"> + <resources vcpu="16" memory="32Gb" disk="64Gb" /> + </nodes> + + <nodes count="2" deploy:tags="delta" deploy:region="aws-us-west-2a aws-eu-west-1a"> + <resources vcpu="16" memory="32Gb" disk="64Gb" /> + </nodes> + + <nodes count="2" deploy:tags="prod-a" deploy:region="aws-us-west-2a aws-eu-west-1a"> + <resources vcpu="16" memory="32Gb" disk="64Gb" /> + </nodes> + + <nodes count="2" deploy:tags="prod-b" deploy:region="aws-us-west-2a aws-eu-west-1a"> + <resources vcpu="16" memory="32Gb" disk="64Gb" /> + </nodes> + + <nodes count="2" deploy:tags="prod-c" deploy:region="aws-us-west-2a aws-eu-west-1a"> + <resources vcpu="16" memory="32Gb" disk="64Gb" /> + </nodes> + + <nodes count="3" deploy:tags="beta" deploy:region="aws-us-west-2a aws-eu-west-1a aws-ap-southeast-1a"> + <resources vcpu="16" memory="32Gb" disk="64Gb" /> + </nodes> + + <nodes count="5" deploy:tags="beta" deploy:region="aws-us-east-1b"> + <resources vcpu="16" memory="32Gb" disk="64Gb" /> + </nodes> + + <nodes count="5" deploy:tags="gamma" deploy:region="aws-us-west-2a aws-eu-west-1a"> + <resources vcpu="16" memory="32Gb" disk="64Gb" /> + </nodes> + + <nodes count="7" deploy:tags="gamma" deploy:region="aws-ap-southeast-1a"> + <resources vcpu="16" memory="32Gb" disk="64Gb" /> + </nodes> + + <search/> + + </container> + + <container id="visitor" version="1.0"> + <nodes count="2"> + <resources vcpu="8" memory="16Gb" disk="32Gb" /> + </nodes> + + <search/> + </container> + + <content id="all" version="1.0"> + <nodes deploy:tags="gamma" deploy:region="aws-ap-southeast-1a" count="10" groups="10"> + <resources vcpu="72" memory="144Gb" disk="1800Gb" disk-speed="fast" storage-type="local" /> + </nodes> + + <nodes deploy:tags="beta" deploy:region="aws-us-east-1b" count="10" groups="10"> + <resources vcpu="48" memory="96Gb" disk="1800Gb" disk-speed="fast" storage-type="local" /> + </nodes> + + <nodes deploy:environment="dev" deploy:instance="staging1" count="1" groups="1"> + <resources vcpu="48" memory="96Gb" disk="1800Gb" disk-speed="fast" storage-type="local" /> + </nodes> + + <nodes deploy:tags="gamma" deploy:region="aws-us-east-1b" count="25" groups="25"> + <resources vcpu="96" memory="192Gb" disk="3600Gb" disk-speed="fast" storage-type="local" /> + </nodes> + + <nodes count="2" groups="2" /> + + <nodes deploy:environment="dev" deploy:instance="staging1-root" count="2" groups="2"> + <resources vcpu="48" memory="96Gb" disk="1800Gb" disk-speed="fast" storage-type="local" /> + </nodes> + + <nodes deploy:tags="cd" deploy:region="aws-ap-southeast-1a" count="2" groups="2"> + <resources vcpu="72" memory="144Gb" disk="1800Gb" disk-speed="fast" storage-type="local" /> + </nodes> + + <nodes deploy:tags="delta" deploy:region="aws-ap-southeast-1a" count="2" groups="2"> + <resources vcpu="72" memory="144Gb" disk="1800Gb" disk-speed="fast" storage-type="local" /> + </nodes> + + <nodes deploy:tags="prod-a" deploy:region="aws-ap-southeast-1a" count="2" groups="2"> + <resources vcpu="72" memory="144Gb" disk="1800Gb" disk-speed="fast" storage-type="local" /> + </nodes> + + <nodes deploy:tags="prod-b" deploy:region="aws-ap-southeast-1a" count="2" groups="2"> + <resources vcpu="72" memory="144Gb" disk="1800Gb" disk-speed="fast" storage-type="local" /> + </nodes> + + <nodes deploy:tags="prod-c" deploy:region="aws-ap-southeast-1a" count="2" groups="2"> + <resources vcpu="72" memory="144Gb" disk="1800Gb" disk-speed="fast" storage-type="local" /> + </nodes> + + <nodes deploy:tags="cd" deploy:region="aws-eu-west-1a" count="2" groups="2"> + <resources vcpu="72" memory="144Gb" disk="1800Gb" disk-speed="fast" storage-type="local" /> + </nodes> + + <nodes deploy:tags="delta" deploy:region="aws-eu-west-1a" count="2" groups="2"> + <resources vcpu="72" memory="144Gb" disk="1800Gb" disk-speed="fast" storage-type="local" /> + </nodes> + + <nodes deploy:tags="prod-a" deploy:region="aws-eu-west-1a" count="2" groups="2"> + <resources vcpu="72" memory="144Gb" disk="1800Gb" disk-speed="fast" storage-type="local" /> + </nodes> + + <nodes deploy:tags="prod-b" deploy:region="aws-eu-west-1a" count="2" groups="2"> + <resources vcpu="72" memory="144Gb" disk="1800Gb" disk-speed="fast" storage-type="local" /> + </nodes> + + <nodes deploy:tags="prod-c" deploy:region="aws-eu-west-1a" count="2" groups="2"> + <resources vcpu="72" memory="144Gb" disk="1800Gb" disk-speed="fast" storage-type="local" /> + </nodes> + + <nodes deploy:tags="cd" deploy:region="aws-us-east-1b" count="2" groups="2"> + <resources vcpu="96" memory="192Gb" disk="3600Gb" disk-speed="fast" storage-type="local" /> + </nodes> + + <nodes deploy:tags="delta" deploy:region="aws-us-east-1b" count="2" groups="2"> + <resources vcpu="96" memory="192Gb" disk="3600Gb" disk-speed="fast" storage-type="local" /> + </nodes> + + <nodes deploy:tags="prod-a" deploy:region="aws-us-east-1b" count="2" groups="2"> + <resources vcpu="96" memory="192Gb" disk="3600Gb" disk-speed="fast" storage-type="local" /> + </nodes> + + <nodes deploy:tags="prod-b" deploy:region="aws-us-east-1b" count="2" groups="2"> + <resources vcpu="96" memory="192Gb" disk="3600Gb" disk-speed="fast" storage-type="local" /> + </nodes> + + <nodes deploy:tags="prod-c" deploy:region="aws-us-east-1b" count="2" groups="2"> + <resources vcpu="96" memory="192Gb" disk="3600Gb" disk-speed="fast" storage-type="local" /> + </nodes> + + <nodes deploy:tags="cd" deploy:region="aws-us-west-2a" count="2" groups="2"> + <resources vcpu="96" memory="192Gb" disk="3600Gb" disk-speed="fast" storage-type="local" /> + </nodes> + + <nodes deploy:tags="delta" deploy:region="aws-us-west-2a" count="2" groups="2"> + <resources vcpu="96" memory="192Gb" disk="3600Gb" disk-speed="fast" storage-type="local" /> + </nodes> + + <nodes deploy:tags="prod-a" deploy:region="aws-us-west-2a" count="2" groups="2"> + <resources vcpu="96" memory="192Gb" disk="3600Gb" disk-speed="fast" storage-type="local" /> + </nodes> + + <nodes deploy:tags="prod-b" deploy:region="aws-us-west-2a" count="2" groups="2"> + <resources vcpu="96" memory="192Gb" disk="3600Gb" disk-speed="fast" storage-type="local" /> + </nodes> + + <nodes deploy:tags="prod-c" deploy:region="aws-us-west-2a" count="2" groups="2"> + <resources vcpu="96" memory="192Gb" disk="3600Gb" disk-speed="fast" storage-type="local" /> + </nodes> + + <nodes deploy:tags="beta" deploy:region="aws-eu-west-1a" count="3" groups="3"> + <resources vcpu="36" memory="72Gb" disk="900Gb" disk-speed="fast" storage-type="local" /> + </nodes> + + <nodes deploy:tags="beta" deploy:region="aws-us-west-2a" count="3" groups="3"> + <resources vcpu="48" memory="96Gb" disk="1800Gb" disk-speed="fast" storage-type="local" /> + </nodes> + + <nodes deploy:environment="perf" count="4" groups="4"> + <resources vcpu="48" memory="96Gb" disk="1800Gb" disk-speed="fast" storage-type="local" /> + </nodes> + + <nodes deploy:tags="beta" deploy:region="aws-ap-southeast-1a" count="4" groups="4"> + <resources vcpu="36" memory="72Gb" disk="900Gb" disk-speed="fast" storage-type="local" /> + </nodes> + + <nodes deploy:tags="gamma" deploy:region="aws-us-west-2a" count="5" groups="5"> + <resources vcpu="96" memory="192Gb" disk="3600Gb" disk-speed="fast" storage-type="local" /> + </nodes> + + <nodes deploy:tags="gamma" deploy:region="aws-eu-west-1a" count="8" groups="8"> + <resources vcpu="72" memory="144Gb" disk="1800Gb" disk-speed="fast" storage-type="local" /> + </nodes> + + <redundancy>1</redundancy> + + </content> + + <content id="filedocument" version="1.0"> + <nodes count="2" groups="2"> + <resources vcpu="4" memory="8Gb" disk="32Gb" /> + </nodes> + <redundancy>1</redundancy> + </content> + +</services>
\ No newline at end of file diff --git a/config-model-api/abi-spec.json b/config-model-api/abi-spec.json deleted file mode 100644 index e5c2be4eb43..00000000000 --- a/config-model-api/abi-spec.json +++ /dev/null @@ -1,727 +0,0 @@ -{ - "com.yahoo.config.application.api.ApplicationFile$MetaData" : { - "superClass" : "java.lang.Object", - "interfaces" : [ ], - "attributes" : [ - "public" - ], - "methods" : [ - "public void <init>()", - "public void <init>(java.lang.String, java.lang.String)", - "public java.lang.String getStatus()", - "public java.lang.String getMd5()" - ], - "fields" : [ - "public java.lang.String status", - "public java.lang.String md5" - ] - }, - "com.yahoo.config.application.api.ApplicationFile$PathFilter" : { - "superClass" : "java.lang.Object", - "interfaces" : [ ], - "attributes" : [ - "public", - "interface", - "abstract" - ], - "methods" : [ - "public abstract boolean accept(com.yahoo.path.Path)" - ], - "fields" : [ ] - }, - "com.yahoo.config.application.api.ApplicationFile" : { - "superClass" : "java.lang.Object", - "interfaces" : [ - "java.lang.Comparable" - ], - "attributes" : [ - "public", - "abstract" - ], - "methods" : [ - "protected void <init>(com.yahoo.path.Path)", - "public abstract boolean isDirectory()", - "public abstract boolean exists()", - "public abstract java.io.Reader createReader()", - "public abstract java.io.InputStream createInputStream()", - "public abstract com.yahoo.config.application.api.ApplicationFile createDirectory()", - "public abstract com.yahoo.config.application.api.ApplicationFile writeFile(java.io.Reader)", - "public abstract com.yahoo.config.application.api.ApplicationFile appendFile(java.lang.String)", - "public java.util.List listFiles()", - "public abstract java.util.List listFiles(com.yahoo.config.application.api.ApplicationFile$PathFilter)", - "public java.util.List listFiles(boolean)", - "public abstract com.yahoo.config.application.api.ApplicationFile delete()", - "public com.yahoo.path.Path getPath()", - "public java.lang.String toString()", - "public boolean equals(java.lang.Object)", - "protected com.yahoo.path.Path getMetaPath()", - "public abstract com.yahoo.config.application.api.ApplicationFile$MetaData getMetaData()" - ], - "fields" : [ - "public static final java.lang.String ContentStatusNew", - "public static final java.lang.String ContentStatusChanged", - "public static final java.lang.String ContentStatusDeleted", - "protected final com.yahoo.path.Path path" - ] - }, - "com.yahoo.config.application.api.ApplicationMetaData" : { - "superClass" : "java.lang.Object", - "interfaces" : [ ], - "attributes" : [ - "public" - ], - "methods" : [ - "public void <init>(java.lang.String, java.lang.Long, boolean, com.yahoo.config.provision.ApplicationId, com.yahoo.config.provision.Tags, java.lang.String, java.lang.Long, long)", - "public void <init>(java.lang.String, java.lang.Long, boolean, com.yahoo.config.provision.ApplicationId, java.lang.String, java.lang.Long, long)", - "public void <init>(java.lang.String, java.lang.String, java.lang.Long, boolean, com.yahoo.config.provision.ApplicationId, java.lang.String, java.lang.Long, long)", - "public java.lang.String getDeployedByUser()", - "public java.lang.String getDeployPath()", - "public com.yahoo.config.provision.ApplicationId getApplicationId()", - "public com.yahoo.config.provision.Tags getTags()", - "public java.lang.Long getDeployTimestamp()", - "public java.lang.Long getGeneration()", - "public boolean isInternalRedeploy()", - "public java.lang.String getChecksum()", - "public long getPreviousActiveGeneration()", - "public java.lang.String toString()", - "public static com.yahoo.config.application.api.ApplicationMetaData fromJsonString(java.lang.String)", - "public com.yahoo.slime.Slime getSlime()", - "public java.lang.String asJsonString()", - "public byte[] asJsonBytes()" - ], - "fields" : [ ] - }, - "com.yahoo.config.application.api.ApplicationPackage" : { - "superClass" : "java.lang.Object", - "interfaces" : [ ], - "attributes" : [ - "public", - "interface", - "abstract" - ], - "methods" : [ - "public abstract com.yahoo.config.provision.ApplicationId getApplicationId()", - "public abstract java.io.Reader getServices()", - "public abstract java.io.Reader getHosts()", - "public java.util.List getUserIncludeDirs()", - "public void validateIncludeDir(java.lang.String)", - "public abstract java.util.Map getAllExistingConfigDefs()", - "public abstract java.util.List getFiles(com.yahoo.path.Path, java.lang.String, boolean)", - "public java.util.List getFiles(com.yahoo.path.Path, java.lang.String)", - "public java.util.Optional getMajorVersion()", - "public abstract com.yahoo.config.application.api.ApplicationFile getFile(com.yahoo.path.Path)", - "public java.util.List getQueryProfileFiles()", - "public java.util.List getQueryProfileTypeFiles()", - "public java.util.List getPageTemplateFiles()", - "public com.yahoo.config.application.api.ApplicationFile getClientSecurityFile()", - "public abstract java.lang.String getHostSource()", - "public abstract java.lang.String getServicesSource()", - "public abstract java.util.Optional getDeployment()", - "public abstract java.util.Optional getValidationOverrides()", - "public abstract java.util.List getComponentsInfo(com.yahoo.component.Version)", - "public abstract java.io.Reader getRankingExpression(java.lang.String)", - "public static java.lang.String getFileName(java.util.jar.JarEntry)", - "public abstract com.yahoo.config.application.api.ApplicationMetaData getMetaData()", - "public abstract java.io.File getFileReference(com.yahoo.path.Path)", - "public void validateXML()", - "public void validateXMLFor(java.util.Optional)", - "public void writeMetaData()", - "public java.util.Optional getAllocatedHosts()", - "public java.util.Map getFileRegistries()", - "public java.util.Map legacyOverrides()", - "public abstract java.util.Collection getSchemas()", - "public com.yahoo.config.application.api.ApplicationPackage preprocess(com.yahoo.config.provision.Zone, com.yahoo.config.application.api.DeployLogger)" - ], - "fields" : [ - "public static final java.lang.String HOSTS", - "public static final java.lang.String SERVICES", - "public static final com.yahoo.path.Path SCHEMAS_DIR", - "public static final com.yahoo.path.Path SEARCH_DEFINITIONS_DIR", - "public static final java.lang.String COMPONENT_DIR", - "public static final java.lang.String SEARCHCHAINS_DIR", - "public static final java.lang.String DOCPROCCHAINS_DIR", - "public static final java.lang.String PROCESSORCHAINS_DIR", - "public static final java.lang.String ROUTINGTABLES_DIR", - "public static final com.yahoo.path.Path MODELS_DIR", - "public static final com.yahoo.path.Path MODELS_GENERATED_DIR", - "public static final com.yahoo.path.Path MODELS_GENERATED_REPLICATED_DIR", - "public static final com.yahoo.path.Path CONSTANTS_DIR", - "public static final java.lang.String CONFIG_DEFINITIONS_DIR", - "public static final com.yahoo.path.Path QUERY_PROFILES_DIR", - "public static final com.yahoo.path.Path QUERY_PROFILE_TYPES_DIR", - "public static final com.yahoo.path.Path PAGE_TEMPLATES_DIR", - "public static final com.yahoo.path.Path RULES_DIR", - "public static final com.yahoo.path.Path DEPLOYMENT_FILE", - "public static final com.yahoo.path.Path VALIDATION_OVERRIDES", - "public static final com.yahoo.path.Path SECURITY_DIR", - "public static final java.lang.String SD_NAME_SUFFIX", - "public static final java.lang.String RANKEXPRESSION_NAME_SUFFIX", - "public static final java.lang.String RANKPROFILE_NAME_SUFFIX", - "public static final java.lang.String RULES_NAME_SUFFIX", - "public static final java.lang.String EXT_DIR", - "public static final java.lang.String PERMANENT_SERVICES" - ] - }, - "com.yahoo.config.application.api.ComponentInfo" : { - "superClass" : "java.lang.Object", - "interfaces" : [ ], - "attributes" : [ - "public" - ], - "methods" : [ - "public void <init>(java.lang.String)", - "public java.lang.String getPathRelativeToAppDir()", - "public java.lang.String toString()" - ], - "fields" : [ ] - }, - "com.yahoo.config.application.api.DeployLogger" : { - "superClass" : "java.lang.Object", - "interfaces" : [ ], - "attributes" : [ - "public", - "interface", - "abstract" - ], - "methods" : [ - "public abstract void log(java.util.logging.Level, java.lang.String)", - "public void logApplicationPackage(java.util.logging.Level, java.lang.String)" - ], - "fields" : [ ] - }, - "com.yahoo.config.application.api.DeploymentInstanceSpec" : { - "superClass" : "com.yahoo.config.application.api.DeploymentSpec$Steps", - "interfaces" : [ ], - "attributes" : [ - "public" - ], - "methods" : [ - "public void <init>(com.yahoo.config.provision.InstanceName, com.yahoo.config.provision.Tags, java.util.List, com.yahoo.config.application.api.DeploymentSpec$UpgradePolicy, com.yahoo.config.application.api.DeploymentSpec$RevisionTarget, com.yahoo.config.application.api.DeploymentSpec$RevisionChange, com.yahoo.config.application.api.DeploymentSpec$UpgradeRollout, int, int, int, java.util.List, java.util.Optional, java.util.Optional, java.util.Optional, com.yahoo.config.application.api.Notifications, java.util.List, java.time.Instant)", - "public com.yahoo.config.provision.InstanceName name()", - "public com.yahoo.config.provision.Tags tags()", - "public com.yahoo.config.application.api.DeploymentSpec$UpgradePolicy upgradePolicy()", - "public com.yahoo.config.application.api.DeploymentSpec$RevisionTarget revisionTarget()", - "public com.yahoo.config.application.api.DeploymentSpec$RevisionChange revisionChange()", - "public com.yahoo.config.application.api.DeploymentSpec$UpgradeRollout upgradeRollout()", - "public int minRisk()", - "public int maxRisk()", - "public int maxIdleHours()", - "public java.util.List changeBlocker()", - "public java.util.Optional globalServiceId()", - "public boolean canUpgradeAt(java.time.Instant)", - "public boolean canChangeRevisionAt(java.time.Instant)", - "public java.util.Optional athenzService(com.yahoo.config.provision.Environment, com.yahoo.config.provision.RegionName)", - "public java.util.Optional cloudAccount(com.yahoo.config.provision.Environment, java.util.Optional)", - "public com.yahoo.config.application.api.Notifications notifications()", - "public java.util.List endpoints()", - "public boolean deploysTo(com.yahoo.config.provision.Environment, com.yahoo.config.provision.RegionName)", - "public boolean equals(java.lang.Object)", - "public int hashCode()", - "public java.lang.String toString()" - ], - "fields" : [ ] - }, - "com.yahoo.config.application.api.DeploymentSpec$ChangeBlocker" : { - "superClass" : "java.lang.Object", - "interfaces" : [ ], - "attributes" : [ - "public" - ], - "methods" : [ - "public void <init>(boolean, boolean, com.yahoo.config.application.api.TimeWindow)", - "public boolean blocksRevisions()", - "public boolean blocksVersions()", - "public com.yahoo.config.application.api.TimeWindow window()", - "public java.lang.String toString()" - ], - "fields" : [ ] - }, - "com.yahoo.config.application.api.DeploymentSpec$DeclaredTest" : { - "superClass" : "com.yahoo.config.application.api.DeploymentSpec$Step", - "interfaces" : [ ], - "attributes" : [ - "public" - ], - "methods" : [ - "public void <init>(com.yahoo.config.provision.RegionName)", - "public boolean concerns(com.yahoo.config.provision.Environment, java.util.Optional)", - "public boolean isTest()", - "public com.yahoo.config.provision.RegionName region()", - "public boolean equals(java.lang.Object)", - "public int hashCode()", - "public java.lang.String toString()" - ], - "fields" : [ ] - }, - "com.yahoo.config.application.api.DeploymentSpec$DeclaredZone" : { - "superClass" : "com.yahoo.config.application.api.DeploymentSpec$Step", - "interfaces" : [ ], - "attributes" : [ - "public" - ], - "methods" : [ - "public void <init>(com.yahoo.config.provision.Environment)", - "public void <init>(com.yahoo.config.provision.Environment, java.util.Optional, boolean, java.util.Optional, java.util.Optional, java.util.Optional)", - "public com.yahoo.config.provision.Environment environment()", - "public java.util.Optional region()", - "public boolean active()", - "public java.util.Optional testerFlavor()", - "public java.util.Optional athenzService()", - "public java.util.Optional cloudAccount()", - "public java.util.List zones()", - "public boolean concerns(com.yahoo.config.provision.Environment, java.util.Optional)", - "public boolean isTest()", - "public int hashCode()", - "public boolean equals(java.lang.Object)", - "public java.lang.String toString()" - ], - "fields" : [ ] - }, - "com.yahoo.config.application.api.DeploymentSpec$Delay" : { - "superClass" : "com.yahoo.config.application.api.DeploymentSpec$Step", - "interfaces" : [ ], - "attributes" : [ - "public" - ], - "methods" : [ - "public void <init>(java.time.Duration)", - "public java.time.Duration delay()", - "public boolean concerns(com.yahoo.config.provision.Environment, java.util.Optional)", - "public java.lang.String toString()" - ], - "fields" : [ ] - }, - "com.yahoo.config.application.api.DeploymentSpec$DeprecatedElement" : { - "superClass" : "java.lang.Object", - "interfaces" : [ ], - "attributes" : [ - "public" - ], - "methods" : [ - "public void <init>(int, java.lang.String, java.util.List, java.lang.String)", - "public int majorVersion()", - "public java.lang.String humanReadableString()", - "public java.lang.String toString()" - ], - "fields" : [ ] - }, - "com.yahoo.config.application.api.DeploymentSpec$ParallelSteps" : { - "superClass" : "com.yahoo.config.application.api.DeploymentSpec$Steps", - "interfaces" : [ ], - "attributes" : [ - "public" - ], - "methods" : [ - "public void <init>(java.util.List)", - "public java.time.Duration delay()", - "public boolean isOrdered()", - "public boolean equals(java.lang.Object)", - "public int hashCode()", - "public java.lang.String toString()" - ], - "fields" : [ ] - }, - "com.yahoo.config.application.api.DeploymentSpec$RevisionChange" : { - "superClass" : "java.lang.Enum", - "interfaces" : [ ], - "attributes" : [ - "public", - "final", - "enum" - ], - "methods" : [ - "public static com.yahoo.config.application.api.DeploymentSpec$RevisionChange[] values()", - "public static com.yahoo.config.application.api.DeploymentSpec$RevisionChange valueOf(java.lang.String)" - ], - "fields" : [ - "public static final enum com.yahoo.config.application.api.DeploymentSpec$RevisionChange whenClear", - "public static final enum com.yahoo.config.application.api.DeploymentSpec$RevisionChange whenFailing", - "public static final enum com.yahoo.config.application.api.DeploymentSpec$RevisionChange always" - ] - }, - "com.yahoo.config.application.api.DeploymentSpec$RevisionTarget" : { - "superClass" : "java.lang.Enum", - "interfaces" : [ ], - "attributes" : [ - "public", - "final", - "enum" - ], - "methods" : [ - "public static com.yahoo.config.application.api.DeploymentSpec$RevisionTarget[] values()", - "public static com.yahoo.config.application.api.DeploymentSpec$RevisionTarget valueOf(java.lang.String)" - ], - "fields" : [ - "public static final enum com.yahoo.config.application.api.DeploymentSpec$RevisionTarget next", - "public static final enum com.yahoo.config.application.api.DeploymentSpec$RevisionTarget latest" - ] - }, - "com.yahoo.config.application.api.DeploymentSpec$Step" : { - "superClass" : "java.lang.Object", - "interfaces" : [ ], - "attributes" : [ - "public", - "abstract" - ], - "methods" : [ - "public void <init>()", - "public final boolean concerns(com.yahoo.config.provision.Environment)", - "public abstract boolean concerns(com.yahoo.config.provision.Environment, java.util.Optional)", - "public java.util.List zones()", - "public java.time.Duration delay()", - "public java.util.List steps()", - "public boolean isTest()", - "public boolean isOrdered()" - ], - "fields" : [ ] - }, - "com.yahoo.config.application.api.DeploymentSpec$Steps" : { - "superClass" : "com.yahoo.config.application.api.DeploymentSpec$Step", - "interfaces" : [ ], - "attributes" : [ - "public" - ], - "methods" : [ - "public void <init>(java.util.List)", - "public java.util.List zones()", - "public java.util.List steps()", - "public boolean concerns(com.yahoo.config.provision.Environment, java.util.Optional)", - "public java.time.Duration delay()", - "public boolean equals(java.lang.Object)", - "public int hashCode()", - "public java.lang.String toString()" - ], - "fields" : [ ] - }, - "com.yahoo.config.application.api.DeploymentSpec$UpgradePolicy" : { - "superClass" : "java.lang.Enum", - "interfaces" : [ ], - "attributes" : [ - "public", - "final", - "enum" - ], - "methods" : [ - "public static com.yahoo.config.application.api.DeploymentSpec$UpgradePolicy[] values()", - "public static com.yahoo.config.application.api.DeploymentSpec$UpgradePolicy valueOf(java.lang.String)" - ], - "fields" : [ - "public static final enum com.yahoo.config.application.api.DeploymentSpec$UpgradePolicy canary", - "public static final enum com.yahoo.config.application.api.DeploymentSpec$UpgradePolicy defaultPolicy", - "public static final enum com.yahoo.config.application.api.DeploymentSpec$UpgradePolicy conservative" - ] - }, - "com.yahoo.config.application.api.DeploymentSpec$UpgradeRollout" : { - "superClass" : "java.lang.Enum", - "interfaces" : [ ], - "attributes" : [ - "public", - "final", - "enum" - ], - "methods" : [ - "public static com.yahoo.config.application.api.DeploymentSpec$UpgradeRollout[] values()", - "public static com.yahoo.config.application.api.DeploymentSpec$UpgradeRollout valueOf(java.lang.String)" - ], - "fields" : [ - "public static final enum com.yahoo.config.application.api.DeploymentSpec$UpgradeRollout separate", - "public static final enum com.yahoo.config.application.api.DeploymentSpec$UpgradeRollout leading", - "public static final enum com.yahoo.config.application.api.DeploymentSpec$UpgradeRollout simultaneous" - ] - }, - "com.yahoo.config.application.api.DeploymentSpec" : { - "superClass" : "java.lang.Object", - "interfaces" : [ ], - "attributes" : [ - "public" - ], - "methods" : [ - "public void <init>(java.util.List, java.util.Optional, java.util.Optional, java.util.Optional, java.util.Optional, java.util.List, java.lang.String, java.util.List)", - "public java.util.Optional majorVersion()", - "public java.util.List steps()", - "public java.util.Optional athenzDomain()", - "public java.util.Optional athenzService()", - "public java.util.Optional cloudAccount()", - "public java.lang.String xmlForm()", - "public java.util.Optional instance(com.yahoo.config.provision.InstanceName)", - "public com.yahoo.config.application.api.DeploymentInstanceSpec requireInstance(java.lang.String)", - "public com.yahoo.config.application.api.DeploymentInstanceSpec requireInstance(com.yahoo.config.provision.InstanceName)", - "public java.util.List instanceNames()", - "public java.util.List instances()", - "public java.util.List endpoints()", - "public java.util.List deprecatedElements()", - "public static com.yahoo.config.application.api.DeploymentSpec fromXml(java.io.Reader)", - "public static com.yahoo.config.application.api.DeploymentSpec fromXml(java.lang.String)", - "public static com.yahoo.config.application.api.DeploymentSpec fromXml(java.lang.String, boolean)", - "public static java.lang.String toMessageString(java.lang.Throwable)", - "public boolean equals(java.lang.Object)", - "public int hashCode()", - "public int deployableHashCode()" - ], - "fields" : [ - "public static final com.yahoo.config.application.api.DeploymentSpec empty" - ] - }, - "com.yahoo.config.application.api.Endpoint$Level" : { - "superClass" : "java.lang.Enum", - "interfaces" : [ ], - "attributes" : [ - "public", - "final", - "enum" - ], - "methods" : [ - "public static com.yahoo.config.application.api.Endpoint$Level[] values()", - "public static com.yahoo.config.application.api.Endpoint$Level valueOf(java.lang.String)" - ], - "fields" : [ - "public static final enum com.yahoo.config.application.api.Endpoint$Level application", - "public static final enum com.yahoo.config.application.api.Endpoint$Level instance" - ] - }, - "com.yahoo.config.application.api.Endpoint$Target" : { - "superClass" : "java.lang.Object", - "interfaces" : [ ], - "attributes" : [ - "public" - ], - "methods" : [ - "public void <init>(com.yahoo.config.provision.RegionName, com.yahoo.config.provision.InstanceName, int)", - "public com.yahoo.config.provision.RegionName region()", - "public com.yahoo.config.provision.InstanceName instance()", - "public int weight()", - "public java.lang.String toString()", - "public boolean equals(java.lang.Object)", - "public int hashCode()" - ], - "fields" : [ ] - }, - "com.yahoo.config.application.api.Endpoint" : { - "superClass" : "java.lang.Object", - "interfaces" : [ ], - "attributes" : [ - "public" - ], - "methods" : [ - "public void <init>(java.lang.String, java.lang.String, com.yahoo.config.application.api.Endpoint$Level, java.util.List)", - "public java.lang.String endpointId()", - "public java.lang.String containerId()", - "public java.util.List regions()", - "public com.yahoo.config.application.api.Endpoint$Level level()", - "public java.util.List targets()", - "public com.yahoo.config.application.api.Endpoint withTargets(java.util.List)", - "public boolean equals(java.lang.Object)", - "public int hashCode()", - "public java.lang.String toString()" - ], - "fields" : [ - "public static final java.lang.String DEFAULT_ID" - ] - }, - "com.yahoo.config.application.api.FileRegistry$Entry" : { - "superClass" : "java.lang.Object", - "interfaces" : [ ], - "attributes" : [ - "public" - ], - "methods" : [ - "public void <init>(java.lang.String, com.yahoo.config.FileReference)" - ], - "fields" : [ - "public final java.lang.String relativePath", - "public final com.yahoo.config.FileReference reference" - ] - }, - "com.yahoo.config.application.api.FileRegistry" : { - "superClass" : "java.lang.Object", - "interfaces" : [ ], - "attributes" : [ - "public", - "interface", - "abstract" - ], - "methods" : [ - "public abstract com.yahoo.config.FileReference addFile(java.lang.String)", - "public abstract com.yahoo.config.FileReference addUri(java.lang.String)", - "public abstract com.yahoo.config.FileReference addBlob(java.lang.String, java.nio.ByteBuffer)", - "public com.yahoo.config.FileReference addApplicationPackage()", - "public abstract java.util.List export()" - ], - "fields" : [ ] - }, - "com.yahoo.config.application.api.Notifications$Role" : { - "superClass" : "java.lang.Enum", - "interfaces" : [ ], - "attributes" : [ - "public", - "final", - "enum" - ], - "methods" : [ - "public static com.yahoo.config.application.api.Notifications$Role[] values()", - "public static com.yahoo.config.application.api.Notifications$Role valueOf(java.lang.String)", - "public static java.lang.String toValue(com.yahoo.config.application.api.Notifications$Role)", - "public static com.yahoo.config.application.api.Notifications$Role fromValue(java.lang.String)" - ], - "fields" : [ - "public static final enum com.yahoo.config.application.api.Notifications$Role author" - ] - }, - "com.yahoo.config.application.api.Notifications$When" : { - "superClass" : "java.lang.Enum", - "interfaces" : [ ], - "attributes" : [ - "public", - "final", - "enum" - ], - "methods" : [ - "public static com.yahoo.config.application.api.Notifications$When[] values()", - "public static com.yahoo.config.application.api.Notifications$When valueOf(java.lang.String)", - "public static java.lang.String toValue(com.yahoo.config.application.api.Notifications$When)", - "public static com.yahoo.config.application.api.Notifications$When fromValue(java.lang.String)" - ], - "fields" : [ - "public static final enum com.yahoo.config.application.api.Notifications$When failing", - "public static final enum com.yahoo.config.application.api.Notifications$When failingCommit" - ] - }, - "com.yahoo.config.application.api.Notifications" : { - "superClass" : "java.lang.Object", - "interfaces" : [ ], - "attributes" : [ - "public" - ], - "methods" : [ - "public static com.yahoo.config.application.api.Notifications none()", - "public static com.yahoo.config.application.api.Notifications of(java.util.Map, java.util.Map)", - "public java.util.Set emailAddressesFor(com.yahoo.config.application.api.Notifications$When)", - "public java.util.Set emailRolesFor(com.yahoo.config.application.api.Notifications$When)" - ], - "fields" : [ ] - }, - "com.yahoo.config.application.api.TimeWindow$LocalDateRange" : { - "superClass" : "java.lang.Object", - "interfaces" : [ ], - "attributes" : [ - "public" - ], - "methods" : [ - "public java.util.Optional start()", - "public java.util.Optional end()", - "public java.lang.String toString()" - ], - "fields" : [ ] - }, - "com.yahoo.config.application.api.TimeWindow" : { - "superClass" : "java.lang.Object", - "interfaces" : [ ], - "attributes" : [ - "public" - ], - "methods" : [ - "public java.util.List days()", - "public java.util.List hours()", - "public java.time.ZoneId zone()", - "public com.yahoo.config.application.api.TimeWindow$LocalDateRange dateRange()", - "public boolean includes(java.time.Instant)", - "public java.lang.String toString()", - "public static com.yahoo.config.application.api.TimeWindow from(java.lang.String, java.lang.String, java.lang.String, java.lang.String, java.lang.String)" - ], - "fields" : [ ] - }, - "com.yahoo.config.application.api.UnparsedConfigDefinition" : { - "superClass" : "java.lang.Object", - "interfaces" : [ ], - "attributes" : [ - "public", - "interface", - "abstract" - ], - "methods" : [ - "public abstract com.yahoo.vespa.config.ConfigDefinition parse()", - "public abstract java.lang.String getUnparsedContent()" - ], - "fields" : [ ] - }, - "com.yahoo.config.application.api.ValidationId" : { - "superClass" : "java.lang.Enum", - "interfaces" : [ ], - "attributes" : [ - "public", - "final", - "enum" - ], - "methods" : [ - "public static com.yahoo.config.application.api.ValidationId[] values()", - "public static com.yahoo.config.application.api.ValidationId valueOf(java.lang.String)", - "public java.lang.String value()", - "public java.lang.String toString()", - "public static java.util.Optional from(java.lang.String)" - ], - "fields" : [ - "public static final enum com.yahoo.config.application.api.ValidationId indexingChange", - "public static final enum com.yahoo.config.application.api.ValidationId indexModeChange", - "public static final enum com.yahoo.config.application.api.ValidationId fieldTypeChange", - "public static final enum com.yahoo.config.application.api.ValidationId clusterSizeReduction", - "public static final enum com.yahoo.config.application.api.ValidationId tensorTypeChange", - "public static final enum com.yahoo.config.application.api.ValidationId resourcesReduction", - "public static final enum com.yahoo.config.application.api.ValidationId contentTypeRemoval", - "public static final enum com.yahoo.config.application.api.ValidationId contentClusterRemoval", - "public static final enum com.yahoo.config.application.api.ValidationId deploymentRemoval", - "public static final enum com.yahoo.config.application.api.ValidationId globalDocumentChange", - "public static final enum com.yahoo.config.application.api.ValidationId configModelVersionMismatch", - "public static final enum com.yahoo.config.application.api.ValidationId skipOldConfigModels", - "public static final enum com.yahoo.config.application.api.ValidationId accessControl", - "public static final enum com.yahoo.config.application.api.ValidationId globalEndpointChange", - "public static final enum com.yahoo.config.application.api.ValidationId redundancyIncrease", - "public static final enum com.yahoo.config.application.api.ValidationId redundancyOne", - "public static final enum com.yahoo.config.application.api.ValidationId pagedSettingRemoval", - "public static final enum com.yahoo.config.application.api.ValidationId certificateRemoval" - ] - }, - "com.yahoo.config.application.api.ValidationOverrides$Allow" : { - "superClass" : "java.lang.Object", - "interfaces" : [ ], - "attributes" : [ - "public" - ], - "methods" : [ - "public void <init>(com.yahoo.config.application.api.ValidationId, java.time.Instant)", - "public boolean allows(com.yahoo.config.application.api.ValidationId, java.time.Instant)", - "public java.lang.String toString()" - ], - "fields" : [ ] - }, - "com.yahoo.config.application.api.ValidationOverrides$ValidationException" : { - "superClass" : "java.lang.IllegalArgumentException", - "interfaces" : [ ], - "attributes" : [ - "public" - ], - "methods" : [ ], - "fields" : [ ] - }, - "com.yahoo.config.application.api.ValidationOverrides" : { - "superClass" : "java.lang.Object", - "interfaces" : [ ], - "attributes" : [ - "public" - ], - "methods" : [ - "public void <init>(java.util.List)", - "public void invalid(java.util.Map, java.time.Instant)", - "public void invalid(com.yahoo.config.application.api.ValidationId, java.lang.String, java.time.Instant)", - "public boolean allows(java.lang.String, java.time.Instant)", - "public boolean allows(com.yahoo.config.application.api.ValidationId, java.time.Instant)", - "public boolean validate(java.time.Instant)", - "public java.lang.String xmlForm()", - "public static java.lang.String toAllowMessage(com.yahoo.config.application.api.ValidationId)", - "public static com.yahoo.config.application.api.ValidationOverrides fromXml(java.io.Reader)", - "public static com.yahoo.config.application.api.ValidationOverrides fromXml(java.lang.String)" - ], - "fields" : [ - "public static final com.yahoo.config.application.api.ValidationOverrides empty" - ] - } -}
\ No newline at end of file diff --git a/config-model-api/pom.xml b/config-model-api/pom.xml index 1d28965fa3c..5053e4b3472 100644 --- a/config-model-api/pom.xml +++ b/config-model-api/pom.xml @@ -111,10 +111,6 @@ </execution> </executions> </plugin> - <plugin> - <groupId>com.yahoo.vespa</groupId> - <artifactId>abi-check-plugin</artifactId> - </plugin> </plugins> </build> </project> diff --git a/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationMetaData.java b/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationMetaData.java index c830c2baa1f..18d3b81197e 100644 --- a/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationMetaData.java +++ b/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationMetaData.java @@ -10,9 +10,6 @@ import com.yahoo.slime.SlimeUtils; import com.yahoo.text.Utf8; import java.io.IOException; -import java.util.Arrays; -import java.util.Set; -import java.util.stream.Collectors; /** * Metadata about an application package. @@ -25,28 +22,25 @@ public class ApplicationMetaData { private final long deployTimestamp; private final boolean internalRedeploy; private final ApplicationId applicationId; - private final Tags tags; private final String checksum; private final long generation; private final long previousActiveGeneration; public ApplicationMetaData(String deployedFromDir, Long deployTimestamp, boolean internalRedeploy, - ApplicationId applicationId, Tags tags, - String checksum, Long generation, long previousActiveGeneration) { + ApplicationId applicationId, String checksum, Long generation, long previousActiveGeneration) { this.deployedFromDir = deployedFromDir; this.deployTimestamp = deployTimestamp; this.internalRedeploy = internalRedeploy; this.applicationId = applicationId; - this.tags = tags; this.checksum = checksum; this.generation = generation; this.previousActiveGeneration = previousActiveGeneration; } @Deprecated // TODO: Remove on Vespa 9 - public ApplicationMetaData(String deployedFromDir, Long deployTimestamp, boolean internalRedeploy, - ApplicationId applicationId, String checksum, Long generation, long previousActiveGeneration) { - this(deployedFromDir, deployTimestamp, internalRedeploy, applicationId, Tags.empty(), checksum, generation, previousActiveGeneration); + public ApplicationMetaData(String deployedFromDir, Long deployTimestamp, boolean internalRedeploy, ApplicationId applicationId, + Tags ignored, String checksum, Long generation, long previousActiveGeneration) { + this(deployedFromDir, deployTimestamp, internalRedeploy, applicationId, checksum, generation, previousActiveGeneration); } @Deprecated // TODO: Remove on Vespa 9 @@ -63,6 +57,9 @@ public class ApplicationMetaData { @Deprecated // TODO: Remove in Vespa 9 public String getDeployedByUser() { return "unknown"; } + @Deprecated // TODO: Remove in Vespa 9 + public Tags getTags() { return Tags.empty(); } + /** * Gets the directory where the application was deployed from. * Will return null if a problem occurred while getting metadata @@ -73,8 +70,6 @@ public class ApplicationMetaData { public ApplicationId getApplicationId() { return applicationId; } - public Tags getTags() { return tags; } - /** * Gets the time the application was deployed. * Will return null if a problem occurred while getting metadata. @@ -95,7 +90,7 @@ public class ApplicationMetaData { */ public boolean isInternalRedeploy() { return internalRedeploy; } - /** Returns an md5 hash of the contents of the application package */ + /** Returns an MD5 hash of the contents of the application package */ public String getChecksum() { return checksum; } /** Returns the previously active generation at the point when this application was created. */ @@ -117,7 +112,6 @@ public class ApplicationMetaData { deploy.field("timestamp").asLong(), booleanField("internalRedeploy", false, deploy), ApplicationId.fromSerializedForm(app.field("id").asString()), - Tags.fromString(deploy.field("tags").asString()), app.field("checksum").asString(), app.field("generation").asLong(), app.field("previousActiveGeneration").asLong()); @@ -133,7 +127,6 @@ public class ApplicationMetaData { deploy.setString("from", deployedFromDir); deploy.setLong("timestamp", deployTimestamp); deploy.setBool("internalRedeploy", internalRedeploy); - deploy.setString("tags", tags.asString()); Cursor app = meta.setObject("application"); app.setString("id", applicationId.serializedForm()); app.setString("checksum", checksum); diff --git a/config-model-api/src/main/java/com/yahoo/config/application/api/package-info.java b/config-model-api/src/main/java/com/yahoo/config/application/api/package-info.java index 32536ebb778..c1f7d4bd844 100644 --- a/config-model-api/src/main/java/com/yahoo/config/application/api/package-info.java +++ b/config-model-api/src/main/java/com/yahoo/config/application/api/package-info.java @@ -1,7 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. @ExportPackage -@PublicApi package com.yahoo.config.application.api; -import com.yahoo.api.annotations.PublicApi; import com.yahoo.osgi.annotation.ExportPackage; diff --git a/config-model-api/src/main/java/com/yahoo/config/application/api/xml/package-info.java b/config-model-api/src/main/java/com/yahoo/config/application/api/xml/package-info.java new file mode 100644 index 00000000000..5ec2ebfa8f9 --- /dev/null +++ b/config-model-api/src/main/java/com/yahoo/config/application/api/xml/package-info.java @@ -0,0 +1,5 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.config.application.api.xml; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/package-info.java b/config-model-api/src/main/java/com/yahoo/config/model/api/package-info.java index 698fd1a7dbb..9560a9658d5 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/package-info.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/package-info.java @@ -3,4 +3,3 @@ package com.yahoo.config.model.api; import com.yahoo.osgi.annotation.ExportPackage; -import com.yahoo.api.annotations.PublicApi; diff --git a/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java b/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java index 2d98c8b35c0..9ee279c68d3 100644 --- a/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java +++ b/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java @@ -10,7 +10,6 @@ import com.yahoo.config.application.api.UnparsedConfigDefinition; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.InstanceName; -import com.yahoo.config.provision.Tags; import com.yahoo.config.provision.TenantName; import com.yahoo.io.IOUtils; import com.yahoo.io.reader.NamedReader; @@ -86,7 +85,6 @@ public class MockApplicationPackage implements ApplicationPackage { ApplicationId.from(TenantName.defaultName(), ApplicationName.from(APPLICATION_NAME), InstanceName.defaultName()), - Tags.empty(), "checksum", APPLICATION_GENERATION, 0L); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/DefaultMetrics.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/DefaultMetrics.java index 6f4535bf4a1..dcd3c10c01b 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/DefaultMetrics.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/DefaultMetrics.java @@ -2,6 +2,8 @@ package com.yahoo.vespa.model.admin.monitoring; +import com.yahoo.metrics.ContainerMetrics; + import java.util.Collections; import java.util.LinkedHashSet; import java.util.Set; @@ -36,12 +38,12 @@ public class DefaultMetrics { } private static void addContainerMetrics(Set<Metric> metrics) { - metrics.add(new Metric("http.status.1xx.rate")); - metrics.add(new Metric("http.status.2xx.rate")); - metrics.add(new Metric("http.status.3xx.rate")); - metrics.add(new Metric("http.status.4xx.rate")); - metrics.add(new Metric("http.status.5xx.rate")); - metrics.add(new Metric("jdisc.gc.ms.average")); + metrics.add(new Metric(ContainerMetrics.HTTP_STATUS_1XX.rate())); + metrics.add(new Metric(ContainerMetrics.HTTP_STATUS_2XX.rate())); + metrics.add(new Metric(ContainerMetrics.HTTP_STATUS_3XX.rate())); + metrics.add(new Metric(ContainerMetrics.HTTP_STATUS_4XX.rate())); + metrics.add(new Metric(ContainerMetrics.HTTP_STATUS_5XX.rate())); + metrics.add(new Metric(ContainerMetrics.JDISC_GC_MS.average())); metrics.add(new Metric("mem.heap.free.average")); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java index 4d4670aeedc..db28ca0e172 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java @@ -1,6 +1,8 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.admin.monitoring; +import com.yahoo.metrics.ContainerMetrics; + import java.util.Collections; import java.util.LinkedHashSet; import java.util.List; @@ -120,24 +122,24 @@ public class VespaMetricSet { metrics.add(new Metric("handled.latency.sum")); metrics.add(new Metric("handled.latency.count")); - metrics.add(new Metric("serverRejectedRequests.rate")); - metrics.add(new Metric("serverRejectedRequests.count")); + metrics.add(new Metric("serverRejectedRequests.rate")); // TODO: Remove on Vespa 9. Use jdisc.thread_pool.rejected_tasks. + metrics.add(new Metric("serverRejectedRequests.count")); // TODO: Remove on Vespa 9. Use jdisc.thread_pool.rejected_tasks. - metrics.add(new Metric("serverThreadPoolSize.max")); - metrics.add(new Metric("serverThreadPoolSize.last")); + metrics.add(new Metric("serverThreadPoolSize.max")); // TODO: Remove on Vespa 9. Use jdisc.thread_pool.size. + metrics.add(new Metric("serverThreadPoolSize.last")); // TODO: Remove on Vespa 9. Use jdisc.thread_pool.size. - metrics.add(new Metric("serverActiveThreads.min")); - metrics.add(new Metric("serverActiveThreads.max")); - metrics.add(new Metric("serverActiveThreads.sum")); - metrics.add(new Metric("serverActiveThreads.count")); - metrics.add(new Metric("serverActiveThreads.last")); + metrics.add(new Metric("serverActiveThreads.min")); // TODO: Remove on Vespa 9. Use jdisc.thread_pool.active_threads. + metrics.add(new Metric("serverActiveThreads.max")); // TODO: Remove on Vespa 9. Use jdisc.thread_pool.active_threads. + metrics.add(new Metric("serverActiveThreads.sum")); // TODO: Remove on Vespa 9. Use jdisc.thread_pool.active_threads. + metrics.add(new Metric("serverActiveThreads.count")); // TODO: Remove on Vespa 9. Use jdisc.thread_pool.active_threads. + metrics.add(new Metric("serverActiveThreads.last")); // TODO: Remove on Vespa 9. Use jdisc.thread_pool.active_threads. metrics.add(new Metric("serverNumOpenConnections.average")); metrics.add(new Metric("serverNumOpenConnections.max")); metrics.add(new Metric("serverNumOpenConnections.last")); - metrics.add(new Metric("serverNumConnections.average")); - metrics.add(new Metric("serverNumConnections.max")); - metrics.add(new Metric("serverNumConnections.last")); + metrics.add(new Metric(ContainerMetrics.SERVER_NUM_CONNECTIONS.average())); + metrics.add(new Metric(ContainerMetrics.SERVER_NUM_CONNECTIONS.max())); + metrics.add(new Metric(ContainerMetrics.SERVER_NUM_CONNECTIONS.last())); metrics.add(new Metric("serverBytesReceived.sum")); metrics.add(new Metric("serverBytesReceived.count")); @@ -146,20 +148,20 @@ public class VespaMetricSet { { List<String> suffixes = List.of("sum", "count", "last", "min", "max"); - addMetric(metrics, "jdisc.thread_pool.unhandled_exceptions", suffixes); - addMetric(metrics, "jdisc.thread_pool.work_queue.capacity", suffixes); - addMetric(metrics, "jdisc.thread_pool.work_queue.size", suffixes); - addMetric(metrics, "jdisc.thread_pool.rejected_tasks", suffixes); - addMetric(metrics, "jdisc.thread_pool.size", suffixes); - addMetric(metrics, "jdisc.thread_pool.max_allowed_size", suffixes); - addMetric(metrics, "jdisc.thread_pool.active_threads", suffixes); - - addMetric(metrics, "jdisc.http.jetty.threadpool.thread.max", suffixes); - addMetric(metrics, "jdisc.http.jetty.threadpool.thread.min", suffixes); - addMetric(metrics, "jdisc.http.jetty.threadpool.thread.reserved", suffixes); - addMetric(metrics, "jdisc.http.jetty.threadpool.thread.busy", suffixes); - addMetric(metrics, "jdisc.http.jetty.threadpool.thread.total", suffixes); - addMetric(metrics, "jdisc.http.jetty.threadpool.queue.size", suffixes); + addMetric(metrics, ContainerMetrics.JDISC_THREAD_POOL_UNHANDLED_EXCEPTIONS.baseName(), suffixes); + addMetric(metrics, ContainerMetrics.JDISC_THREAD_POOL_WORK_QUEUE_CAPACITY.baseName(), suffixes); + addMetric(metrics, ContainerMetrics.JDISC_THREAD_POOL_WORK_QUEUE_SIZE.baseName(), suffixes); + addMetric(metrics, ContainerMetrics.JDISC_THREAD_POOL_REJECTED_TASKS.baseName(), suffixes); + addMetric(metrics, ContainerMetrics.JDISC_THREAD_POOL_SIZE.baseName(), suffixes); + addMetric(metrics, ContainerMetrics.JDISC_THREAD_POOL_MAX_ALLOWED_SIZE.baseName(), suffixes); + addMetric(metrics, ContainerMetrics.JDISC_THREAD_POOL_ACTIVE_THREADS.baseName(), suffixes); + + addMetric(metrics, ContainerMetrics.JETTY_THREADPOOL_MAX_THREADS.baseName(), suffixes); + addMetric(metrics, ContainerMetrics.JETTY_THREADPOOL_MIN_THREADS.baseName(), suffixes); + addMetric(metrics, ContainerMetrics.JETTY_THREADPOOL_RESERVED_THREADS.baseName(), suffixes); + addMetric(metrics, ContainerMetrics.JETTY_THREADPOOL_BUSY_THREADS.baseName(), suffixes); + addMetric(metrics, ContainerMetrics.JETTY_THREADPOOL_TOTAL_THREADS.baseName(), suffixes); + addMetric(metrics, ContainerMetrics.JETTY_THREADPOOL_QUEUE_SIZE.baseName(), suffixes); } metrics.add(new Metric("httpapi_latency.max")); @@ -197,9 +199,9 @@ public class VespaMetricSet { metrics.add(new Metric("jdisc.gc.count.average")); metrics.add(new Metric("jdisc.gc.count.max")); metrics.add(new Metric("jdisc.gc.count.last")); - metrics.add(new Metric("jdisc.gc.ms.average")); - metrics.add(new Metric("jdisc.gc.ms.max")); - metrics.add(new Metric("jdisc.gc.ms.last")); + metrics.add(new Metric(ContainerMetrics.JDISC_GC_MS.average())); + metrics.add(new Metric(ContainerMetrics.JDISC_GC_MS.max())); + metrics.add(new Metric(ContainerMetrics.JDISC_GC_MS.last())); metrics.add(new Metric("jdisc.deactivated_containers.total.last")); metrics.add(new Metric("jdisc.deactivated_containers.with_retained_refs.last")); @@ -218,11 +220,11 @@ public class VespaMetricSet { metrics.add(new Metric("jdisc.http.request.prematurely_closed.rate")); addMetric(metrics, "jdisc.http.request.requests_per_connection", List.of("sum", "count", "min", "max", "average")); - metrics.add(new Metric("http.status.1xx.rate")); - metrics.add(new Metric("http.status.2xx.rate")); - metrics.add(new Metric("http.status.3xx.rate")); - metrics.add(new Metric("http.status.4xx.rate")); - metrics.add(new Metric("http.status.5xx.rate")); + metrics.add(new Metric(ContainerMetrics.HTTP_STATUS_1XX.rate())); + metrics.add(new Metric(ContainerMetrics.HTTP_STATUS_2XX.rate())); + metrics.add(new Metric(ContainerMetrics.HTTP_STATUS_3XX.rate())); + metrics.add(new Metric(ContainerMetrics.HTTP_STATUS_4XX.rate())); + metrics.add(new Metric(ContainerMetrics.HTTP_STATUS_5XX.rate())); metrics.add(new Metric("jdisc.http.request.uri_length.max")); metrics.add(new Metric("jdisc.http.request.uri_length.sum")); diff --git a/config-model/src/test/java/com/yahoo/config/model/ApplicationDeployTest.java b/config-model/src/test/java/com/yahoo/config/model/ApplicationDeployTest.java index 6507341670f..47e1b35291d 100644 --- a/config-model/src/test/java/com/yahoo/config/model/ApplicationDeployTest.java +++ b/config-model/src/test/java/com/yahoo/config/model/ApplicationDeployTest.java @@ -10,7 +10,6 @@ import com.yahoo.config.model.application.provider.DeployData; import com.yahoo.config.model.application.provider.FilesApplicationPackage; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.Tags; import com.yahoo.document.DataType; import com.yahoo.io.IOUtils; import com.yahoo.path.Path; @@ -227,7 +226,6 @@ public class ApplicationDeployTest { ApplicationId applicationId = ApplicationId.from("tenant1", "application1", "instance1"); DeployData deployData = new DeployData("bar", applicationId, - Tags.fromString("tag1 tag2"), 13L, false, 1337L, @@ -238,7 +236,6 @@ public class ApplicationDeployTest { ApplicationMetaData meta = newApp.getMetaData(); assertEquals("bar", meta.getDeployPath()); assertEquals(applicationId, meta.getApplicationId()); - assertEquals(Tags.fromString("tag1 tag2"), meta.getTags()); assertEquals(13L, (long) meta.getDeployTimestamp()); assertEquals(1337L, (long) meta.getGeneration()); assertEquals(3L, meta.getPreviousActiveGeneration()); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java index 4dccebb98b9..b11355df689 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java @@ -23,7 +23,6 @@ import com.yahoo.config.provision.InfraDeployer; import com.yahoo.config.provision.Provisioner; import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.SystemName; -import com.yahoo.config.provision.Tags; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.Zone; import com.yahoo.config.provision.exception.ActivationConflictException; @@ -364,7 +363,6 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye private PrepareResult deploy(File applicationDir, PrepareParams prepareParams, DeployHandlerLogger logger) { long sessionId = createSession(prepareParams.getApplicationId(), - prepareParams.tags(), prepareParams.getTimeoutBudget(), applicationDir, logger); @@ -861,21 +859,21 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye return sessionRepository.createSessionFromExisting(fromSession, internalRedeploy, timeoutBudget, deployLogger).getSessionId(); } - public long createSession(ApplicationId applicationId, Tags tags, TimeoutBudget timeoutBudget, InputStream in, + public long createSession(ApplicationId applicationId, TimeoutBudget timeoutBudget, InputStream in, String contentType, DeployLogger logger) { File tempDir = uncheck(() -> Files.createTempDirectory("deploy")).toFile(); long sessionId; try { - sessionId = createSession(applicationId, tags, timeoutBudget, decompressApplication(in, contentType, tempDir), logger); + sessionId = createSession(applicationId, timeoutBudget, decompressApplication(in, contentType, tempDir), logger); } finally { cleanupTempDirectory(tempDir, logger); } return sessionId; } - public long createSession(ApplicationId applicationId, Tags tags, TimeoutBudget timeoutBudget, File applicationDirectory, DeployLogger deployLogger) { + public long createSession(ApplicationId applicationId, TimeoutBudget timeoutBudget, File applicationDirectory, DeployLogger deployLogger) { SessionRepository sessionRepository = getTenant(applicationId).getSessionRepository(); - Session session = sessionRepository.createSessionFromApplicationPackage(applicationDirectory, applicationId, tags, timeoutBudget, deployLogger); + Session session = sessionRepository.createSessionFromApplicationPackage(applicationDirectory, applicationId, timeoutBudget, deployLogger); return session.getSessionId(); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionCreateHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionCreateHandler.java index 71702e2926c..72f22aff7e2 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionCreateHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionCreateHandler.java @@ -6,7 +6,6 @@ import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.InstanceName; -import com.yahoo.config.provision.Tags; import com.yahoo.config.provision.TenantName; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; @@ -64,7 +63,6 @@ public class SessionCreateHandler extends SessionHandler { // TODO: Avoid using application id here at all ApplicationId applicationId = ApplicationId.from(tenantName, ApplicationName.defaultName(), InstanceName.defaultName()); sessionId = applicationRepository.createSession(applicationId, - Tags.empty(), timeoutBudget, request.getData(), request.getHeader(ApplicationApiHandler.contentTypeHeader), diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/PrepareParams.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/PrepareParams.java index a7a22393771..239026249e5 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/PrepareParams.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/PrepareParams.java @@ -10,7 +10,6 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.AthenzDomain; import com.yahoo.config.provision.CloudAccount; import com.yahoo.config.provision.DockerImage; -import com.yahoo.config.provision.Tags; import com.yahoo.config.provision.TenantName; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.security.X509CertificateUtils; @@ -42,7 +41,6 @@ public final class PrepareParams { static final String APPLICATION_NAME_PARAM_NAME = "applicationName"; static final String INSTANCE_PARAM_NAME = "instance"; - static final String TAGS_PARAM_NAME = "tags"; static final String IGNORE_VALIDATION_PARAM_NAME = "ignoreValidationErrors"; static final String DRY_RUN_PARAM_NAME = "dryRun"; static final String VERBOSE_PARAM_NAME = "verbose"; @@ -59,7 +57,6 @@ public final class PrepareParams { static final String CLOUD_ACCOUNT = "cloudAccount"; private final ApplicationId applicationId; - private final Tags tags; private final TimeoutBudget timeoutBudget; private final boolean ignoreValidationErrors; private final boolean dryRun; @@ -78,7 +75,6 @@ public final class PrepareParams { private final Optional<CloudAccount> cloudAccount; private PrepareParams(ApplicationId applicationId, - Tags tags, TimeoutBudget timeoutBudget, boolean ignoreValidationErrors, boolean dryRun, @@ -97,7 +93,6 @@ public final class PrepareParams { Optional<CloudAccount> cloudAccount) { this.timeoutBudget = timeoutBudget; this.applicationId = Objects.requireNonNull(applicationId); - this.tags = tags; this.ignoreValidationErrors = ignoreValidationErrors; this.dryRun = dryRun; this.verbose = verbose; @@ -124,7 +119,6 @@ public final class PrepareParams { private boolean force = false; private boolean waitForResourcesInPrepare = false; private ApplicationId applicationId = null; - private Tags tags = Tags.empty(); private TimeoutBudget timeoutBudget = new TimeoutBudget(Clock.systemUTC(), Duration.ofSeconds(60)); private Optional<Version> vespaVersion = Optional.empty(); private List<ContainerEndpoint> containerEndpoints = null; @@ -143,11 +137,6 @@ public final class PrepareParams { return this; } - public Builder tags(Tags tags) { - this.tags = tags; - return this; - } - public Builder ignoreValidationErrors(boolean ignoreValidationErrors) { this.ignoreValidationErrors = ignoreValidationErrors; return this; @@ -279,7 +268,6 @@ public final class PrepareParams { public PrepareParams build() { return new PrepareParams(applicationId, - tags, timeoutBudget, ignoreValidationErrors, dryRun, @@ -306,7 +294,6 @@ public final class PrepareParams { .verbose(request.getBooleanProperty(VERBOSE_PARAM_NAME)) .timeoutBudget(SessionHandler.getTimeoutBudget(request, barrierTimeout)) .applicationId(createApplicationId(request, tenant)) - .tags(Tags.fromString(request.getProperty(TAGS_PARAM_NAME))) .vespaVersion(request.getProperty(VESPA_VERSION_PARAM_NAME)) .containerEndpoints(request.getProperty(CONTAINER_ENDPOINTS_PARAM_NAME)) .endpointCertificateMetadata(request.getProperty(ENDPOINT_CERTIFICATE_METADATA_PARAM_NAME)) @@ -329,7 +316,6 @@ public final class PrepareParams { .verbose(booleanValue(params, VERBOSE_PARAM_NAME)) .timeoutBudget(SessionHandler.getTimeoutBudget(getTimeout(params, barrierTimeout))) .applicationId(createApplicationId(params, tenant)) - .tags(Tags.fromString(params.field(TAGS_PARAM_NAME).asString())) .vespaVersion(SlimeUtils.optionalString(params.field(VESPA_VERSION_PARAM_NAME)).orElse(null)) .containerEndpointList(deserialize(params.field(CONTAINER_ENDPOINTS_PARAM_NAME), ContainerEndpointSerializer::endpointListFromSlime, List.of())) .endpointCertificateMetadata(deserialize(params.field(ENDPOINT_CERTIFICATE_METADATA_PARAM_NAME), EndpointCertificateMetadataSerializer::fromSlime)) @@ -404,8 +390,6 @@ public final class PrepareParams { public ApplicationId getApplicationId() { return applicationId; } - public Tags tags() { return tags; } - /** Returns the Vespa version the nodes running the prepared system should have, or empty to use the system version */ public Optional<Version> vespaVersion() { return vespaVersion; } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/Session.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/Session.java index 835381b316d..94b3bd96620 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/Session.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/Session.java @@ -13,7 +13,6 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.AthenzDomain; import com.yahoo.config.provision.CloudAccount; import com.yahoo.config.provision.DockerImage; -import com.yahoo.config.provision.Tags; import com.yahoo.config.provision.TenantName; import com.yahoo.path.Path; import com.yahoo.transaction.Transaction; @@ -120,10 +119,6 @@ public abstract class Session implements Comparable<Session> { sessionZooKeeperClient.writeApplicationId(applicationId); } - public void setTags(Tags tags) { - sessionZooKeeperClient.writeTags(tags); - } - void setApplicationPackageReference(FileReference applicationPackageReference) { sessionZooKeeperClient.writeApplicationPackageReference(Optional.ofNullable(applicationPackageReference)); } @@ -159,10 +154,6 @@ public abstract class Session implements Comparable<Session> { /** Returns application id read from ZooKeeper. Will throw RuntimeException if not found */ public ApplicationId getApplicationId() { return sessionZooKeeperClient.readApplicationId(); } - public Tags getTags() { - return sessionZooKeeperClient.readTags(); - } - /** Returns application id read from ZooKeeper. Will return Optional.empty() if not found */ public Optional<ApplicationId> getOptionalApplicationId() { try { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java index 6589d9d721a..17a019067a3 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java @@ -11,7 +11,9 @@ import com.yahoo.config.application.XmlPreProcessor; import com.yahoo.config.application.api.ApplicationMetaData; import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.config.application.api.DeploymentInstanceSpec; import com.yahoo.config.application.api.FileRegistry; +import com.yahoo.config.application.api.xml.DeploymentSpecXmlReader; import com.yahoo.config.model.api.ConfigDefinitionRepo; import com.yahoo.config.model.api.ContainerEndpoint; import com.yahoo.config.model.api.EndpointCertificateMetadata; @@ -24,6 +26,7 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.AthenzDomain; import com.yahoo.config.provision.CloudAccount; import com.yahoo.config.provision.DockerImage; +import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.Tags; import com.yahoo.config.provision.Zone; import com.yahoo.container.jdisc.secretstore.SecretStore; @@ -156,7 +159,6 @@ public class SessionPreparer { final PrepareParams params; final ApplicationId applicationId; - final Tags tags; /** The repository part of docker image to be used for this deployment */ final Optional<DockerImage> dockerImageRepository; @@ -188,7 +190,6 @@ public class SessionPreparer { this.applicationPackage = applicationPackage; this.sessionZooKeeperClient = sessionZooKeeperClient; this.applicationId = params.getApplicationId(); - this.tags = params.tags(); this.dockerImageRepository = params.dockerImageRepository(); this.vespaVersion = params.vespaVersion().orElse(Vtag.currentVersion); this.containerEndpointsCache = new ContainerEndpointsCache(tenantPath, curator); @@ -267,11 +268,17 @@ public class SessionPreparer { File hostsXml = applicationPackage.getFileReference(Path.fromString("hosts.xml")); // Validate after doing our own preprocessing on these two files - if(servicesXml.exists()) { - vespaPreprocess(applicationPackageDir.getAbsoluteFile(), servicesXml, applicationPackage.getMetaData()); + ApplicationMetaData meta = applicationPackage.getMetaData(); + InstanceName instance = meta.getApplicationId().instance(); + Tags tags = applicationPackage.getDeployment().map(new DeploymentSpecXmlReader(false)::read) + .flatMap(spec -> spec.instance(instance)) + .map(DeploymentInstanceSpec::tags) + .orElse(Tags.empty()); + if (servicesXml.exists()) { + vespaPreprocess(applicationPackageDir.getAbsoluteFile(), servicesXml, meta, tags); } - if(hostsXml.exists()) { - vespaPreprocess(applicationPackageDir.getAbsoluteFile(), hostsXml, applicationPackage.getMetaData()); + if (hostsXml.exists()) { + vespaPreprocess(applicationPackageDir.getAbsoluteFile(), hostsXml, meta, tags); } if (zone.system().isPublic()) { @@ -315,14 +322,15 @@ public class SessionPreparer { } } - void vespaPreprocess(File appDir, File inputXml, ApplicationMetaData metaData) { + void vespaPreprocess(File appDir, File inputXml, ApplicationMetaData metaData, Tags tags) { try { + InstanceName instance = metaData.getApplicationId().instance(); new XmlPreProcessor(appDir, inputXml, - metaData.getApplicationId().instance(), + instance, zone.environment(), zone.region(), - metaData.getTags()) + tags) .run(); } catch (ParserConfigurationException | IOException | SAXException | TransformerException e) { throw new RuntimeException(e); @@ -347,7 +355,6 @@ public class SessionPreparer { writeStateToZooKeeper(sessionZooKeeperClient, preprocessedApplicationPackage, applicationId, - tags, filereference, dockerImageRepository, vespaVersion, @@ -389,7 +396,6 @@ public class SessionPreparer { private void writeStateToZooKeeper(SessionZooKeeperClient zooKeeperClient, ApplicationPackage applicationPackage, ApplicationId applicationId, - Tags tags, FileReference fileReference, Optional<DockerImage> dockerImageRepository, Version vespaVersion, @@ -406,7 +412,6 @@ public class SessionPreparer { zkDeployer.deploy(applicationPackage, fileRegistryMap, allocatedHosts); // Note: When changing the below you need to also change similar calls in SessionRepository.createSessionFromExisting() zooKeeperClient.writeApplicationId(applicationId); - zooKeeperClient.writeTags(tags); zooKeeperClient.writeApplicationPackageReference(Optional.of(fileReference)); zooKeeperClient.writeVespaVersion(vespaVersion); zooKeeperClient.writeDockerImageRepository(dockerImageRepository); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java index d26173b4dd7..306b24a3c69 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java @@ -265,17 +265,14 @@ public class SessionRepository { TimeoutBudget timeoutBudget, DeployLogger deployLogger) { ApplicationId existingApplicationId = existingSession.getApplicationId(); - Tags existingTags = existingSession.getTags(); File existingApp = getSessionAppDir(existingSession.getSessionId()); LocalSession session = createSessionFromApplication(existingApp, existingApplicationId, - existingTags, internalRedeploy, timeoutBudget, deployLogger); // Note: Setters below need to be kept in sync with calls in SessionPreparer.writeStateToZooKeeper() session.setApplicationId(existingApplicationId); - session.setTags(existingTags); session.setApplicationPackageReference(existingSession.getApplicationPackageReference()); session.setVespaVersion(existingSession.getVespaVersion()); session.setDockerImageRepository(existingSession.getDockerImageRepository()); @@ -297,20 +294,19 @@ public class SessionRepository { */ public LocalSession createSessionFromApplicationPackage(File applicationDirectory, ApplicationId applicationId, - Tags tags, TimeoutBudget timeoutBudget, DeployLogger deployLogger) { applicationRepo.createApplication(applicationId); - return createSessionFromApplication(applicationDirectory, applicationId, tags, false, timeoutBudget, deployLogger); + return createSessionFromApplication(applicationDirectory, applicationId, false, timeoutBudget, deployLogger); } /** * Creates a local session based on a remote session and the distributed application package. * Does not wait for session being created on other servers. */ - private void createLocalSession(File applicationFile, ApplicationId applicationId, Tags tags, long sessionId) { + private void createLocalSession(File applicationFile, ApplicationId applicationId, long sessionId) { try { - ApplicationPackage applicationPackage = createApplicationPackage(applicationFile, applicationId, tags, sessionId, false, Optional.empty()); + ApplicationPackage applicationPackage = createApplicationPackage(applicationFile, applicationId, sessionId, false, Optional.empty()); createLocalSession(sessionId, applicationPackage); } catch (Exception e) { throw new RuntimeException("Error creating session " + sessionId, e); @@ -686,13 +682,12 @@ public class SessionRepository { private ApplicationPackage createApplication(File userDir, File configApplicationDir, ApplicationId applicationId, - Tags tags, long sessionId, Optional<Long> currentlyActiveSessionId, boolean internalRedeploy, Optional<DeployLogger> deployLogger) { long deployTimestamp = System.currentTimeMillis(); - DeployData deployData = new DeployData(userDir.getAbsolutePath(), applicationId, tags, deployTimestamp, internalRedeploy, + DeployData deployData = new DeployData(userDir.getAbsolutePath(), applicationId, deployTimestamp, internalRedeploy, sessionId, currentlyActiveSessionId.orElse(nonExistingActiveSessionId)); FilesApplicationPackage app = FilesApplicationPackage.fromFileWithDeployData(configApplicationDir, deployData); validateFileExtensions(applicationId, deployLogger, app); @@ -720,14 +715,13 @@ public class SessionRepository { private LocalSession createSessionFromApplication(File applicationDirectory, ApplicationId applicationId, - Tags tags, boolean internalRedeploy, TimeoutBudget timeoutBudget, DeployLogger deployLogger) { long sessionId = getNextSessionId(); try { ensureSessionPathDoesNotExist(sessionId); - ApplicationPackage app = createApplicationPackage(applicationDirectory, applicationId, tags, sessionId, internalRedeploy, Optional.of(deployLogger)); + ApplicationPackage app = createApplicationPackage(applicationDirectory, applicationId, sessionId, internalRedeploy, Optional.of(deployLogger)); log.log(Level.FINE, () -> TenantRepository.logPre(tenantName) + "Creating session " + sessionId + " in ZooKeeper"); SessionZooKeeperClient sessionZKClient = createSessionZooKeeperClient(sessionId); sessionZKClient.createNewSession(clock.instant()); @@ -743,7 +737,6 @@ public class SessionRepository { private ApplicationPackage createApplicationPackage(File applicationDirectory, ApplicationId applicationId, - Tags tags, long sessionId, boolean internalRedeploy, Optional<DeployLogger> deployLogger) throws IOException { @@ -756,7 +749,6 @@ public class SessionRepository { ApplicationPackage applicationPackage = createApplication(applicationDirectory, userApplicationDir, applicationId, - tags, sessionId, activeSessionId, internalRedeploy, @@ -875,7 +867,7 @@ public class SessionRepository { } ApplicationId applicationId = sessionZKClient.readApplicationId(); log.log(Level.FINE, () -> "Creating local session for tenant '" + tenantName + "' with session id " + sessionId); - createLocalSession(sessionDir, applicationId, sessionZKClient.readTags(), sessionId); + createLocalSession(sessionDir, applicationId, sessionId); } private Optional<Long> getActiveSessionId(ApplicationId applicationId) { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClient.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClient.java index 2353c480a34..4bbb8a63974 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClient.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClient.java @@ -14,7 +14,6 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.AthenzDomain; import com.yahoo.config.provision.CloudAccount; import com.yahoo.config.provision.DockerImage; -import com.yahoo.config.provision.Tags; import com.yahoo.config.provision.TenantName; import com.yahoo.path.Path; import com.yahoo.slime.SlimeUtils; @@ -59,7 +58,6 @@ public class SessionZooKeeperClient { // NOTE: Any state added here MUST also be propagated in com.yahoo.vespa.config.server.deploy.Deployment.prepare() static final String APPLICATION_ID_PATH = "applicationId"; - static final String TAGS_PATH = "tags"; static final String APPLICATION_PACKAGE_REFERENCE_PATH = "applicationPackageReference"; private static final String VERSION_PATH = "version"; private static final String CREATE_TIME_PATH = "createTime"; @@ -175,20 +173,6 @@ public class SessionZooKeeperClient { .orElseThrow(() -> new NotFoundException("Could not find application id for session " + sessionId)); } - private Path tagsPath() { - return sessionPath.append(TAGS_PATH); - } - - public void writeTags(Tags tags) { - curator.set(tagsPath(), Utf8.toBytes(tags.asString())); - } - - public Tags readTags() { - Optional<byte[]> data = curator.getData(tagsPath()); - if (data.isEmpty()) return Tags.empty(); - return Tags.fromString(Utf8.toString(data.get())); - } - void writeApplicationPackageReference(Optional<FileReference> applicationPackageReference) { applicationPackageReference.ifPresent( reference -> curator.set(applicationPackageReferencePath(), Utf8.toBytes(reference.value()))); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java index 8f104a27000..76eb61ed141 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java @@ -17,7 +17,6 @@ import com.yahoo.config.provision.HostFilter; import com.yahoo.config.provision.HostSpec; import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.NetworkPorts; -import com.yahoo.config.provision.Tags; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.exception.ActivationConflictException; import com.yahoo.container.jdisc.HttpResponse; @@ -846,7 +845,7 @@ public class ApplicationRepositoryTest { } private long createSession(ApplicationId applicationId, TimeoutBudget timeoutBudget, File app) { - return applicationRepository.createSession(applicationId, Tags.empty(), timeoutBudget, app, new BaseDeployLogger()); + return applicationRepository.createSession(applicationId, timeoutBudget, app, new BaseDeployLogger()); } private long createSessionFromExisting(ApplicationId applicationId, TimeoutBudget timeoutBudget) { diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClientTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClientTest.java index 653753c97e7..fd6440a9632 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClientTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClientTest.java @@ -13,7 +13,6 @@ import com.yahoo.config.model.application.provider.MockFileRegistry; import com.yahoo.config.provision.AllocatedHosts; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.HostSpec; -import com.yahoo.config.provision.Tags; import com.yahoo.path.Path; import com.yahoo.text.Utf8; import com.yahoo.vespa.config.server.zookeeper.ZKApplicationPackage; @@ -60,7 +59,6 @@ public class ZooKeeperClientTest { ApplicationPackage app = FilesApplicationPackage.fromFileWithDeployData(new File("src/test/apps/zkfeed"), new DeployData("/bar/baz", ApplicationId.from("default", "appName", "default"), - Tags.fromString("tag1 tag2"), 1345L, true, 3L, @@ -123,7 +121,6 @@ public class ZooKeeperClientTest { assertTrue(metaData.getChecksum().length() > 0); assertTrue(metaData.isInternalRedeploy()); assertEquals("/bar/baz", metaData.getDeployPath()); - assertEquals(Tags.fromString("tag1 tag2"), metaData.getTags()); assertEquals(1345, metaData.getDeployTimestamp().longValue()); assertEquals(3, metaData.getGeneration().longValue()); assertEquals(2, metaData.getPreviousActiveGeneration()); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandlerTest.java index 816f7e3dcec..1c71ef0b7fb 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandlerTest.java @@ -6,7 +6,6 @@ import com.yahoo.component.Version; import com.yahoo.config.application.api.ApplicationMetaData; import com.yahoo.config.model.application.provider.BaseDeployLogger; import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.Tags; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.Zone; import com.yahoo.container.jdisc.HttpResponse; @@ -132,7 +131,6 @@ public class SessionActiveHandlerTest { void invoke() { long sessionId = applicationRepository.createSession(applicationId(), - Tags.empty(), new TimeoutBudget(clock, Duration.ofSeconds(10)), testApp, new BaseDeployLogger()); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareHandlerTest.java index 525a969ed1e..de6073bb1ea 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareHandlerTest.java @@ -10,7 +10,6 @@ import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.DockerImage; import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.NodeAllocationException; -import com.yahoo.config.provision.Tags; import com.yahoo.config.provision.TenantName; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.jdisc.http.HttpRequest; @@ -325,7 +324,7 @@ public class SessionPrepareHandlerTest extends SessionHandlerTest { } private long createSession(ApplicationId applicationId) { - return applicationRepository.createSession(applicationId, Tags.empty(), timeoutBudget, app, new BaseDeployLogger()); + return applicationRepository.createSession(applicationId, timeoutBudget, app, new BaseDeployLogger()); } private static class FailingSessionPrepareHandler extends SessionPrepareHandler { diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/session/PrepareParamsTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/session/PrepareParamsTest.java index 82d6f183761..2c898b8bf7d 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/session/PrepareParamsTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/session/PrepareParamsTest.java @@ -7,7 +7,6 @@ import com.yahoo.config.model.api.EndpointCertificateMetadata; import com.yahoo.config.model.api.TenantSecretStore; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.CloudAccount; -import com.yahoo.config.provision.Tags; import com.yahoo.config.provision.TenantName; import com.yahoo.container.jdisc.HttpRequest; @@ -63,7 +62,6 @@ public class PrepareParamsTest { PrepareParams prepareParams = createParams("http://foo:19071/application/v2/", TenantName.defaultName()); assertEquals(ApplicationId.defaultId(), prepareParams.getApplicationId()); - assertTrue(prepareParams.tags().isEmpty()); assertFalse(prepareParams.isDryRun()); assertFalse(prepareParams.isVerbose()); assertFalse(prepareParams.ignoreValidationErrors()); @@ -74,18 +72,6 @@ public class PrepareParamsTest { } @Test - public void testTagsParsing() throws IOException { - var prepareParams = createParams(request + "&" + PrepareParams.TAGS_PARAM_NAME + "=tag1%20tag2", TenantName.from("foo")); - assertEquals(Tags.fromString("tag1 tag2"), prepareParams.tags()); - - // Verify using json object - var slime = SlimeUtils.jsonToSlime(json); - slime.get().setString(PrepareParams.TAGS_PARAM_NAME, "tag1 tag2"); - PrepareParams prepareParamsJson = PrepareParams.fromJson(SlimeUtils.toJsonBytes(slime), TenantName.from("foo"), Duration.ofSeconds(60)); - assertPrepareParamsEqual(prepareParams, prepareParamsJson); - } - - @Test public void testCorrectParsingWithContainerEndpoints() throws IOException { var endpoints = List.of(new ContainerEndpoint("qrs1", ApplicationClusterEndpoint.Scope.global, List.of("c1.example.com", @@ -221,7 +207,6 @@ public class PrepareParamsTest { assertEquals(urlParams.force(), jsonParams.force()); assertEquals(urlParams.waitForResourcesInPrepare(), jsonParams.waitForResourcesInPrepare()); assertEquals(urlParams.getApplicationId(), jsonParams.getApplicationId()); - assertEquals(urlParams.tags(), jsonParams.tags()); assertEquals(urlParams.getTimeoutBudget().timeout(), jsonParams.getTimeoutBudget().timeout()); assertEquals(urlParams.vespaVersion(), jsonParams.vespaVersion()); assertEquals(urlParams.containerEndpoints(), jsonParams.containerEndpoints()); diff --git a/container-core/src/main/java/com/yahoo/container/handler/VipStatusHandler.java b/container-core/src/main/java/com/yahoo/container/handler/VipStatusHandler.java index 8cfa1cf8b7b..ed2d044c38c 100644 --- a/container-core/src/main/java/com/yahoo/container/handler/VipStatusHandler.java +++ b/container-core/src/main/java/com/yahoo/container/handler/VipStatusHandler.java @@ -19,6 +19,8 @@ import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.container.jdisc.ThreadedHttpRequestHandler; import com.yahoo.jdisc.Metric; import java.util.logging.Level; + +import com.yahoo.metrics.ContainerMetrics; import com.yahoo.text.Utf8; import com.yahoo.vespa.defaults.Defaults; @@ -31,7 +33,7 @@ import com.yahoo.vespa.defaults.Defaults; */ public final class VipStatusHandler extends ThreadedHttpRequestHandler { - private static final String NUM_REQUESTS_METRIC = "jdisc.http.requests.status"; + private static final String NUM_REQUESTS_METRIC = ContainerMetrics.JDISC_HTTP_REQUESTS_STATUS.baseName(); private final boolean accessDisk; private final File statusFile; diff --git a/container-core/src/main/java/com/yahoo/container/handler/threadpool/ThreadPoolMetric.java b/container-core/src/main/java/com/yahoo/container/handler/threadpool/ThreadPoolMetric.java index 1c7a1cc4ebe..3f53b05dd6a 100644 --- a/container-core/src/main/java/com/yahoo/container/handler/threadpool/ThreadPoolMetric.java +++ b/container-core/src/main/java/com/yahoo/container/handler/threadpool/ThreadPoolMetric.java @@ -2,6 +2,7 @@ package com.yahoo.container.handler.threadpool; import com.yahoo.jdisc.Metric; +import com.yahoo.metrics.ContainerMetrics; import java.util.Map; @@ -24,28 +25,36 @@ class ThreadPoolMetric { void reportRejectRequest() { metric.add("serverRejectedRequests", 1L, defaultContext); - metric.add("jdisc.thread_pool.rejected_tasks", 1L, defaultContext); + metric.add(ContainerMetrics.JDISC_THREAD_POOL_REJECTED_TASKS.baseName(), 1L, defaultContext); } void reportThreadPoolSize(long size) { metric.set("serverThreadPoolSize", size, defaultContext); - metric.set("jdisc.thread_pool.size", size, defaultContext); + metric.set(ContainerMetrics.JDISC_THREAD_POOL_SIZE.baseName(), size, defaultContext); } - void reportMaxAllowedThreadPoolSize(long size) { metric.set("jdisc.thread_pool.max_allowed_size", size, defaultContext); } + void reportMaxAllowedThreadPoolSize(long size) { + metric.set(ContainerMetrics.JDISC_THREAD_POOL_MAX_ALLOWED_SIZE.baseName(), size, defaultContext); + } void reportActiveThreads(long threads) { metric.set("serverActiveThreads", threads, defaultContext); - metric.set("jdisc.thread_pool.active_threads", threads, defaultContext); + metric.set(ContainerMetrics.JDISC_THREAD_POOL_ACTIVE_THREADS.baseName(), threads, defaultContext); + } + + void reportWorkQueueCapacity(long capacity) { + metric.set(ContainerMetrics.JDISC_THREAD_POOL_WORK_QUEUE_CAPACITY.baseName(), capacity, defaultContext); + } + + void reportWorkQueueSize(long size) { + metric.set(ContainerMetrics.JDISC_THREAD_POOL_WORK_QUEUE_SIZE.baseName(), size, defaultContext); } - void reportWorkQueueCapacity(long capacity) { metric.set("jdisc.thread_pool.work_queue.capacity", capacity, defaultContext); } - void reportWorkQueueSize(long size) { metric.set("jdisc.thread_pool.work_queue.size", size, defaultContext); } void reportUnhandledException(Throwable t) { Metric.Context ctx = metric.createContext(Map.of( THREAD_POOL_NAME_DIMENSION, threadPoolName, "exception", t.getClass().getSimpleName())); - metric.set("jdisc.thread_pool.unhandled_exceptions", 1L, ctx); + metric.set(ContainerMetrics.JDISC_THREAD_POOL_UNHANDLED_EXCEPTIONS.baseName(), 1L, ctx); } } diff --git a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/MetricDefinitions.java b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/MetricDefinitions.java index 29858644d56..8cf5d19ee4e 100644 --- a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/MetricDefinitions.java +++ b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/MetricDefinitions.java @@ -1,6 +1,8 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.jdisc.http.server.jetty; +import com.yahoo.metrics.ContainerMetrics; + /** * Name and dimensions for jdisc/container metrics * @@ -30,7 +32,7 @@ class MetricDefinitions { static final String NUM_BYTES_RECEIVED = "serverBytesReceived"; static final String NUM_BYTES_SENT = "serverBytesSent"; - static final String NUM_CONNECTIONS = "serverNumConnections"; + static final String NUM_CONNECTIONS = ContainerMetrics.SERVER_NUM_CONNECTIONS.baseName(); /* For historical reasons, these are all aliases for the same metric. 'jdisc.http' should ideally be the only one. */ static final String JDISC_HTTP_REQUESTS = "jdisc.http.requests"; @@ -45,11 +47,11 @@ class MetricDefinitions { static final String TOTAL_FAILED_LATENCY = "serverTotalFailedResponseLatency"; static final String TIME_TO_FIRST_BYTE = "serverTimeToFirstByte"; - static final String RESPONSES_1XX = "http.status.1xx"; - static final String RESPONSES_2XX = "http.status.2xx"; - static final String RESPONSES_3XX = "http.status.3xx"; - static final String RESPONSES_4XX = "http.status.4xx"; - static final String RESPONSES_5XX = "http.status.5xx"; + static final String RESPONSES_1XX = ContainerMetrics.HTTP_STATUS_1XX.baseName(); + static final String RESPONSES_2XX = ContainerMetrics.HTTP_STATUS_2XX.baseName(); + static final String RESPONSES_3XX = ContainerMetrics.HTTP_STATUS_3XX.baseName(); + static final String RESPONSES_4XX = ContainerMetrics.HTTP_STATUS_4XX.baseName(); + static final String RESPONSES_5XX = ContainerMetrics.HTTP_STATUS_5XX.baseName(); static final String STARTED_MILLIS = "serverStartedMillis"; @@ -64,13 +66,13 @@ class MetricDefinitions { static final String SSL_HANDSHAKE_FAILURE_UNKNOWN = "jdisc.http.ssl.handshake.failure.unknown"; static final String SSL_HANDSHAKE_FAILURE_CONNECTION_CLOSED = "jdisc.http.ssl.handshake.failure.connection_closed"; - static final String JETTY_THREADPOOL_MAX_THREADS = "jdisc.http.jetty.threadpool.thread.max"; - static final String JETTY_THREADPOOL_MIN_THREADS = "jdisc.http.jetty.threadpool.thread.min"; - static final String JETTY_THREADPOOL_RESERVED_THREADS = "jdisc.http.jetty.threadpool.thread.reserved"; - static final String JETTY_THREADPOOL_BUSY_THREADS = "jdisc.http.jetty.threadpool.thread.busy"; - static final String JETTY_THREADPOOL_IDLE_THREADS = "jdisc.http.jetty.threadpool.thread.idle"; - static final String JETTY_THREADPOOL_TOTAL_THREADS = "jdisc.http.jetty.threadpool.thread.total"; - static final String JETTY_THREADPOOL_QUEUE_SIZE = "jdisc.http.jetty.threadpool.queue.size"; + static final String JETTY_THREADPOOL_MAX_THREADS = ContainerMetrics.JETTY_THREADPOOL_MAX_THREADS.baseName(); + static final String JETTY_THREADPOOL_MIN_THREADS = ContainerMetrics.JETTY_THREADPOOL_MIN_THREADS.baseName(); + static final String JETTY_THREADPOOL_RESERVED_THREADS = ContainerMetrics.JETTY_THREADPOOL_RESERVED_THREADS.baseName(); + static final String JETTY_THREADPOOL_BUSY_THREADS = ContainerMetrics.JETTY_THREADPOOL_BUSY_THREADS.baseName(); + static final String JETTY_THREADPOOL_IDLE_THREADS = ContainerMetrics.JETTY_THREADPOOL_IDLE_THREADS.baseName(); + static final String JETTY_THREADPOOL_TOTAL_THREADS = ContainerMetrics.JETTY_THREADPOOL_TOTAL_THREADS.baseName(); + static final String JETTY_THREADPOOL_QUEUE_SIZE = ContainerMetrics.JETTY_THREADPOOL_QUEUE_SIZE.baseName(); static final String FILTERING_REQUEST_HANDLED = "jdisc.http.filtering.request.handled"; static final String FILTERING_REQUEST_UNHANDLED = "jdisc.http.filtering.request.unhandled"; diff --git a/container-core/src/main/java/com/yahoo/metrics/ContainerMetrics.java b/container-core/src/main/java/com/yahoo/metrics/ContainerMetrics.java new file mode 100644 index 00000000000..50371598d08 --- /dev/null +++ b/container-core/src/main/java/com/yahoo/metrics/ContainerMetrics.java @@ -0,0 +1,94 @@ +package com.yahoo.metrics; + +/** + * @author gjoranv + */ +public enum ContainerMetrics { + + HTTP_STATUS_1XX("http.status.1xx", Unit.RESPONSE, "Number of responses with a 1xx status"), + HTTP_STATUS_2XX("http.status.2xx", Unit.RESPONSE, "Number of responses with a 2xx status"), + HTTP_STATUS_3XX("http.status.3xx", Unit.RESPONSE, "Number of responses with a 3xx status"), + HTTP_STATUS_4XX("http.status.4xx", Unit.RESPONSE, "Number of responses with a 4xx status"), + HTTP_STATUS_5XX("http.status.5xx", Unit.RESPONSE, "Number of responses with a 5xx status"), + + JDISC_GC_MS("jdisc.gc.ms", Unit.MILLISECOND, "Time spent in garbage collection"), + + JDISC_HTTP_REQUESTS_STATUS("jdisc.http.requests.status", Unit.REQUEST, "Number of requests to the built-in status handler"), + + JDISC_THREAD_POOL_UNHANDLED_EXCEPTIONS("jdisc.thread_pool.unhandled_exceptions", Unit.THREAD, "Number of exceptions thrown by tasks"), + JDISC_THREAD_POOL_WORK_QUEUE_CAPACITY("jdisc.thread_pool.work_queue.capacity", Unit.THREAD, "Capacity of the task queue"), + JDISC_THREAD_POOL_WORK_QUEUE_SIZE("jdisc.thread_pool.work_queue.size", Unit.THREAD, "Size of the task queue"), + JDISC_THREAD_POOL_REJECTED_TASKS("jdisc.thread_pool.rejected_tasks", Unit.THREAD, "Number of tasks rejected by the thread pool"), + JDISC_THREAD_POOL_SIZE("jdisc.thread_pool.size", Unit.THREAD, "Size of the thread pool"), + JDISC_THREAD_POOL_MAX_ALLOWED_SIZE("jdisc.thread_pool.max_allowed_size", Unit.THREAD, "The maximum allowed number of threads in the pool"), + JDISC_THREAD_POOL_ACTIVE_THREADS("jdisc.thread_pool.active_threads", Unit.THREAD, "Number of threads that are active"), + + JETTY_THREADPOOL_MAX_THREADS("jdisc.http.jetty.threadpool.thread.max", Unit.THREAD, "Configured maximum number of threads"), + JETTY_THREADPOOL_MIN_THREADS("jdisc.http.jetty.threadpool.thread.min", Unit.THREAD, "Configured minimum number of threads"), + JETTY_THREADPOOL_RESERVED_THREADS("jdisc.http.jetty.threadpool.thread.reserved", Unit.THREAD, "Configured number of reserved threads or -1 for heuristic"), + JETTY_THREADPOOL_BUSY_THREADS("jdisc.http.jetty.threadpool.thread.busy", Unit.THREAD, "Number of threads executing internal and transient jobs"), + JETTY_THREADPOOL_IDLE_THREADS("jdisc.http.jetty.threadpool.thread.idle", Unit.THREAD, "Number of idle threads"), + JETTY_THREADPOOL_TOTAL_THREADS("jdisc.http.jetty.threadpool.thread.total", Unit.THREAD, "Current number of threads"), + JETTY_THREADPOOL_QUEUE_SIZE("jdisc.http.jetty.threadpool.queue.size", Unit.THREAD, "Current size of the job queue"), + + SERVER_NUM_CONNECTIONS("serverNumConnections", Unit.CONNECTION, "The total number of connections opened"); + + private final String name; + private final Unit unit; + private final String description; + + ContainerMetrics(String name, Unit unit, String description) { + this.name = name; + this.unit = unit; + this.description = description; + } + + public String baseName() { + return name; + } + + public String description() { + return description + " (unit: " + unit.shortName() + ")"; + } + + private String withSuffix(Suffix suffix) { + return baseName() + "." + suffix.suffix(); + } + + public String ninety_five_percentile() { + return withSuffix(Suffix.ninety_five_percentile); + } + + public String ninety_nine_percentile() { + return withSuffix(Suffix.ninety_nine_percentile); + } + + public String average() { + return withSuffix(Suffix.average); + } + + public String count() { + return withSuffix(Suffix.count); + } + + public String last() { + return withSuffix(Suffix.last); + } + + public String max() { + return withSuffix(Suffix.max); + } + + public String min() { + return withSuffix(Suffix.min); + } + + public String rate() { + return withSuffix(Suffix.rate); + } + + public String sum() { + return withSuffix(Suffix.sum); + } + +}
\ No newline at end of file diff --git a/container-core/src/main/java/com/yahoo/metrics/Suffix.java b/container-core/src/main/java/com/yahoo/metrics/Suffix.java new file mode 100644 index 00000000000..16dfd428f6c --- /dev/null +++ b/container-core/src/main/java/com/yahoo/metrics/Suffix.java @@ -0,0 +1,24 @@ +package com.yahoo.metrics; + +public enum Suffix { + ninety_five_percentile("95percentile"), + ninety_nine_percentile("99percentile"), + average("average"), + count("count"), + last("last"), + max("max"), + min("min"), + rate("rate"), + sum("sum"); + + private final String suffix; + + Suffix(String suffix) { + this.suffix = suffix; + } + + public String suffix() { + return suffix; + } + +} diff --git a/container-core/src/main/java/com/yahoo/metrics/Unit.java b/container-core/src/main/java/com/yahoo/metrics/Unit.java new file mode 100644 index 00000000000..836dab29f67 --- /dev/null +++ b/container-core/src/main/java/com/yahoo/metrics/Unit.java @@ -0,0 +1,86 @@ +package com.yahoo.metrics; + +/** + * @author gjoranv + */ +public enum Unit { + + BYTE(BaseUnit.BYTE), + CONNECTION(BaseUnit.CONNECTION), + DOCUMENT(BaseUnit.DOCUMENT), + DOCUMENT_PER_SECOND(BaseUnit.DOCUMENT, BaseUnit.SECOND), + FRACTION(BaseUnit.FRACTION), + HIT(BaseUnit.HIT), + HIT_PER_QUERY(BaseUnit.HIT, BaseUnit.QUERY), + MILLISECOND(BaseUnit.MILLISECOND), + OPERATION_PER_SECOND(BaseUnit.OPERATION, BaseUnit.SECOND), + QUERY(BaseUnit.QUERY), + QUERY_PER_SECOND(BaseUnit.QUERY, BaseUnit.SECOND), + REQUEST(BaseUnit.REQUEST), + RESPONSE(BaseUnit.RESPONSE), + RESPONSE_PER_SECOND(BaseUnit.RESPONSE, BaseUnit.SECOND), + SECOND(BaseUnit.SECOND), + THREAD(BaseUnit.THREAD); + + + private final BaseUnit unit; + private final BaseUnit perUnit; + + Unit(BaseUnit unit) { + this(unit, null); + } + + Unit(BaseUnit unit, BaseUnit perUnit) { + this.unit = unit; + this.perUnit = perUnit; + } + + public String fullName() { + return perUnit == null ? + unit.fullName() : + unit.fullName() + "/" + perUnit.fullName(); + } + + public String shortName() { + return perUnit == null ? + unit.shortName : + unit.shortName + "/" + perUnit.shortName; + } + + private enum BaseUnit { + + BYTE("byte"), + CONNECTION("connection"), + DOCUMENT("document"), + FRACTION("fraction"), + HIT("hit"), + MILLISECOND("millisecond", "ms"), + OPERATION("operation"), + QUERY("query"), + REQUEST("request"), + RESPONSE("response"), + SECOND("second", "s"), + THREAD("thread"); + + private final String fullName; + private final String shortName; + + BaseUnit(String fullName) { + this(fullName, fullName); + } + + BaseUnit(String fullName, String shortName) { + this.fullName = fullName; + this.shortName = shortName; + } + + public String fullName() { + return fullName; + } + + public String shortName() { + return shortName; + } + + } +} diff --git a/container-core/src/main/java/com/yahoo/osgi/OsgiImpl.java b/container-core/src/main/java/com/yahoo/osgi/OsgiImpl.java index 4acdc3d544b..9d023e2a7ca 100644 --- a/container-core/src/main/java/com/yahoo/osgi/OsgiImpl.java +++ b/container-core/src/main/java/com/yahoo/osgi/OsgiImpl.java @@ -62,6 +62,11 @@ public class OsgiImpl implements Osgi { if (bundle != null) { return resolveFromBundle(spec, bundle); } else { + if (jdiscOsgi.isFelixFramework() && ! spec.bundle.equals(spec.classId)) { + // Bundle was explicitly specified, but not found. + throw new IllegalArgumentException("Could not find bundle " + spec.bundle + " to create a component with class '" + + spec.classId.getName() + ". " + bundleResolutionErrorMessage(spec.bundle)); + } return resolveFromThisBundleOrSystemBundle(spec); } } diff --git a/container-core/src/test/java/com/yahoo/container/handler/threadpool/ContainerThreadPoolImplTest.java b/container-core/src/test/java/com/yahoo/container/handler/threadpool/ContainerThreadPoolImplTest.java index 4639022d767..606f8052670 100644 --- a/container-core/src/test/java/com/yahoo/container/handler/threadpool/ContainerThreadPoolImplTest.java +++ b/container-core/src/test/java/com/yahoo/container/handler/threadpool/ContainerThreadPoolImplTest.java @@ -6,6 +6,7 @@ import com.yahoo.concurrent.Receiver; import com.yahoo.container.protect.ProcessTerminator; import com.yahoo.container.test.MetricMock; import com.yahoo.jdisc.Metric; +import com.yahoo.metrics.ContainerMetrics; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; @@ -81,10 +82,10 @@ public class ContainerThreadPoolImplTest { assertEquals(1200, executor.getQueue().remainingCapacity()); assertEquals(7, metrics.innvocations().size()); assertEquals(3L, metrics.innvocations().get("serverThreadPoolSize").val); - assertEquals(3L, metrics.innvocations().get("jdisc.thread_pool.max_allowed_size").val); + assertEquals(3L, metrics.innvocations().get(ContainerMetrics.JDISC_THREAD_POOL_MAX_ALLOWED_SIZE.baseName()).val); assertEquals(0L, metrics.innvocations().get("serverActiveThreads").val); - assertEquals(1200L, metrics.innvocations().get("jdisc.thread_pool.work_queue.capacity").val); - assertEquals(0L, metrics.innvocations().get("jdisc.thread_pool.work_queue.size").val); + assertEquals(1200L, metrics.innvocations().get(ContainerMetrics.JDISC_THREAD_POOL_WORK_QUEUE_CAPACITY.baseName()).val); + assertEquals(0L, metrics.innvocations().get(ContainerMetrics.JDISC_THREAD_POOL_WORK_QUEUE_SIZE.baseName()).val); } @Test @@ -95,10 +96,10 @@ public class ContainerThreadPoolImplTest { assertEquals(0, executor.getQueue().remainingCapacity()); assertEquals(7, metrics.innvocations().size()); assertEquals(64L, metrics.innvocations().get("serverThreadPoolSize").val); - assertEquals(64L, metrics.innvocations().get("jdisc.thread_pool.max_allowed_size").val); + assertEquals(64L, metrics.innvocations().get(ContainerMetrics.JDISC_THREAD_POOL_MAX_ALLOWED_SIZE.baseName()).val); assertEquals(0L, metrics.innvocations().get("serverActiveThreads").val); - assertEquals(64L, metrics.innvocations().get("jdisc.thread_pool.work_queue.capacity").val); - assertEquals(0L, metrics.innvocations().get("jdisc.thread_pool.work_queue.size").val); + assertEquals(64L, metrics.innvocations().get(ContainerMetrics.JDISC_THREAD_POOL_WORK_QUEUE_CAPACITY.baseName()).val); + assertEquals(0L, metrics.innvocations().get(ContainerMetrics.JDISC_THREAD_POOL_WORK_QUEUE_SIZE.baseName()).val); } @Test diff --git a/container-disc/src/main/java/com/yahoo/container/jdisc/metric/GarbageCollectionMetrics.java b/container-disc/src/main/java/com/yahoo/container/jdisc/metric/GarbageCollectionMetrics.java index 879778487f5..28af6f7c56c 100644 --- a/container-disc/src/main/java/com/yahoo/container/jdisc/metric/GarbageCollectionMetrics.java +++ b/container-disc/src/main/java/com/yahoo/container/jdisc/metric/GarbageCollectionMetrics.java @@ -2,6 +2,7 @@ package com.yahoo.container.jdisc.metric; import com.yahoo.jdisc.Metric; +import com.yahoo.metrics.ContainerMetrics; import java.lang.management.GarbageCollectorMXBean; import java.lang.management.ManagementFactory; @@ -19,7 +20,7 @@ import java.util.Map; public class GarbageCollectionMetrics { private static final String GC_COUNT = "jdisc.gc.count"; - private static final String GC_TIME = "jdisc.gc.ms"; + private static final String GC_TIME = ContainerMetrics.JDISC_GC_MS.baseName(); private static final String DIMENSION_KEY = "gcName"; public static final Duration REPORTING_INTERVAL = Duration.ofSeconds(62); diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/DeploymentData.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/DeploymentData.java index 63c744c385d..5a55768ad2c 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/DeploymentData.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/DeploymentData.java @@ -5,7 +5,6 @@ import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.CloudAccount; import com.yahoo.config.provision.DockerImage; -import com.yahoo.config.provision.Tags; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.athenz.api.AthenzDomain; import com.yahoo.vespa.hosted.controller.api.integration.billing.Quota; @@ -32,7 +31,6 @@ import static java.util.Objects.requireNonNull; public class DeploymentData { private final ApplicationId instance; - private final Tags tags; private final ZoneId zone; private final Supplier<InputStream> applicationPackage; private final Version platform; @@ -46,7 +44,7 @@ public class DeploymentData { private final Supplier<Optional<CloudAccount>> cloudAccount; private final boolean dryRun; - public DeploymentData(ApplicationId instance, Tags tags, ZoneId zone, Supplier<InputStream> applicationPackage, Version platform, + public DeploymentData(ApplicationId instance, ZoneId zone, Supplier<InputStream> applicationPackage, Version platform, Set<ContainerEndpoint> containerEndpoints, Supplier<Optional<EndpointCertificateMetadata>> endpointCertificateMetadata, Optional<DockerImage> dockerImageRepo, @@ -57,7 +55,6 @@ public class DeploymentData { Supplier<Optional<CloudAccount>> cloudAccount, boolean dryRun) { this.instance = requireNonNull(instance); - this.tags = requireNonNull(tags); this.zone = requireNonNull(zone); this.applicationPackage = requireNonNull(applicationPackage); this.platform = requireNonNull(platform); @@ -76,8 +73,6 @@ public class DeploymentData { return instance; } - public Tags tags() { return tags; } - public ZoneId zone() { return zone; } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/LoadBalancer.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/LoadBalancer.java index 01a5f9d60bb..a4e26fbe7b3 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/LoadBalancer.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/LoadBalancer.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.hosted.controller.api.integration.configserver; import ai.vespa.http.DomainName; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.CloudAccount; import com.yahoo.config.provision.ClusterSpec; import java.util.List; @@ -23,10 +24,12 @@ public class LoadBalancer { private final Optional<String> ipAddress; private final State state; private final Optional<String> dnsZone; + private final Optional<CloudAccount> cloudAccount; private final Optional<PrivateServiceInfo> service; public LoadBalancer(String id, ApplicationId application, ClusterSpec.Id cluster, Optional<DomainName> hostname, - Optional<String> ipAddress, State state, Optional<String> dnsZone, Optional<PrivateServiceInfo> service) { + Optional<String> ipAddress, State state, Optional<String> dnsZone, + Optional<CloudAccount> cloudAccount, Optional<PrivateServiceInfo> service) { this.id = Objects.requireNonNull(id, "id must be non-null"); this.application = Objects.requireNonNull(application, "application must be non-null"); this.cluster = Objects.requireNonNull(cluster, "cluster must be non-null"); @@ -34,6 +37,7 @@ public class LoadBalancer { this.ipAddress = Objects.requireNonNull(ipAddress, "ipAddress must be non-null"); this.state = Objects.requireNonNull(state, "state must be non-null"); this.dnsZone = Objects.requireNonNull(dnsZone, "dnsZone must be non-null"); + this.cloudAccount = Objects.requireNonNull(cloudAccount, "cloudAccount must be non-null"); this.service = Objects.requireNonNull(service, "service must be non-null"); } @@ -65,6 +69,10 @@ public class LoadBalancer { return state; } + public Optional<CloudAccount> cloudAccount() { + return cloudAccount; + } + public Optional<PrivateServiceInfo> service() { return service; } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java index 28673cdc1d2..db188ea3e7e 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java @@ -12,7 +12,6 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.CloudAccount; import com.yahoo.config.provision.DockerImage; import com.yahoo.config.provision.InstanceName; -import com.yahoo.config.provision.Tags; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.text.Text; @@ -175,9 +174,7 @@ public class ApplicationController { lockApplicationIfPresent(id, application -> { for (var declaredInstance : application.get().deploymentSpec().instances()) if ( ! application.get().instances().containsKey(declaredInstance.name())) - application = withNewInstance(application, - id.instance(declaredInstance.name()), - declaredInstance.tags()); + application = withNewInstance(application, id.instance(declaredInstance.name())); store(application); }); count++; @@ -459,14 +456,14 @@ public class ApplicationController { * * @throws IllegalArgumentException if the instance already exists, or has an invalid instance name. */ - public void createInstance(ApplicationId id, Tags tags) { + public void createInstance(ApplicationId id) { lockApplicationOrThrow(TenantAndApplicationId.from(id), application -> { - store(withNewInstance(application, id, tags)); + store(withNewInstance(application, id)); }); } /** Returns given application with a new instance */ - public LockedApplication withNewInstance(LockedApplication application, ApplicationId instance, Tags tags) { + public LockedApplication withNewInstance(LockedApplication application, ApplicationId instance) { if (instance.instance().isTester()) throw new IllegalArgumentException("'" + instance + "' is a tester application!"); InstanceId.validate(instance.instance().value()); @@ -477,7 +474,7 @@ public class ApplicationController { throw new IllegalArgumentException("Could not create '" + instance + "': Instance " + dashToUnderscore(instance) + " already exists"); log.info("Created " + instance); - return application.withNewInstance(instance.instance(), tags); + return application.withNewInstance(instance.instance()); } /** Deploys an application package for an existing application instance. */ @@ -520,7 +517,7 @@ public class ApplicationController { }; // Carry out deployment without holding the application lock. - DeploymentResult result = deploy(job.application(), instance.tags(), applicationPackage, zone, platform, containerEndpoints, + DeploymentResult result = deploy(job.application(), applicationPackage, zone, platform, containerEndpoints, endpointCertificateMetadata, run.isDryRun(), run.testerCertificate()); @@ -558,8 +555,8 @@ public class ApplicationController { } } - /** Stores the deployment spec and validation overrides from the application package, and runs cleanup. */ - public void storeWithUpdatedConfig(LockedApplication application, ApplicationPackage applicationPackage) { + /** Stores the deployment spec and validation overrides from the application package, and runs cleanup. Returns new instances. */ + public List<InstanceName> storeWithUpdatedConfig(LockedApplication application, ApplicationPackage applicationPackage) { validatePackage(applicationPackage, application.get()); application = application.with(applicationPackage.deploymentSpec()); @@ -569,9 +566,7 @@ public class ApplicationController { var declaredInstances = applicationPackage.deploymentSpec().instances(); for (var declaredInstance : declaredInstances) { if ( ! existingInstances.containsKey(declaredInstance.name())) - application = withNewInstance(application, application.get().id().instance(declaredInstance.name()), declaredInstance.tags()); - else if ( ! existingInstances.get(declaredInstance.name()).tags().equals(declaredInstance.tags())) - application = application.with(declaredInstance.name(), instance -> instance.with(declaredInstance.tags())); + application = withNewInstance(application, application.get().id().instance(declaredInstance.name())); } // Delete zones not listed in DeploymentSpec, if allowed @@ -592,6 +587,10 @@ public class ApplicationController { } store(application); + return declaredInstances.stream() + .map(DeploymentInstanceSpec::name) + .filter(instance -> ! existingInstances.containsKey(instance)) + .toList(); } /** Deploy a system application to given zone */ @@ -610,7 +609,7 @@ public class ApplicationController { ApplicationPackageStream applicationPackage = new ApplicationPackageStream( () -> new ByteArrayInputStream(artifactRepository.getSystemApplicationPackage(application.id(), zone, version)) ); - return deploy(application.id(), Tags.empty(), applicationPackage, zone, version, Set.of(), Optional::empty, false, Optional.empty()); + return deploy(application.id(), applicationPackage, zone, version, Set.of(), Optional::empty, false, Optional.empty()); } else { throw new RuntimeException("This system application does not have an application package: " + application.id().toShortString()); } @@ -618,10 +617,10 @@ public class ApplicationController { /** Deploys the given tester application to the given zone. */ public DeploymentResult deployTester(TesterId tester, ApplicationPackageStream applicationPackage, ZoneId zone, Version platform) { - return deploy(tester.id(), Tags.empty(), applicationPackage, zone, platform, Set.of(), Optional::empty, false, Optional.empty()); + return deploy(tester.id(), applicationPackage, zone, platform, Set.of(), Optional::empty, false, Optional.empty()); } - private DeploymentResult deploy(ApplicationId application, Tags tags, ApplicationPackageStream applicationPackage, + private DeploymentResult deploy(ApplicationId application, ApplicationPackageStream applicationPackage, ZoneId zone, Version platform, Set<ContainerEndpoint> endpoints, Supplier<Optional<EndpointCertificateMetadata>> endpointCertificateMetadata, boolean dryRun, Optional<X509Certificate> testerCertificate) { @@ -655,7 +654,7 @@ public class ApplicationController { } Supplier<Optional<CloudAccount>> cloudAccount = () -> decideCloudAccountOf(deployment, applicationPackage.truncatedPackage().deploymentSpec()); ConfigServer.PreparedApplication preparedApplication = - configServer.deploy(new DeploymentData(application, tags, zone, applicationPackage::zipStream, platform, + configServer.deploy(new DeploymentData(application, zone, applicationPackage::zipStream, platform, endpoints, endpointCertificateMetadata, dockerImageRepo, domain, deploymentQuota, tenantSecretStores, operatorCertificates, cloudAccount, dryRun)); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Instance.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Instance.java index 430bafe5c44..d66d1491f73 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Instance.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Instance.java @@ -5,7 +5,6 @@ import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.InstanceName; -import com.yahoo.config.provision.Tags; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RevisionId; @@ -41,7 +40,6 @@ import java.util.stream.Collectors; public class Instance { private final ApplicationId id; - private final Tags tags; private final Map<ZoneId, Deployment> deployments; private final List<AssignedRotation> rotations; private final RotationStatus rotationStatus; @@ -49,15 +47,14 @@ public class Instance { private final Change change; /** Creates an empty instance */ - public Instance(ApplicationId id, Tags tags) { - this(id, tags, Set.of(), Map.of(), List.of(), RotationStatus.EMPTY, Change.empty()); + public Instance(ApplicationId id) { + this(id, Set.of(), Map.of(), List.of(), RotationStatus.EMPTY, Change.empty()); } /** Creates an empty instance*/ - public Instance(ApplicationId id, Tags tags, Collection<Deployment> deployments, Map<JobType, Instant> jobPauses, + public Instance(ApplicationId id, Collection<Deployment> deployments, Map<JobType, Instant> jobPauses, List<AssignedRotation> rotations, RotationStatus rotationStatus, Change change) { this.id = Objects.requireNonNull(id, "id cannot be null"); - this.tags = Objects.requireNonNull(tags, "tags cannot be null"); this.deployments = Objects.requireNonNull(deployments, "deployments cannot be null").stream() .collect(Collectors.toUnmodifiableMap(Deployment::zone, Function.identity())); this.jobPauses = Map.copyOf(Objects.requireNonNull(jobPauses, "deploymentJobs cannot be null")); @@ -66,10 +63,6 @@ public class Instance { this.change = Objects.requireNonNull(change, "change cannot be null"); } - public Instance with(Tags tags) { - return new Instance(id, tags, deployments.values(), jobPauses, rotations, rotationStatus, change); - } - public Instance withNewDeployment(ZoneId zone, RevisionId revision, Version version, Instant instant, Map<DeploymentMetrics.Warning, Integer> warnings, QuotaUsage quotaUsage) { // Use info from previous deployment if available, otherwise create a new one. @@ -94,7 +87,7 @@ public class Instance { else jobPauses.remove(jobType); - return new Instance(id, tags, deployments.values(), jobPauses, rotations, rotationStatus, change); + return new Instance(id, deployments.values(), jobPauses, rotations, rotationStatus, change); } public Instance recordActivityAt(Instant instant, ZoneId zone) { @@ -125,15 +118,15 @@ public class Instance { } public Instance with(List<AssignedRotation> assignedRotations) { - return new Instance(id, tags, deployments.values(), jobPauses, assignedRotations, rotationStatus, change); + return new Instance(id, deployments.values(), jobPauses, assignedRotations, rotationStatus, change); } public Instance with(RotationStatus rotationStatus) { - return new Instance(id, tags, deployments.values(), jobPauses, rotations, rotationStatus, change); + return new Instance(id, deployments.values(), jobPauses, rotations, rotationStatus, change); } public Instance withChange(Change change) { - return new Instance(id, tags, deployments.values(), jobPauses, rotations, rotationStatus, change); + return new Instance(id, deployments.values(), jobPauses, rotations, rotationStatus, change); } private Instance with(Deployment deployment) { @@ -143,15 +136,13 @@ public class Instance { } private Instance with(Map<ZoneId, Deployment> deployments) { - return new Instance(id, tags, deployments.values(), jobPauses, rotations, rotationStatus, change); + return new Instance(id, deployments.values(), jobPauses, rotations, rotationStatus, change); } public ApplicationId id() { return id; } public InstanceName name() { return id.instance(); } - public Tags tags() { return tags; } - /** Returns an immutable map of the current deployments of this */ public Map<ZoneId, Deployment> deployments() { return deployments; } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java index fa702a166d2..b99c52d1533 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java @@ -4,9 +4,7 @@ package com.yahoo.vespa.hosted.controller; import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.application.api.ValidationOverrides; import com.yahoo.config.provision.InstanceName; -import com.yahoo.config.provision.Tags; import com.yahoo.transaction.Mutex; -import com.yahoo.vespa.curator.Lock; import com.yahoo.vespa.hosted.controller.api.integration.organization.IssueId; import com.yahoo.vespa.hosted.controller.api.integration.organization.User; import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; @@ -89,9 +87,9 @@ public class LockedApplication { projectId, revisions, instances.values()); } - LockedApplication withNewInstance(InstanceName instance, Tags tags) { + LockedApplication withNewInstance(InstanceName instance) { var instances = new HashMap<>(this.instances); - instances.put(instance, new Instance(id.instance(instance), tags)); + instances.put(instance, new Instance(id.instance(instance))); return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, deploymentIssueId, ownershipIssueId, owner, majorVersion, metrics, deployKeys, projectId, instances, revisions); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackage.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackage.java index 5becf5dad54..4a8bc3cd09a 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackage.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackage.java @@ -2,8 +2,10 @@ package com.yahoo.vespa.hosted.controller.application.pkg; import com.google.common.hash.Funnel; +import com.google.common.hash.HashFunction; import com.google.common.hash.Hasher; import com.google.common.hash.Hashing; +import com.google.common.hash.HashingOutputStream; import com.yahoo.component.Version; import com.yahoo.vespa.archive.ArchiveStreamReader; import com.yahoo.vespa.archive.ArchiveStreamReader.ArchiveFile; @@ -18,7 +20,6 @@ import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.Tags; -import com.yahoo.security.X509CertificateUtils; import com.yahoo.slime.Inspector; import com.yahoo.slime.Slime; import com.yahoo.slime.SlimeUtils; @@ -27,12 +28,13 @@ import com.yahoo.vespa.hosted.controller.deployment.ZipBuilder; import com.yahoo.yolean.Exceptions; import java.io.ByteArrayInputStream; +import java.io.IOException; import java.io.InputStreamReader; import java.io.OutputStream; +import java.io.UncheckedIOException; import java.nio.file.NoSuchFileException; import java.nio.file.Path; import java.nio.file.Paths; -import java.security.cert.X509Certificate; import java.time.Duration; import java.time.Instant; import java.time.ZoneOffset; @@ -42,6 +44,7 @@ import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Optional; +import java.util.Random; import java.util.Set; import java.util.SortedMap; import java.util.TreeMap; @@ -219,20 +222,18 @@ public class ApplicationPackage { // Hashes all files and settings that require a deployment to be forwarded to configservers private String calculateBundleHash(byte[] zippedContent) { Predicate<String> entryMatcher = name -> ! name.endsWith(deploymentFile) && ! name.endsWith(buildMetaFile); - SortedMap<String, Long> crcByEntry = new TreeMap<>(); Options options = Options.standard().pathPredicate(entryMatcher); + HashingOutputStream hashOut = new HashingOutputStream(Hashing.murmur3_128(-1), OutputStream.nullOutputStream()); ArchiveFile file; try (ArchiveStreamReader reader = ArchiveStreamReader.ofZip(new ByteArrayInputStream(zippedContent), options)) { - OutputStream discard = OutputStream.nullOutputStream(); - while ((file = reader.readNextTo(discard)) != null) { - crcByEntry.put(file.path().toString(), file.crc32().orElse(-1)); + while ((file = reader.readNextTo(hashOut)) != null) { + hashOut.write(file.path().toString().getBytes(UTF_8)); } } - Funnel<SortedMap<String, Long>> funnel = (from, into) -> from.forEach((key, value) -> { - into.putBytes(key.getBytes()); - into.putLong(value); - }); - return hasher().putObject(crcByEntry, funnel) + catch (IOException e) { + throw new UncheckedIOException(e); + } + return hasher().putLong(hashOut.hash().asLong()) .putInt(deploymentSpec.deployableHashCode()) .hash().toString(); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatus.java index ad2274c4e30..dd285917f2a 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatus.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatus.java @@ -39,7 +39,9 @@ import java.util.Collections; import java.util.Deque; import java.util.HashMap; import java.util.HashSet; +import java.util.Iterator; import java.util.LinkedHashMap; +import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Objects; @@ -738,13 +740,16 @@ public class DeploymentStatus { */ private List<JobId> prerequisiteTests(JobId prodJob, JobType testType) { List<JobId> tests = new ArrayList<>(); - Deque<InstanceName> instances = new ArrayDeque<>(); - instances.add(prodJob.application().instance()); - while ( ! instances.isEmpty()) { - InstanceName instance = instances.poll(); + Set<InstanceName> seen = new LinkedHashSet<>(); + Deque<InstanceName> pending = new ArrayDeque<>(); + pending.add(prodJob.application().instance()); + while ( ! pending.isEmpty()) { + InstanceName instance = pending.poll(); Optional<JobId> test = declaredTest(application().id().instance(instance), testType); if (test.isPresent()) tests.add(test.get()); - else instances.addAll(instanceSteps().get(instance).dependencies().stream().map(StepStatus::instance).toList()); + else instanceSteps().get(instance).dependencies().stream().map(StepStatus::instance).forEach(dependency -> { + if (seen.add(dependency)) pending.add(dependency); + }); } if (tests.isEmpty()) tests.add(firstDeclaredOrElseImplicitTest(testType)); return tests; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java index f94bd51fe4c..24bab28c520 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java @@ -6,8 +6,8 @@ import com.yahoo.component.Version; import com.yahoo.component.VersionCompatibility; import com.yahoo.concurrent.UncheckedTimeoutException; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.SystemName; -import com.yahoo.config.provision.Tags; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.transaction.Mutex; import com.yahoo.vespa.flags.FetchVector.Dimension; @@ -29,6 +29,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.deployment.TestReport; import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterCloud; import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterId; import com.yahoo.vespa.hosted.controller.application.ApplicationList; +import com.yahoo.vespa.hosted.controller.application.Change; import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; @@ -589,9 +590,11 @@ public class JobController { validate(id, submission); - applications.storeWithUpdatedConfig(application, submission.applicationPackage()); + List<InstanceName> newInstances = applications.storeWithUpdatedConfig(application, submission.applicationPackage()); if (application.get().projectId().isPresent()) applications.deploymentTrigger().triggerNewRevision(id); + for (InstanceName instance : newInstances) + controller.applications().deploymentTrigger().forceChange(id.instance(instance), Change.of(version.get().id())); }); return version.get(); } @@ -708,7 +711,7 @@ public class JobController { controller.applications().lockApplicationOrThrow(TenantAndApplicationId.from(id), application -> { if ( ! application.get().instances().containsKey(id.instance())) - application = controller.applications().withNewInstance(application, id, Tags.empty()); + application = controller.applications().withNewInstance(application, id); // TODO(mpolden): Enable for public CD once all tests have been updated if (controller.system() != SystemName.PublicCd) { controller.applications().validatePackage(applicationPackage, application.get()); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java index 37c45f38e36..2a9724bb911 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java @@ -8,8 +8,6 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.RegionName; -import com.yahoo.config.provision.SystemName; -import com.yahoo.config.provision.Tags; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.security.KeyUtils; import com.yahoo.slime.ArrayTraverser; @@ -98,7 +96,6 @@ public class ApplicationSerializer { // Instance fields private static final String instanceNameField = "instanceName"; - private static final String tagsField = "tags"; private static final String deploymentsField = "deployments"; private static final String deploymentJobsField = "deploymentJobs"; // TODO jonmv: clean up serialisation format private static final String assignedRotationsField = "assignedRotations"; @@ -186,7 +183,6 @@ public class ApplicationSerializer { for (Instance instance : application.instances().values()) { Cursor instanceObject = array.addObject(); instanceObject.setString(instanceNameField, instance.name().value()); - instanceObject.setString(tagsField, instance.tags().asString()); deploymentsToSlime(instance.deployments().values(), instanceObject.setArray(deploymentsField)); toSlime(instance.jobPauses(), instanceObject.setObject(deploymentJobsField)); assignedRotationsToSlime(instance.rotations(), instanceObject); @@ -383,14 +379,12 @@ public class ApplicationSerializer { List<Instance> instances = new ArrayList<>(); field.traverse((ArrayTraverser) (name, object) -> { InstanceName instanceName = InstanceName.from(object.field(instanceNameField).asString()); - Tags tags = Tags.fromString(object.field(tagsField).asString()); List < Deployment > deployments = deploymentsFromSlime(object.field(deploymentsField), id.instance(instanceName)); Map<JobType, Instant> jobPauses = jobPausesFromSlime(object.field(deploymentJobsField)); List<AssignedRotation> assignedRotations = assignedRotationsFromSlime(object); RotationStatus rotationStatus = rotationStatusFromSlime(object); Change change = changeFromSlime(object.field(deployingField)); instances.add(new Instance(id.instance(instanceName), - tags, deployments, jobPauses, assignedRotations, diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index 2d78667cb99..1505d7e2ca8 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -22,7 +22,6 @@ import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.NodeResources; -import com.yahoo.config.provision.Tags; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.zone.RoutingMethod; import com.yahoo.config.provision.zone.ZoneId; @@ -31,7 +30,6 @@ import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.container.jdisc.ThreadedHttpRequestHandler; import com.yahoo.io.IOUtils; -import com.yahoo.jdisc.Response; import com.yahoo.jdisc.http.filter.security.misc.User; import com.yahoo.restapi.ByteArrayResponse; import com.yahoo.restapi.ErrorResponse; @@ -1361,12 +1359,9 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { toSlime(cluster.min(), clusterObject.setObject("min")); toSlime(cluster.max(), clusterObject.setObject("max")); toSlime(cluster.current(), clusterObject.setObject("current")); - toSlime(cluster.target(), cluster, clusterObject.setObject("target")); - toSlime(cluster.suggested(), cluster, clusterObject.setObject("suggested")); - legacyUtilizationToSlime(cluster.target().peak(), cluster.target().ideal(), clusterObject.setObject("utilization")); // TODO: Remove after January 2023 + toSlime(cluster.target(), clusterObject.setObject("target")); + toSlime(cluster.suggested(), clusterObject.setObject("suggested")); scalingEventsToSlime(cluster.scalingEvents(), clusterObject.setArray("scalingEvents")); - clusterObject.setString("autoscalingStatusCode", cluster.target().status()); // TODO: Remove after January 2023 - clusterObject.setString("autoscalingStatus", cluster.target().description()); // TODO: Remove after January 2023 clusterObject.setLong("scalingDuration", cluster.scalingDuration().toMillis()); } return new SlimeJsonResponse(slime); @@ -2129,7 +2124,7 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { if (controller.applications().getApplication(applicationId).isEmpty()) createApplication(tenantName, applicationName, request); - controller.applications().createInstance(applicationId.instance(instanceName), Tags.empty()); + controller.applications().createInstance(applicationId.instance(instanceName)); Slime slime = new Slime(); toSlime(applicationId.instance(instanceName), slime.setObject(), request); @@ -2713,12 +2708,7 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { object.setDouble("cost", cost); } - private void toSlime(Cluster.Autoscaling autoscaling, Cluster cluster, Cursor autoscalingObject) { - // TODO: Remove after January 2023 - if (autoscaling.resources().isPresent() - && ! autoscaling.resources().get().justNumbers().equals(cluster.current().justNumbers())) - toSlime(autoscaling.resources().get(), autoscalingObject); - + private void toSlime(Cluster.Autoscaling autoscaling, Cursor autoscalingObject) { autoscalingObject.setString("status", autoscaling.status()); autoscalingObject.setString("description", autoscaling.description()); autoscaling.resources().ifPresent(resources -> toSlime(resources, autoscalingObject.setObject("resources"))); @@ -2733,17 +2723,6 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { loadObject.setDouble("disk", load.disk()); } - private void legacyUtilizationToSlime(Load peak, Load ideal, Cursor utilizationObject) { - utilizationObject.setDouble("idealCpu", ideal.cpu()); - utilizationObject.setDouble("peakCpu", peak.cpu()); - - utilizationObject.setDouble("idealMemory", ideal.memory()); - utilizationObject.setDouble("peakMemory", peak.memory()); - - utilizationObject.setDouble("idealDisk", ideal.disk()); - utilizationObject.setDouble("peakDisk", peak.disk()); - } - private void scalingEventsToSlime(List<Cluster.ScalingEvent> scalingEvents, Cursor scalingEventsArray) { for (Cluster.ScalingEvent scalingEvent : scalingEvents) { Cursor scalingEventObject = scalingEventsArray.addObject(); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java index d721528f13b..fe55f018655 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java @@ -374,30 +374,34 @@ public class RoutingPolicies { } private void setPrivateDns(Endpoint endpoint, LoadBalancerAllocation allocation) { - controller.serviceRegistry().vpcEndpointService() - .setPrivateDns(DomainName.of(endpoint.dnsName()), - new ClusterId(allocation.deployment, endpoint.cluster()), - controller.applications().decideCloudAccountOf(allocation.deployment, allocation.deploymentSpec)) - .ifPresent(challenge -> { - try { - nameServiceForwarderIn(allocation.deployment.zoneId()).createTxt(challenge.name(), List.of(challenge.data()), Priority.high); - Instant doom = controller.clock().instant().plusSeconds(30); - while (controller.clock().instant().isBefore(doom)) { - try (Mutex lock = controller.curator().lockNameServiceQueue()) { - if (controller.curator().readNameServiceQueue().requests().stream() - .noneMatch(request -> request.name().equals(Optional.of(challenge.name())))) { - challenge.trigger().run(); - return; - } - } - Thread.sleep(100); - } - throw new UncheckedTimeoutException("timed out waiting for DNS challenge to be processed"); - } - catch (InterruptedException e) { - throw new UncheckedInterruptedException("interrupted waiting for DNS challenge to be processed", e, true); - } - }); + allocation.loadBalancers.stream() + .filter(lb -> lb.service().isPresent()) + .findFirst() + .flatMap(lbWithPrivateService -> + controller.serviceRegistry().vpcEndpointService() + .setPrivateDns(DomainName.of(endpoint.dnsName()), + new ClusterId(allocation.deployment, endpoint.cluster()), + lbWithPrivateService.cloudAccount())) + .ifPresent(challenge -> { + try { + nameServiceForwarderIn(allocation.deployment.zoneId()).createTxt(challenge.name(), List.of(challenge.data()), Priority.high); + Instant doom = controller.clock().instant().plusSeconds(30); + while (controller.clock().instant().isBefore(doom)) { + try (Mutex lock = controller.curator().lockNameServiceQueue()) { + if (controller.curator().readNameServiceQueue().requests().stream() + .noneMatch(request -> request.name().equals(Optional.of(challenge.name())))) { + challenge.trigger().run(); + return; + } + } + Thread.sleep(100); + } + throw new UncheckedTimeoutException("timed out waiting for DNS challenge to be processed"); + } + catch (InterruptedException e) { + throw new UncheckedInterruptedException("interrupted waiting for DNS challenge to be processed", e, true); + } + }); } /** diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java index 8fdff787420..95b81dffaed 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java @@ -15,7 +15,6 @@ import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.RegionName; -import com.yahoo.config.provision.Tags; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.zone.RoutingMethod; import com.yahoo.config.provision.zone.ZoneId; @@ -1406,7 +1405,7 @@ public class ControllerTest { void testDeactivateDeploymentUnknownByController() { DeploymentContext context = tester.newDeploymentContext(); DeploymentId deployment = context.deploymentIdIn(ZoneId.from("prod", "us-west-1")); - DeploymentData deploymentData = new DeploymentData(deployment.applicationId(), Tags.empty(), deployment.zoneId(), InputStream::nullInputStream, Version.fromString("6.1"), + DeploymentData deploymentData = new DeploymentData(deployment.applicationId(), deployment.zoneId(), InputStream::nullInputStream, Version.fromString("6.1"), Set.of(), Optional::empty, Optional.empty(), Optional.empty(), Quota::unlimited, List.of(), List.of(), Optional::empty, false); tester.configServer().deploy(deploymentData); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java index 38a4945e354..43a85aa6aca 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java @@ -7,7 +7,6 @@ import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.SystemName; -import com.yahoo.config.provision.Tags; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.zone.RoutingMethod; import com.yahoo.config.provision.zone.ZoneApi; @@ -371,7 +370,7 @@ public final class ControllerTester { public Application createApplication(String tenant, String applicationName, String instanceName) { Application application = createApplication(tenant, applicationName); - controller().applications().createInstance(application.id().instance(instanceName), Tags.empty()); + controller().applications().createInstance(application.id().instance(instanceName)); return application; } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/DeploymentQuotaCalculatorTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/DeploymentQuotaCalculatorTest.java index db45933f498..a199ef9e34e 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/DeploymentQuotaCalculatorTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/DeploymentQuotaCalculatorTest.java @@ -8,12 +8,10 @@ import com.yahoo.config.application.api.ValidationOverrides; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.RegionName; -import com.yahoo.config.provision.Tags; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Instance; import com.yahoo.vespa.hosted.controller.api.integration.billing.Quota; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RevisionId; import com.yahoo.vespa.hosted.controller.api.integration.noderepository.ApplicationData; import com.yahoo.vespa.hosted.controller.deployment.RevisionHistory; @@ -39,23 +37,24 @@ public class DeploymentQuotaCalculatorTest { void quota_is_divided_among_prod_instances() { Quota calculated = DeploymentQuotaCalculator.calculate(Quota.unlimited().withBudget(10), List.of(), ApplicationId.defaultId(), ZoneId.defaultId(), DeploymentSpec.fromXml( - "<deployment version='1.0'>\n" + - " <instance id='instance1'> \n" + - " <test />\n" + - " <staging />\n" + - " <prod>\n" + - " <region active=\"true\">us-east-1</region>\n" + - " <region active=\"false\">us-west-1</region>\n" + - " </prod>\n" + - " </instance>\n" + - " <instance id='instance2'>\n" + - " <perf/>\n" + - " <dev/>\n" + - " <prod>\n" + - " <region active=\"true\">us-north-1</region>\n" + - " </prod>\n" + - " </instance>\n" + - "</deployment>")); + """ + <deployment version='1.0'> + <instance id='instance1'>\s + <test /> + <staging /> + <prod> + <region active="true">us-east-1</region> + <region active="false">us-west-1</region> + </prod> + </instance> + <instance id='instance2'> + <perf/> + <dev/> + <prod> + <region active="true">us-north-1</region> + </prod> + </instance> + </deployment>""")); assertEquals(10d / 3, calculated.budget().orElseThrow().doubleValue(), 1e-5); } @@ -64,23 +63,24 @@ public class DeploymentQuotaCalculatorTest { var existing_dev_deployment = new Application(TenantAndApplicationId.from(ApplicationId.defaultId()), Instant.EPOCH, DeploymentSpec.empty, ValidationOverrides.empty, Optional.empty(), Optional.empty(), Optional.empty(), OptionalInt.empty(), new ApplicationMetrics(1, 1), Set.of(), OptionalLong.empty(), RevisionHistory.empty(), - List.of(new Instance(ApplicationId.defaultId(), Tags.empty()).withNewDeployment(ZoneId.from(Environment.dev, RegionName.defaultName()), + List.of(new Instance(ApplicationId.defaultId()).withNewDeployment(ZoneId.from(Environment.dev, RegionName.defaultName()), RevisionId.forProduction(1), Version.emptyVersion, Instant.EPOCH, Map.of(), QuotaUsage.create(0.53d)))); Quota calculated = DeploymentQuotaCalculator.calculate(Quota.unlimited().withBudget(2), List.of(existing_dev_deployment), ApplicationId.defaultId(), ZoneId.defaultId(), DeploymentSpec.fromXml( - "<deployment version='1.0'>\n" + - " <instance id='default'> \n" + - " <test />\n" + - " <staging />\n" + - " <prod>\n" + - " <region active=\"true\">us-east-1</region>\n" + - " <region active=\"false\">us-west-1</region>\n" + - " <region active=\"true\">us-north-1</region>\n" + - " <region active=\"true\">us-south-1</region>\n" + - " </prod>\n" + - " </instance>\n" + - "</deployment>")); + """ + <deployment version='1.0'> + <instance id='default'>\s + <test /> + <staging /> + <prod> + <region active="true">us-east-1</region> + <region active="false">us-west-1</region> + <region active="true">us-north-1</region> + <region active="true">us-south-1</region> + </prod> + </instance> + </deployment>""")); assertEquals((2d - 0.53d) / 4d, calculated.budget().orElseThrow().doubleValue(), 1e-5); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificatesTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificatesTest.java index d79a81c7746..8a8500448d8 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificatesTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificatesTest.java @@ -9,7 +9,6 @@ import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.SystemName; -import com.yahoo.config.provision.Tags; import com.yahoo.config.provision.zone.RoutingMethod; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.security.KeyAlgorithm; @@ -105,7 +104,7 @@ public class EndpointCertificatesTest { return x509CertificateBuilder.build(); } - private final Instance testInstance = new Instance(ApplicationId.defaultId(), Tags.empty()); + private final Instance testInstance = new Instance(ApplicationId.defaultId()); private final String testKeyName = "testKeyName"; private final String testCertName = "testCertName"; private ZoneId testZone; @@ -245,7 +244,7 @@ public class EndpointCertificatesTest { @Test void includes_application_endpoint_when_declared() { - Instance instance = new Instance(ApplicationId.from("t1", "a1", "default"), Tags.empty()); + Instance instance = new Instance(ApplicationId.from("t1", "a1", "default")); ZoneId zone1 = ZoneId.from(Environment.prod, RegionName.from("aws-us-east-1c")); ZoneId zone2 = ZoneId.from(Environment.prod, RegionName.from("aws-us-west-2a")); ControllerTester tester = publicTester(); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java index d5f636b5294..9895cd68004 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java @@ -996,7 +996,7 @@ public class DeploymentTriggerTest { @Test void testUserInstancesNotInDeploymentSpec() { var app = tester.newDeploymentContext(); - tester.controller().applications().createInstance(app.application().id().instance("user"), Tags.empty()); + tester.controller().applications().createInstance(app.application().id().instance("user")); app.submit().deploy(); } @@ -1371,52 +1371,52 @@ public class DeploymentTriggerTest { app1.runJob(productionUsWest1); tester.triggerJobs(); - assertEquals(2, tester.jobs().active().size()); + assertEquals(3, tester.jobs().active().size()); app1.runJob(productionUsEast3); tester.triggerJobs(); - assertEquals(1, tester.jobs().active().size()); + assertEquals(2, tester.jobs().active().size()); tester.clock().advance(Duration.ofHours(2)); app1.runJob(productionEuWest1); tester.triggerJobs(); - assertEquals(1, tester.jobs().active().size()); + assertEquals(2, tester.jobs().active().size()); app2.assertNotRunning(testEuWest1); app2.runJob(productionEuWest1); tester.triggerJobs(); - assertEquals(1, tester.jobs().active().size()); + assertEquals(2, tester.jobs().active().size()); app2.runJob(testEuWest1); tester.triggerJobs(); - assertEquals(List.of(), tester.jobs().active()); + assertEquals(1, tester.jobs().active().size()); tester.clock().advance(Duration.ofHours(1)); app1.runJob(productionUsCentral1); tester.triggerJobs(); - assertEquals(3, tester.jobs().active().size()); + assertEquals(4, tester.jobs().active().size()); app1.runJob(testUsCentral1); tester.triggerJobs(); - assertEquals(2, tester.jobs().active().size()); + assertEquals(3, tester.jobs().active().size()); app1.runJob(productionApNortheast2); tester.triggerJobs(); - assertEquals(1, tester.jobs().active().size()); + assertEquals(2, tester.jobs().active().size()); app1.runJob(productionApNortheast1); tester.triggerJobs(); - assertEquals(List.of(), tester.jobs().active()); + assertEquals(1, tester.jobs().active().size()); tester.clock().advance(Duration.ofMinutes(30)); tester.triggerJobs(); - assertEquals(List.of(), tester.jobs().active()); + assertEquals(1, tester.jobs().active().size()); tester.clock().advance(Duration.ofMinutes(30)); app1.runJob(testApNortheast1); tester.triggerJobs(); - assertEquals(1, tester.jobs().active().size()); + assertEquals(2, tester.jobs().active().size()); app1.runJob(testApNortheast2); tester.triggerJobs(); - assertEquals(1, tester.jobs().active().size()); + assertEquals(2, tester.jobs().active().size()); app1.runJob(testUsEast3); tester.triggerJobs(); - assertEquals(1, tester.jobs().active().size()); + assertEquals(2, tester.jobs().active().size()); app1.runJob(productionApSoutheast1); tester.triggerJobs(); assertEquals(1, tester.jobs().active().size()); @@ -2514,7 +2514,7 @@ public class DeploymentTriggerTest { Version version3 = new Version("6.4"); tester.controllerTester().upgradeSystem(version3); - tests.runJob(systemTest) // Success in default cloud. + tests.runJob(systemTest) // Success in default cloud. .failDeployment(centuariTest); // Failure in centauri cloud. tester.upgrader().run(); @@ -2897,4 +2897,234 @@ public class DeploymentTriggerTest { assertEquals(Change.empty(), app.instance().change()); } + @Test + void miniBenchmark() { + String spec = """ + <deployment version="1.0"> + <parallel> + <instance id="instance0"> + <test tester-flavor="d-8-16-10" /> + <block-change version="true" revision="false" days="mon-fri,sun" hours="4-23" time-zone="UTC" /> + </instance> + <instance id="instance1"> + <test tester-flavor="d-8-16-10" /> + <block-change version="true" revision="false" days="mon-fri,sun" hours="4-23" time-zone="UTC" /> + </instance> + <instance id="instance2"> + <test tester-flavor="d-8-16-10" /> + <block-change version="true" revision="false" days="mon-fri,sun" hours="4-23" time-zone="UTC" /> + </instance> + <instance id="instance3"> + <test tester-flavor="d-8-16-10" /> + <block-change version="true" revision="false" days="mon-fri,sun" hours="4-23" time-zone="UTC" /> + </instance> + <instance id="stress"> + <staging /> + <block-change version="true" revision="false" days="mon-fri,sun" hours="4-23" time-zone="UTC" /> + </instance> + </parallel> + <instance id="beta1"> + <block-change version="true" revision="false" days="mon-fri,sun" hours="4-23" time-zone="UTC" /> + <block-change version="true" revision="false" days="sat" hours="0-23" time-zone="UTC" /> + <upgrade revision-change='when-clear' rollout='separate' revision-target='next' policy='conservative'/> + <prod> + <parallel> + <steps> + <region>us-east-3</region> + <test>us-east-3</test> + </steps> + <steps> + <region>us-west-1</region> + <test>us-west-1</test> + </steps> + <steps> + <region>eu-west-1</region> + <test>eu-west-1</test> + </steps> + <steps> + <region>us-central-1</region> + <test>us-central-1</test> + </steps> + </parallel> + </prod> + </instance> + <instance id="gamma5"> + <block-change version="true" revision="false" days="mon-fri,sun" hours="4-23" time-zone="UTC" /> + <block-change version="true" revision="false" days="sat" hours="0-23" time-zone="UTC" /> + <upgrade revision-change='when-clear' rollout='separate' revision-target='next' policy='conservative'/> + <prod> + <parallel> + <steps> + <region>us-east-3</region> + <test>us-east-3</test> + </steps> + <steps> + <region>us-west-1</region> + <test>us-west-1</test> + </steps> + <steps> + <region>eu-west-1</region> + <test>eu-west-1</test> + </steps> + <steps> + <region>us-central-1</region> + <test>us-central-1</test> + </steps> + </parallel> + </prod> + </instance> + <instance id="delta21"> + <block-change version="true" revision="false" days="mon-fri,sun" hours="4-23" time-zone="UTC" /> + <block-change version="true" revision="false" days="sat" hours="0-23" time-zone="UTC" /> + <upgrade revision-change='when-clear' rollout='separate' revision-target='next' policy='conservative'/> + <prod> + <parallel> + <steps> + <region>us-east-3</region> + <test>us-east-3</test> + </steps> + <steps> + <region>us-west-1</region> + <test>us-west-1</test> + </steps> + <steps> + <region>eu-west-1</region> + <test>eu-west-1</test> + </steps> + <steps> + <region>us-central-1</region> + <test>us-central-1</test> + </steps> + </parallel> + </prod> + </instance> + <instance id="prod21a"> + <block-change version="true" revision="false" days="mon-fri,sun" hours="4-23" time-zone="UTC" /> + <block-change version="true" revision="false" days="sat" hours="0-23" time-zone="UTC" /> + <upgrade revision-change='when-clear' rollout='separate' revision-target='next' policy='conservative'/> + <prod> + <parallel> + <steps> + <region>us-east-3</region> + <test>us-east-3</test> + </steps> + <steps> + <region>us-west-1</region> + <test>us-west-1</test> + </steps> + <steps> + <region>eu-west-1</region> + <test>eu-west-1</test> + </steps> + <steps> + <region>us-central-1</region> + <test>us-central-1</test> + </steps> + </parallel> + </prod> + </instance> + <instance id="prod21b"> + <block-change version="true" revision="false" days="mon-fri,sun" hours="4-23" time-zone="UTC" /> + <block-change version="true" revision="false" days="sat" hours="0-23" time-zone="UTC" /> + <upgrade revision-change='when-clear' rollout='separate' revision-target='next' policy='conservative'/> + <prod> + <parallel> + <steps> + <region>us-east-3</region> + <test>us-east-3</test> + </steps> + <steps> + <region>us-west-1</region> + <test>us-west-1</test> + </steps> + <steps> + <region>eu-west-1</region> + <test>eu-west-1</test> + </steps> + <steps> + <region>us-central-1</region> + <test>us-central-1</test> + </steps> + </parallel> + </prod> + </instance> + <instance id="prod21c"> + <block-change version="true" revision="false" days="mon-fri,sun" hours="4-23" time-zone="UTC" /> + <block-change version="true" revision="false" days="sat" hours="0-23" time-zone="UTC" /> + <upgrade revision-change='when-clear' rollout='separate' revision-target='next' policy='conservative'/> + <prod> + <parallel> + <steps> + <region>us-east-3</region> + <test>us-east-3</test> + </steps> + <steps> + <region>us-west-1</region> + <test>us-west-1</test> + </steps> + <steps> + <region>eu-west-1</region> + <test>eu-west-1</test> + </steps> + <steps> + <region>us-central-1</region> + <test>us-central-1</test> + </steps> + </parallel> + </prod> + </instance> + <instance id="cd10"> + <block-change version="true" revision="false" days="mon-fri,sun" hours="4-23" time-zone="UTC" /> + <block-change version="true" revision="false" days="sat" hours="0-23" time-zone="UTC" /> + <upgrade revision-change='when-clear' rollout='separate' revision-target='next' policy='conservative'/> + <prod> + <parallel> + <steps> + <region>us-east-3</region> + <test>us-east-3</test> + </steps> + <steps> + <region>us-west-1</region> + <test>us-west-1</test> + </steps> + <steps> + <region>eu-west-1</region> + <test>eu-west-1</test> + </steps> + <steps> + <region>us-central-1</region> + <test>us-central-1</test> + </steps> + </parallel> + </prod> + </instance> + <instance id="prod1"> + <block-change version="true" revision="false" days="mon-fri,sun" hours="4-23" time-zone="UTC" /> + <block-change version="true" revision="false" days="sat" hours="0-23" time-zone="UTC" /> + <upgrade revision-change='when-clear' rollout='separate' revision-target='next' policy='conservative'/> + <prod> + <parallel> + <steps> + <region>us-east-3</region> + <test>us-east-3</test> + </steps> + <steps> + <region>us-west-1</region> + <test>us-west-1</test> + </steps> + <steps> + <region>eu-west-1</region> + <test>eu-west-1</test> + </steps> + <steps> + <region>us-central-1</region> + <test>us-central-1</test> + </steps> + </parallel> + </prod> + </instance> + </deployment>"""; + tester.newDeploymentContext("t", "a", "prod1").submit(ApplicationPackageBuilder.fromDeploymentXml(spec)).deploy(); + } + } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java index 448bb9ac15f..2c1d7315adc 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java @@ -416,6 +416,7 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer Optional.empty(), LoadBalancer.State.active, Optional.of("dns-zone-1"), + Optional.empty(), Optional.of(new PrivateServiceInfo("service", List.of("arne")))))); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java index 04a623f819b..faef6de94ca 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java @@ -7,7 +7,6 @@ import com.yahoo.config.application.api.ValidationOverrides; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.RegionName; -import com.yahoo.config.provision.Tags; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.security.KeyUtils; import com.yahoo.slime.SlimeUtils; @@ -133,14 +132,12 @@ public class ApplicationSerializerTest { Map.of(new JobId(id1, DeploymentContext.productionUsEast3), List.of(applicationVersion2))); List<Instance> instances = List.of(new Instance(id1, - Tags.fromString("tag1 tag2"), deployments, Map.of(DeploymentContext.systemTest, Instant.ofEpochMilli(333)), List.of(rotation("foo", "default", "my-rotation", Set.of("us-west-1"))), rotationStatus, Change.of(new Version("6.1"))), new Instance(id3, - Tags.empty(), List.of(), Map.of(), List.of(), @@ -184,9 +181,6 @@ public class ApplicationSerializerTest { assertEquals(original.revisions().production(), serialized.revisions().production()); assertEquals(original.revisions().development(), serialized.revisions().development()); - assertEquals(original.require(id1.instance()).tags(), serialized.require(id1.instance()).tags()); - assertEquals(original.require(id3.instance()).tags(), serialized.require(id3.instance()).tags()); - assertEquals(original.deploymentSpec().xmlForm(), serialized.deploymentSpec().xmlForm()); assertEquals(original.validationOverrides().xmlForm(), serialized.validationOverrides().xmlForm()); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-clusters.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-clusters.json index c013ccb00fe..d9021b1d894 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-clusters.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-clusters.json @@ -42,17 +42,6 @@ "cost": 0.22 }, "target": { - "nodes": 2, - "groups": 1, - "nodeResources": { - "vcpu": 3.0, - "memoryGb": 8.0, - "diskGb": 50.0, - "bandwidthGbps": 1.0, - "diskSpeed": "slow", - "storageType": "remote" - }, - "cost": 0.29, "status": "ideal", "description": "Cluster is ideally scaled", "resources": { @@ -95,14 +84,6 @@ "disk": 0.0 } }, - "utilization": { - "idealCpu": 0.2, - "peakCpu": 0.35, - "idealMemory": 0.5, - "peakMemory": 0.65, - "idealDisk": 0.8, - "peakDisk": 1.0 - }, "scalingEvents": [ { "from": { @@ -135,8 +116,6 @@ "completion": 2234 } ], - "autoscalingStatusCode": "ideal", - "autoscalingStatus": "Cluster is ideally scaled", "scalingDuration": 360000 } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview.json index 38e9d8c823e..35dd6fc5398 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview.json @@ -547,7 +547,14 @@ ], "declared": true, "instance": "instance2", - "deploying": { }, + "deploying": { + "application": { + "build": 4, + "compileVersion": "6.1.0", + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" + } + }, "latestVersions": { "platform": { "platform": "6.1.0", @@ -569,7 +576,7 @@ "commit": "commit1" }, "at": 1000, - "upgrade": true, + "upgrade": false, "available": [ { "application": { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/athenz/AthenzApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/athenz/AthenzApiTest.java index 2f4b4154c08..3a539987443 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/athenz/AthenzApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/athenz/AthenzApiTest.java @@ -3,7 +3,6 @@ package com.yahoo.vespa.hosted.controller.restapi.athenz; import com.yahoo.application.container.handler.Request; import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.Tags; import com.yahoo.vespa.athenz.api.AthenzDomain; import com.yahoo.vespa.hosted.controller.ControllerTester; import com.yahoo.vespa.hosted.controller.restapi.ContainerTester; @@ -26,8 +25,8 @@ public class AthenzApiTest extends ControllerContainerTest { controllerTester.createTenant("sandbox", AthenzApiHandler.sandboxDomainIn(tester.controller().system()), 123L); controllerTester.createApplication("sandbox", "app", "default"); - tester.controller().applications().createInstance(ApplicationId.from("sandbox", "app", hostedOperator.getName()), Tags.empty()); - tester.controller().applications().createInstance(ApplicationId.from("sandbox", "app", defaultUser.getName()), Tags.empty()); + tester.controller().applications().createInstance(ApplicationId.from("sandbox", "app", hostedOperator.getName())); + tester.controller().applications().createInstance(ApplicationId.from("sandbox", "app", defaultUser.getName())); controllerTester.createApplication("sandbox", "opp", "default"); controllerTester.createTenant("tenant1", "domain1", 123L); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java index 09ebd062eae..18f685452f2 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java @@ -497,6 +497,7 @@ public class RoutingPoliciesTest { Optional.empty(), LoadBalancer.State.active, Optional.of("dns-zone-1"), + Optional.empty(), Optional.empty()); tester.controllerTester().configServer().putLoadBalancers(zone1, List.of(loadBalancer)); @@ -953,6 +954,7 @@ public class RoutingPoliciesTest { ipAddress, LoadBalancer.State.active, Optional.of("dns-zone-1").filter(__ -> lbHostname.isPresent()), + Optional.empty(), Optional.empty())); } return loadBalancers; diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java index 6f3d2d74bc6..f0182ae36e4 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java @@ -27,27 +27,21 @@ import com.yahoo.vespa.hosted.node.admin.task.util.process.Terminal; import java.net.URI; import java.nio.file.Files; import java.nio.file.Path; -import java.nio.file.Paths; import java.time.Clock; import java.time.Duration; import java.time.Instant; import java.time.ZoneOffset; import java.time.format.DateTimeFormatter; import java.util.ArrayList; -import java.util.Collections; import java.util.Comparator; -import java.util.HashMap; import java.util.List; -import java.util.Map; import java.util.Optional; import java.util.concurrent.TimeUnit; import java.util.function.Function; import java.util.logging.Level; import java.util.logging.Logger; -import java.util.stream.Collectors; import static com.yahoo.vespa.hosted.node.admin.maintenance.disk.DiskCleanupRule.Priority; -import static com.yahoo.yolean.Exceptions.uncheck; /** * @author freva @@ -172,33 +166,7 @@ public class StorageMaintainer { /** Checks if container has any new coredumps, reports and archives them if so */ public void handleCoreDumpsForContainer(NodeAgentContext context, Optional<Container> container, boolean throwIfCoreBeingWritten) { if (context.isDisabled(NodeAgentTask.CoreDumps)) return; - coredumpHandler.converge(context, () -> getCoredumpNodeAttributes(context, container), - container.map(Container::image), throwIfCoreBeingWritten); - } - - private Map<String, Object> getCoredumpNodeAttributes(NodeAgentContext context, Optional<Container> container) { - Map<String, String> attributes = new HashMap<>(); - attributes.put("hostname", context.node().hostname()); - attributes.put("system", context.zone().getSystemName().value()); - attributes.put("region", context.zone().getRegionName().value()); - attributes.put("environment", context.zone().getEnvironment().value()); - attributes.put("flavor", context.node().flavor()); - attributes.put("kernel_version", System.getProperty("os.version")); - attributes.put("cpu_microcode_version", getMicrocodeVersion()); - - container.map(c -> c.image().asString()).ifPresent(image -> attributes.put("docker_image", image)); - container.flatMap(c -> c.image().tag()).ifPresent(version -> attributes.put("vespa_version", version)); - context.node().parentHostname().ifPresent(parent -> attributes.put("parent_hostname", parent)); - context.node().owner().ifPresent(owner -> { - attributes.put("tenant", owner.tenant().value()); - attributes.put("application", owner.application().value()); - attributes.put("instance", owner.instance().value()); - }); - context.node().membership().ifPresent(membership -> { - attributes.put("cluster_id", membership.clusterId()); - attributes.put("cluster_type", membership.type().value()); - }); - return Collections.unmodifiableMap(attributes); + coredumpHandler.converge(context, container.map(Container::image), throwIfCoreBeingWritten); } /** @@ -225,18 +193,4 @@ public class StorageMaintainer { if (context.nodeType() != NodeType.tenant) context.paths().of("/").getFileSystem().createRoot(); } - - private String getMicrocodeVersion() { - String output = uncheck(() -> Files.readAllLines(Paths.get("/proc/cpuinfo")).stream() - .filter(line -> line.startsWith("microcode")) - .findFirst() - .orElse("microcode : UNKNOWN")); - - String[] results = output.split(":"); - if (results.length != 2) { - throw ConvergenceException.ofError("Result from detect microcode command not as expected: " + output); - } - - return results[1].trim(); - } } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandler.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandler.java index e2da984fa10..bfc4c09cf9e 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandler.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandler.java @@ -4,7 +4,6 @@ package com.yahoo.vespa.hosted.node.admin.maintenance.coredump; import com.yahoo.config.provision.DockerImage; import com.yahoo.security.KeyId; import com.yahoo.security.SecretSharedKey; -import com.yahoo.security.SharedKeyGenerator; import com.yahoo.vespa.flags.FetchVector; import com.yahoo.vespa.flags.FlagSource; import com.yahoo.vespa.flags.Flags; @@ -29,11 +28,9 @@ import java.io.OutputStream; import java.io.UncheckedIOException; import java.nio.file.Files; import java.nio.file.Path; -import java.nio.file.Paths; import java.time.Clock; import java.util.Comparator; import java.util.List; -import java.util.Map; import java.util.Optional; import java.util.UUID; import java.util.function.Predicate; @@ -101,8 +98,7 @@ public class CoredumpHandler { } - public void converge(NodeAgentContext context, Supplier<Map<String, Object>> nodeAttributesSupplier, - Optional<DockerImage> dockerImage, boolean throwIfCoreBeingWritten) { + public void converge(NodeAgentContext context, Optional<DockerImage> dockerImage, boolean throwIfCoreBeingWritten) { ContainerPath containerCrashPath = context.paths().of(crashPatchInContainer, context.users().vespa()); ContainerPath containerProcessingPath = containerCrashPath.resolve(PROCESSING_DIRECTORY_NAME); @@ -328,7 +324,7 @@ public class CoredumpHandler { } private String getMicrocodeVersion() { - String output = uncheck(() -> Files.readAllLines(Paths.get("/proc/cpuinfo")).stream() + String output = uncheck(() -> Files.readAllLines(doneCoredumpsPath.getFileSystem().getPath("/proc/cpuinfo")).stream() .filter(line -> line.startsWith("microcode")) .findFirst() .orElse("microcode : UNKNOWN")); diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/DefaultEnvWriter.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/DefaultEnvWriter.java index c5a28c26786..1889332ee49 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/DefaultEnvWriter.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/DefaultEnvWriter.java @@ -5,7 +5,6 @@ import com.yahoo.vespa.hosted.node.admin.component.TaskContext; import java.nio.file.Files; import java.nio.file.Path; -import java.nio.file.Paths; import java.util.ArrayList; import java.util.LinkedHashMap; import java.util.List; @@ -62,7 +61,7 @@ public class DefaultEnvWriter { return false; } else { context.log(logger, "Updating " + defaultEnvFile.toString()); - Path tempFile = Paths.get(defaultEnvFile + ".tmp"); + Path tempFile = defaultEnvFile.resolveSibling(defaultEnvFile.getFileName() + ".tmp"); uncheck(() -> Files.write(tempFile, newDefaultEnvLines)); uncheck(() -> Files.move(tempFile, defaultEnvFile, ATOMIC_MOVE)); return true; diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/UnixPath.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/UnixPath.java index ac5035216e9..fbef3def446 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/UnixPath.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/UnixPath.java @@ -12,7 +12,6 @@ import java.nio.file.NoSuchFileException; import java.nio.file.NotDirectoryException; import java.nio.file.OpenOption; import java.nio.file.Path; -import java.nio.file.Paths; import java.nio.file.StandardCopyOption; import java.nio.file.StandardOpenOption; import java.nio.file.attribute.FileAttribute; @@ -45,7 +44,7 @@ public class UnixPath { private final Path path; public UnixPath(Path path) { this.path = path; } - public UnixPath(String path) { this(Paths.get(path)); } + public UnixPath(String path) { this(Path.of(path)); } public Path toPath() { return path; } public UnixPath resolve(String relativeOrAbsolutePath) { return new UnixPath(path.resolve(relativeOrAbsolutePath)); } diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainerTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainerTest.java index 65b62848d4b..daae19478ed 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainerTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainerTest.java @@ -21,7 +21,6 @@ import java.io.IOException; import java.nio.file.FileSystem; import java.nio.file.Files; import java.nio.file.Path; -import java.nio.file.Paths; import java.time.Duration; import java.time.Instant; import java.util.List; @@ -63,7 +62,7 @@ public class StorageMaintainerTest { @Test void testNonExistingDiskUsed() { - DiskSize size = storageMaintainer.getDiskUsed(null, Paths.get("/fake/path")); + DiskSize size = storageMaintainer.getDiskUsed(null, Path.of("/fake/path")); assertEquals(DiskSize.ZERO, size); } diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandlerTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandlerTest.java index fd7c366fb8b..33d785eb04e 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandlerTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandlerTest.java @@ -160,6 +160,8 @@ public class CoredumpHandlerTest { .setCoreDumpPath(context.paths().of("/home/docker/dumps/container-123/id-123/dump_core.456")) .setDockerImage(DockerImage.fromString("example.com/vespa/ci:6.48.4")); + new UnixPath(fileSystem.getPath("/proc/cpuinfo")).createParents().writeUtf8File("microcode\t: 0xf0"); + ContainerPath coredumpDirectory = context.paths().of("/var/crash/id-123"); Files.createDirectories(coredumpDirectory.pathOnHost()); Files.createFile(coredumpDirectory.resolve("dump_core.456")); diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/DefaultEnvWriterTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/DefaultEnvWriterTest.java index bc461af0eb3..bd523a16705 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/DefaultEnvWriterTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/DefaultEnvWriterTest.java @@ -9,11 +9,12 @@ import java.io.File; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; -import java.nio.file.Paths; import java.util.logging.Logger; import static java.nio.file.StandardCopyOption.REPLACE_EXISTING; -import static org.junit.jupiter.api.Assertions.*; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.times; @@ -27,8 +28,8 @@ public class DefaultEnvWriterTest { @TempDir public File temporaryFolder; - private static final Path EXAMPLE_FILE = Paths.get("src/test/resources/default-env-example.txt"); - private static final Path EXPECTED_RESULT_FILE = Paths.get("src/test/resources/default-env-rewritten.txt"); + private static final Path EXAMPLE_FILE = Path.of("src/test/resources/default-env-example.txt"); + private static final Path EXPECTED_RESULT_FILE = Path.of("src/test/resources/default-env-rewritten.txt"); private final TaskContext context = mock(TaskContext.class); diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileFinderTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileFinderTest.java index ce193059fb2..bbc549230c1 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileFinderTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileFinderTest.java @@ -13,7 +13,6 @@ import java.nio.file.FileSystem; import java.nio.file.Files; import java.nio.file.NoSuchFileException; import java.nio.file.Path; -import java.nio.file.Paths; import java.nio.file.attribute.BasicFileAttributes; import java.nio.file.attribute.FileTime; import java.time.Duration; @@ -26,7 +25,10 @@ import java.util.stream.Collectors; import java.util.stream.Stream; import static java.util.Set.of; -import static org.junit.jupiter.api.Assertions.*; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -78,7 +80,7 @@ public class FileFinderTest { @Test void throws_if_prune_path_not_under_base_path() { assertThrows(IllegalArgumentException.class, () -> { - FileFinder.files(Paths.get("/some/path")).prune(Paths.get("/other/path")); + FileFinder.files(Path.of("/some/path")).prune(Path.of("/other/path")); }); } @@ -193,7 +195,7 @@ public class FileFinderTest { @Test void age_filter_test() { - Path path = Paths.get("/my/fake/path"); + Path path = Path.of("/my/fake/path"); when(attributes.lastModifiedTime()).thenReturn(FileTime.from(Instant.now().minus(Duration.ofHours(1)))); FileFinder.FileAttributes fileAttributes = new FileFinder.FileAttributes(path, attributes); @@ -206,7 +208,7 @@ public class FileFinderTest { @Test void size_filters() { - Path path = Paths.get("/my/fake/path"); + Path path = Path.of("/my/fake/path"); when(attributes.size()).thenReturn(100L); FileFinder.FileAttributes fileAttributes = new FileFinder.FileAttributes(path, attributes); @@ -219,7 +221,7 @@ public class FileFinderTest { @Test void filename_filters() { - Path path = Paths.get("/my/fake/path/some-12352-file.json"); + Path path = Path.of("/my/fake/path/some-12352-file.json"); FileFinder.FileAttributes fileAttributes = new FileFinder.FileAttributes(path, attributes); assertTrue(FileFinder.nameStartsWith("some-").test(fileAttributes)); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java index 4aa54b7f6fa..f792c511adb 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java @@ -56,30 +56,23 @@ public class AutoscalingMaintainer extends NodeRepositoryMaintainer { private void autoscale(ApplicationId application, NodeList applicationNodes) { try { - nodesByCluster(applicationNodes).forEach((clusterId, clusterNodes) -> autoscale(application, clusterId, clusterNodes)); + nodesByCluster(applicationNodes).forEach((clusterId, clusterNodes) -> autoscale(application, clusterId)); } catch (IllegalArgumentException e) { throw new IllegalArgumentException("Illegal arguments for " + application, e); } } - private void autoscale(ApplicationId applicationId, ClusterSpec.Id clusterId, NodeList clusterNodes) { - Optional<Application> application = nodeRepository().applications().get(applicationId); - if (application.isEmpty()) return; - Optional<Cluster> cluster = application.get().cluster(clusterId); - if (cluster.isEmpty()) return; - - Cluster updatedCluster = updateCompletion(cluster.get(), clusterNodes); - var autoscaling = autoscaler.autoscale(application.get(), updatedCluster, clusterNodes); - - if ( ! anyChanges(autoscaling, cluster.get(), updatedCluster, clusterNodes)) return; - + private void autoscale(ApplicationId applicationId, ClusterSpec.Id clusterId) { try (var lock = nodeRepository().applications().lock(applicationId)) { - application = nodeRepository().applications().get(applicationId); + Optional<Application> application = nodeRepository().applications().get(applicationId); if (application.isEmpty()) return; - cluster = application.get().cluster(clusterId); + Optional<Cluster> cluster = application.get().cluster(clusterId); if (cluster.isEmpty()) return; - clusterNodes = nodeRepository().nodes().list(Node.State.active).owner(applicationId).cluster(clusterId); + + NodeList clusterNodes = nodeRepository().nodes().list(Node.State.active).owner(applicationId).cluster(clusterId); + Cluster updatedCluster = updateCompletion(cluster.get(), clusterNodes); + var autoscaling = autoscaler.autoscale(application.get(), updatedCluster, clusterNodes); // 1. Update cluster info updatedCluster = updateCompletion(cluster.get(), clusterNodes); @@ -100,15 +93,6 @@ public class AutoscalingMaintainer extends NodeRepositoryMaintainer { } } - private boolean anyChanges(Autoscaling autoscaling, Cluster cluster, Cluster updatedCluster, NodeList clusterNodes) { - if (updatedCluster != cluster) return true; - if ( ! cluster.target().resources().equals(autoscaling.resources())) return true; - if ( ! cluster.target().status().equals(autoscaling.status())) return true; - if (autoscaling.resources().isPresent() && - !autoscaling.resources().get().equals(new AllocatableClusterResources(clusterNodes, nodeRepository()).advertisedResources())) return true; - return false; - } - private Applications applications() { return nodeRepository().applications(); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MaintenanceDeployment.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MaintenanceDeployment.java index a94adc5aaae..bcc571355e3 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MaintenanceDeployment.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MaintenanceDeployment.java @@ -190,7 +190,8 @@ class MaintenanceDeployment implements Closeable { markPreferToRetire(node, false, agent, nodeRepository); // Necessary if this failed, no-op otherwise // Immediately clean up if we reserved the node but could not activate or reserved a node on the wrong host - expectedNewNode.flatMap(node -> nodeRepository.nodes().node(node.hostname(), Node.State.reserved)) + expectedNewNode.flatMap(node -> nodeRepository.nodes().node(node.hostname())) + .filter(node -> node.state() == Node.State.reserved) .ifPresent(node -> nodeRepository.nodes().deallocate(node, agent, "Expired by " + agent)); } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeHealthTracker.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeHealthTracker.java index b602d2ac7cd..94683d463af 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeHealthTracker.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeHealthTracker.java @@ -96,7 +96,8 @@ public class NodeHealthTracker extends NodeRepositoryMaintainer { /** Get node by given hostname and application. The applicationLock must be held when calling this */ private Optional<Node> getNode(String hostname, ApplicationId application, @SuppressWarnings("unused") Mutex applicationLock) { - return nodeRepository().nodes().node(hostname, Node.State.active) + return nodeRepository().nodes().node(hostname) + .filter(node -> node.state() == Node.State.active) .filter(node -> node.allocation().isPresent()) .filter(node -> node.allocation().get().owner().equals(application)); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/IP.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/IP.java index c0d0b220767..b2becc7ecfd 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/IP.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/IP.java @@ -131,8 +131,9 @@ public class IP { * @throws IllegalArgumentException if there are IP conflicts with existing nodes */ public static List<Node> verify(List<Node> nodes, LockedNodeList allNodes) { + NodeList sortedNodes = allNodes.sortedBy(Comparator.comparing(Node::hostname)); for (var node : nodes) { - for (var other : allNodes) { + for (var other : sortedNodes) { if (node.equals(other)) continue; if (canAssignIpOf(other, node)) continue; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Nodes.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Nodes.java index b507d66e14f..5c2a6601ad8 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Nodes.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Nodes.java @@ -76,11 +76,11 @@ public class Nodes { public void rewrite() { Instant start = clock.instant(); int nodesWritten = 0; - for (Node.State state : Node.State.values()) { - List<Node> nodes = db.readNodes(state); + Map<Node.State, NodeList> nodes = list().groupingBy(Node::state); + for (var kv : nodes.entrySet()) { // TODO(mpolden): This should take the lock before writing - db.writeTo(state, nodes, Agent.system, Optional.empty()); - nodesWritten += nodes.size(); + db.writeTo(kv.getKey(), kv.getValue().asList(), Agent.system, Optional.empty()); + nodesWritten += kv.getValue().size(); } Instant end = clock.instant(); log.log(Level.INFO, String.format("Rewrote %d nodes in %s", nodesWritten, Duration.between(start, end))); @@ -88,24 +88,19 @@ public class Nodes { // ---------------- Query API ---------------------------------------------------------------- - /** - * Finds and returns the node with the hostname in any of the given states, or empty if not found - * - * @param hostname the full host name of the node - * @param inState the states the node may be in. If no states are given, it will be returned from any state - * @return the node, or empty if it was not found in any of the given states - */ - public Optional<Node> node(String hostname, Node.State... inState) { - return db.readNode(hostname, inState); + /** Finds and returns the node with given hostname, or empty if not found */ + public Optional<Node> node(String hostname) { + return db.readNode(hostname); } /** - * Returns a list of nodes in this repository in any of the given states + * Returns an unsorted list of all nodes in this repository, in any of the given states * - * @param inState the states to return nodes from. If no states are given, all nodes of the given type are returned + * @param inState the states to return nodes from. If no states are given, all nodes are returned */ public NodeList list(Node.State... inState) { - return NodeList.copyOf(db.readNodes(inState)); + NodeList nodes = NodeList.copyOf(db.readNodes()); + return inState.length == 0 ? nodes : nodes.state(Set.of(inState)); } /** Returns a locked list of all nodes in this repository */ @@ -768,13 +763,9 @@ public class Nodes { for (int i = 0; i < maxRetries; ++i) { Mutex lockToClose = lock(staleNode, timeout); try { - // As an optimization we first try finding the node in the same state - Optional<Node> freshNode = node(staleNode.hostname(), staleNode.state()); + Optional<Node> freshNode = node(staleNode.hostname()); if (freshNode.isEmpty()) { - freshNode = node(staleNode.hostname()); - if (freshNode.isEmpty()) { - return Optional.empty(); - } + return Optional.empty(); } if (node.type() != NodeType.tenant || diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CachingCurator.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CachingCurator.java index 90d30a3a8bc..589468c48b8 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CachingCurator.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CachingCurator.java @@ -208,7 +208,7 @@ public class CachingCurator { List<String> getChildren(Path path); /** - * Returns the a copy of the content of this child - which may be empty. + * Returns a copy of the content of this child - which may be empty. */ Optional<byte[]> getData(Path path); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDb.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDb.java index cebc185360a..b9821b48fba 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDb.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDb.java @@ -96,7 +96,7 @@ public class CuratorDb { db.create(nodesPath); // TODO(mpolden): Remove state paths after migration to nodesPath for (Node.State state : Node.State.values()) - db.create(toPath(state)); + db.create(toLegacyPath(state)); db.create(applicationsPath); db.create(inactiveJobsPath); db.create(infrastructureVersionsPath); @@ -117,7 +117,7 @@ public class CuratorDb { node = node.with(node.history().recordStateTransition(null, expectedState, agent, clock.instant())); // TODO(mpolden): Remove after migration to nodesPath byte[] serialized = nodeSerializer.toJson(node); - curatorTransaction.add(CuratorOperations.create(toPath(node).getAbsolute(), serialized)); + curatorTransaction.add(CuratorOperations.create(toLegacyPath(node).getAbsolute(), serialized)); curatorTransaction.add(CuratorOperations.create(nodePath(node).getAbsolute(), serialized)); } @@ -137,7 +137,7 @@ public class CuratorDb { /** Removes given nodes in transaction */ public void removeNodes(List<Node> nodes, NestedTransaction transaction) { for (Node node : nodes) { - Path path = toPath(node.state(), node.hostname()); + Path path = toLegacyPath(node.state(), node.hostname()); CuratorTransaction curatorTransaction = db.newCuratorTransactionIn(transaction); // TODO(mpolden): Remove after migration to nodesPath curatorTransaction.add(CuratorOperations.delete(path.getAbsolute())); @@ -237,8 +237,8 @@ public class CuratorDb { private void writeNode(Node.State toState, CuratorTransaction curatorTransaction, Node node, Node newNode) { byte[] nodeData = nodeSerializer.toJson(newNode); { // TODO(mpolden): Remove this after migration to nodesPath - String currentNodePath = toPath(node).getAbsolute(); - String newNodePath = toPath(toState, newNode.hostname()).getAbsolute(); + String currentNodePath = toLegacyPath(node).getAbsolute(); + String newNodePath = toLegacyPath(toState, newNode.hostname()).getAbsolute(); if (newNodePath.equals(currentNodePath)) { curatorTransaction.add(CuratorOperations.setData(currentNodePath, nodeData)); } else { @@ -255,61 +255,39 @@ public class CuratorDb { return node.status(); } - /** - * Returns all nodes which are in one of the given states. - * If no states are given this returns all nodes. - * - * @return the nodes in a mutable list owned by the caller - */ - public List<Node> readNodes(Node.State ... states) { - List<Node> nodes = new ArrayList<>(); - if (states.length == 0) - states = Node.State.values(); + /** Returns all existing nodes */ + public List<Node> readNodes() { CachingCurator.Session session = db.getSession(); - for (Node.State state : states) { - for (String hostname : session.getChildren(toPath(state))) { - Optional<Node> node = readNode(session, hostname, state); - node.ifPresent(nodes::add); // node might disappear between getChildren and getNode - } - } - return nodes; + return session.getChildren(nodesPath).stream() + .flatMap(hostname -> readNode(session, hostname).stream()) + .toList(); } - /** - * Returns a particular node, or empty if this node is not in any of the given states. - * If no states are given this returns the node if it is present in any state. - */ - public Optional<Node> readNode(CachingCurator.Session session, String hostname, Node.State ... states) { - if (states.length == 0) - states = Node.State.values(); - for (Node.State state : states) { - Optional<byte[]> nodeData = session.getData(toPath(state, hostname)); - if (nodeData.isPresent()) - return nodeData.map((data) -> nodeSerializer.fromJson(state, data)); - } - return Optional.empty(); + private Optional<Node> readNode(CachingCurator.Session session, String hostname) { + return session.getData(nodePath(hostname)).map(nodeSerializer::fromJson); } - /** - * Returns a particular node, or empty if this noe is not in any of the given states. - * If no states are given this returns the node if it is present in any state. - */ - public Optional<Node> readNode(String hostname, Node.State ... states) { - return readNode(db.getSession(), hostname, states); + /** Read node with given hostname, if any such node exists */ + public Optional<Node> readNode(String hostname) { + return readNode(db.getSession(), hostname); } - private Path toPath(Node.State nodeState) { return root.append(toDir(nodeState)); } + private Path toLegacyPath(Node.State nodeState) { return root.append(toDir(nodeState)); } - private Path toPath(Node node) { + private Path toLegacyPath(Node node) { return root.append(toDir(node.state())).append(node.hostname()); } - private Path toPath(Node.State nodeState, String nodeName) { + private Path toLegacyPath(Node.State nodeState, String nodeName) { return root.append(toDir(nodeState)).append(nodeName); } private Path nodePath(Node node) { - return nodesPath.append(node.hostname()); + return nodePath(node.hostname()); + } + + private Path nodePath(String hostname) { + return nodesPath.append(hostname); } /** Creates and returns the path to the lock for this application */ diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java index f448266b94b..39cccafb8ef 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java @@ -40,7 +40,6 @@ import com.yahoo.vespa.hosted.provision.node.Status; import com.yahoo.vespa.hosted.provision.node.TrustStoreItem; import java.io.IOException; -import java.nio.charset.StandardCharsets; import java.time.Instant; import java.util.ArrayList; import java.util.Collection; @@ -266,19 +265,16 @@ public class NodeSerializer { // ---------------- Deserialization -------------------------------------------------- - public Node fromJson(Node.State state, byte[] data) { - long key = Hashing.sipHash24().newHasher() - .putString(state.name(), StandardCharsets.UTF_8) - .putBytes(data).hash() - .asLong(); + public Node fromJson(byte[] data) { + long key = Hashing.sipHash24().newHasher().putBytes(data).hash().asLong(); try { - return cache.get(key, () -> nodeFromSlime(state, SlimeUtils.jsonToSlime(data).get())); + return cache.get(key, () -> nodeFromSlime(SlimeUtils.jsonToSlime(data).get())); } catch (ExecutionException e) { throw new UncheckedExecutionException(e); } } - private Node nodeFromSlime(Node.State state, Inspector object) { + private Node nodeFromSlime(Inspector object) { Flavor flavor = flavorFromSlime(object); return new Node(object.field(idKey).asString(), new IP.Config(ipAddressesFromSlime(object, ipAddressesKey), @@ -288,7 +284,7 @@ public class NodeSerializer { SlimeUtils.optionalString(object.field(parentHostnameKey)), flavor, statusFromSlime(object), - state, + nodeStateFromString(object.field(stateKey).asString()), allocationFromSlime(flavor.resources(), object.field(instanceKey)), historyFromSlime(object), nodeTypeFromString(object.field(nodeTypeKey).asString()), diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesResponse.java index 942f029bc6a..fd466108063 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesResponse.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesResponse.java @@ -105,16 +105,12 @@ class NodesResponse extends SlimeJsonResponse { } /** Outputs the nodes in the given states to a node array */ - private void nodesToSlime(Set<Node.State> statesToRead, Cursor parentObject) { + private void nodesToSlime(Set<Node.State> states, Cursor parentObject) { Cursor nodeArray = parentObject.setArray("nodes"); - boolean sortByNodeType = statesToRead.size() == 1; - statesToRead.stream().sorted().forEach(state -> { - NodeList nodes = nodeRepository.nodes().list(state); - if (sortByNodeType) { - nodes = nodes.sortedBy(Comparator.comparing(Node::type)); - } - toSlime(nodes, nodeArray); - }); + NodeList nodes = nodeRepository.nodes().list() + .state(states) + .sortedBy(Comparator.comparing(Node::hostname)); + toSlime(nodes, nodeArray); } private void toSlime(NodeList nodes, Cursor array) { diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeRepositoryTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeRepositoryTest.java index 0a179babc10..7d9a48f6773 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeRepositoryTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeRepositoryTest.java @@ -214,8 +214,6 @@ public class NodeRepositoryTest { tester.addHost("id2", "host1", "default", NodeType.host); host1 = tester.nodeRepository().nodes().node("host1").get(); assertEquals("This is the newly added node", "id2", host1.id()); - assertFalse("The old 'host1' is removed", - tester.nodeRepository().nodes().node("host1", Node.State.deprovisioned).isPresent()); assertFalse("Not transferred from deprovisioned host", host1.status().wantToRetire()); assertFalse("Not transferred from deprovisioned host", host1.status().wantToDeprovision()); assertTrue("Transferred from deprovisioned host", host1.history().hasEventAfter(History.Event.Type.deprovisioned, testStart)); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/RealDataScenarioTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/RealDataScenarioTest.java index e569e9b0382..dc1e1320ff2 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/RealDataScenarioTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/RealDataScenarioTest.java @@ -48,7 +48,6 @@ import java.util.function.Consumer; import java.util.logging.Logger; import java.util.regex.Matcher; import java.util.regex.Pattern; -import java.util.stream.Collectors; import java.util.stream.IntStream; import static com.yahoo.config.provision.NodeResources.DiskSpeed.any; @@ -142,7 +141,7 @@ public class RealDataScenarioTest { Consumer<String> consumer = input -> { if (state.get() != null) { String json = input.substring(input.indexOf("{\""), input.lastIndexOf('}') + 1); - Node node = nodeSerializer.fromJson(state.get(), json.getBytes(UTF_8)); + Node node = nodeSerializer.fromJson(json.getBytes(UTF_8)); nodeRepository.database().addNodesInState(List.of(node), state.get(), Agent.system); state.set(null); } else { diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainerTest.java index 6374bf52687..2da65fc1a2f 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainerTest.java @@ -78,10 +78,20 @@ public class AutoscalingMaintainerTest { tester.maintainer().maintain(); assertTrue(tester.deployer().lastDeployTime(app1).isEmpty()); // since autoscaling is off assertTrue(tester.deployer().lastDeployTime(app2).isPresent()); - assertNotEquals(Load.zero(), - tester.nodeRepository().applications().require(app1).cluster(cluster1.id()).get().target().peak()); - assertNotEquals(Load.zero(), - tester.nodeRepository().applications().require(app1).cluster(cluster1.id()).get().target().ideal()); + Load peakAt90 = tester.nodeRepository().applications().require(app1).cluster(cluster1.id()).get().target().peak(); + Load idealAt90 = tester.nodeRepository().applications().require(app1).cluster(cluster1.id()).get().target().ideal(); + assertNotEquals(Load.zero(), peakAt90); + assertNotEquals(Load.zero(), idealAt90); + + // Verify that load is updated even when there's no other change + tester.clock().advance(Duration.ofMinutes(10)); + tester.addMeasurements(0.8f, 0.8f, 0.8f, 0, 500, app1, cluster1.id()); + tester.maintainer().maintain(); + + Load peakAt80 = tester.nodeRepository().applications().require(app1).cluster(cluster1.id()).get().target().peak(); + Load idealAt80 = tester.nodeRepository().applications().require(app1).cluster(cluster1.id()).get().target().ideal(); + assertNotEquals(peakAt90, peakAt80); + assertNotEquals(idealAt90, idealAt80); } @Test diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java index 4f23ea3a578..44050fa747c 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java @@ -40,7 +40,6 @@ import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.TreeMap; -import java.util.stream.Collectors; import static org.junit.Assert.assertEquals; import static org.mockito.ArgumentMatchers.any; @@ -118,14 +117,14 @@ public class MetricsReporterTest { expectedMetrics.put("suspendedSeconds", 123L); expectedMetrics.put("numberOfServices", 0L); - expectedMetrics.put("cache.nodeObject.hitRate", 0.6D); + expectedMetrics.put("cache.nodeObject.hitRate", 0.7142857142857143D); expectedMetrics.put("cache.nodeObject.evictionCount", 0L); expectedMetrics.put("cache.nodeObject.size", 2L); nodeRepository.nodes().list(); - expectedMetrics.put("cache.curator.hitRate", 0.52D); + expectedMetrics.put("cache.curator.hitRate", 2D/3D); expectedMetrics.put("cache.curator.evictionCount", 0L); - expectedMetrics.put("cache.curator.size", 12L); + expectedMetrics.put("cache.curator.size", 3L); tester.clock().setInstant(Instant.ofEpochSecond(124)); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/os/OsVersionsTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/os/OsVersionsTest.java index 9f9e6b85545..b5735cfae84 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/os/OsVersionsTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/os/OsVersionsTest.java @@ -121,17 +121,16 @@ public class OsVersionsTest { // Activate target for (int i = 0; i < totalNodes; i += maxActiveUpgrades) { versions.resumeUpgradeOf(NodeType.host, true); - var nodes = hostNodes.get(); - var nodesUpgrading = nodes.changingOsVersion(); + NodeList nodes = hostNodes.get(); + NodeList nodesUpgrading = nodes.changingOsVersion(); assertEquals("Target is changed for a subset of nodes", maxActiveUpgrades, nodesUpgrading.size()); assertEquals("Wanted version is set for nodes upgrading", version1, minVersion(nodesUpgrading, OsVersion::wanted)); - var nodesOnLowestVersion = nodes.asList().stream() - .sorted(Comparator.comparing(node -> node.status().osVersion().current().orElse(Version.emptyVersion))) - .toList() - .subList(0, maxActiveUpgrades); + NodeList nodesOnLowestVersion = nodes.sortedBy(Comparator.comparing(node -> node.status().osVersion().current().orElse(Version.emptyVersion))) + .first(maxActiveUpgrades); assertEquals("Nodes on lowest version are told to upgrade", - nodesUpgrading.asList(), nodesOnLowestVersion); + nodesUpgrading.hostnames(), + nodesOnLowestVersion.hostnames()); completeReprovisionOf(nodesUpgrading.asList()); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDbTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDbTest.java index 42a0dd982ad..c0d6ab90f06 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDbTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDbTest.java @@ -28,10 +28,10 @@ public class CuratorDbTest { @Test public void can_read_stored_host_information() throws Exception { - String zkline = "{\"hostname\":\"host1\",\"ipAddresses\":[\"127.0.0.1\"],\"additionalIpAddresses\":[\"127.0.0.2\"],\"openStackId\":\"7951bb9d-3989-4a60-a21c-13690637c8ea\",\"flavor\":\"default\",\"created\":1421054425159, \"type\":\"host\"}"; - curator.framework().create().creatingParentsIfNeeded().forPath("/provision/v1/ready/host1", zkline.getBytes()); + String zkline = "{\"hostname\":\"host1\",\"state\":\"ready\",\"ipAddresses\":[\"127.0.0.1\"],\"additionalIpAddresses\":[\"127.0.0.2\"],\"openStackId\":\"7951bb9d-3989-4a60-a21c-13690637c8ea\",\"flavor\":\"default\",\"created\":1421054425159, \"type\":\"host\"}"; + curator.framework().create().creatingParentsIfNeeded().forPath("/provision/v1/nodes/host1", zkline.getBytes()); - List<Node> allocatedNodes = zkClient.readNodes(Node.State.ready); + List<Node> allocatedNodes = zkClient.readNodes(); assertEquals(1, allocatedNodes.size()); assertEquals(NodeType.host, allocatedNodes.get(0).type()); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializerTest.java index ab487cc7d04..d61a3d95a65 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializerTest.java @@ -67,7 +67,7 @@ public class NodeSerializerTest { public void provisioned_node_serialization() { Node node = createNode(); - Node copy = nodeSerializer.fromJson(Node.State.provisioned, nodeSerializer.toJson(node)); + Node copy = nodeSerializer.fromJson(nodeSerializer.toJson(node)); assertEquals(node.hostname(), copy.hostname()); assertEquals(node.id(), copy.id()); assertEquals(node.state(), copy.state()); @@ -99,7 +99,7 @@ public class NodeSerializerTest { node = node.downAt(Instant.ofEpochMilli(5), Agent.system) .upAt(Instant.ofEpochMilli(6), Agent.system) .downAt(Instant.ofEpochMilli(7), Agent.system); - Node copy = nodeSerializer.fromJson(Node.State.provisioned, nodeSerializer.toJson(node)); + Node copy = nodeSerializer.fromJson(nodeSerializer.toJson(node)); assertEquals(node.id(), copy.id()); assertEquals(node.hostname(), copy.hostname()); @@ -129,6 +129,7 @@ public class NodeSerializerTest { String nodeData = "{\n" + " \"type\" : \"tenant\",\n" + + " \"state\" : \"provisioned\",\n" + " \"rebootGeneration\" : 1,\n" + " \"currentRebootGeneration\" : 2,\n" + " \"flavor\" : \"large\",\n" + @@ -159,7 +160,7 @@ public class NodeSerializerTest { " \"ipAddresses\" : [\"127.0.0.1\"]\n" + "}"; - Node node = nodeSerializer.fromJson(Node.State.provisioned, Utf8.toBytes(nodeData)); + Node node = nodeSerializer.fromJson(Utf8.toBytes(nodeData)); assertEquals("large", node.flavor().name()); assertEquals(1, node.status().reboot().wanted()); @@ -187,7 +188,7 @@ public class NodeSerializerTest { assertEquals(1, node.history().events().size()); clock.advance(Duration.ofMinutes(2)); node = node.retire(Agent.application, clock.instant()); - Node copy = nodeSerializer.fromJson(Node.State.provisioned, nodeSerializer.toJson(node)); + Node copy = nodeSerializer.fromJson(nodeSerializer.toJson(node)); assertEquals(2, copy.history().events().size()); assertEquals(clock.instant().truncatedTo(MILLIS), copy.history().event(History.Event.Type.retired).get().at()); assertEquals(Agent.application, @@ -195,34 +196,37 @@ public class NodeSerializerTest { assertTrue(copy.allocation().get().membership().retired()); Node removable = copy.with(node.allocation().get().removable(true, true)); - Node removableCopy = nodeSerializer.fromJson(Node.State.provisioned, nodeSerializer.toJson(removable)); + Node removableCopy = nodeSerializer.fromJson( nodeSerializer.toJson(removable)); assertTrue(removableCopy.allocation().get().removable()); assertTrue(removableCopy.allocation().get().reusable()); } @Test public void assimilated_node_deserialization() { - Node node = nodeSerializer.fromJson(Node.State.active, ("{\n" + - " \"type\": \"tenant\",\n" + - " \"hostname\": \"assimilate2.vespahosted.yahoo.tld\",\n" + - " \"ipAddresses\": [\"127.0.0.1\"],\n" + - " \"openStackId\": \"\",\n" + - " \"flavor\": \"ugccloud-container\",\n" + - " \"instance\": {\n" + - " \"tenantId\": \"by_mortent\",\n" + - " \"applicationId\": \"ugc-assimilate\",\n" + - " \"instanceId\": \"default\",\n" + - " \"serviceId\": \"container/ugccloud-container/0/0\",\n" + - " \"restartGeneration\": 0,\n" + - " \"wantedVespaVersion\": \"6.42.2\"\n" + - " }\n" + - "}\n").getBytes()); + Node node = nodeSerializer.fromJson((""" + { + "type": "tenant", + "hostname": "assimilate2.vespahosted.yahoo.tld", + "state": "provisioned", + "ipAddresses": ["127.0.0.1"], + "openStackId": "", + "flavor": "ugccloud-container", + "instance": { + "tenantId": "by_mortent", + "applicationId": "ugc-assimilate", + "instanceId": "default", + "serviceId": "container/ugccloud-container/0/0", + "restartGeneration": 0, + "wantedVespaVersion": "6.42.2" + } + } + """).getBytes()); assertEquals(0, node.history().events().size()); assertTrue(node.allocation().isPresent()); assertEquals("ugccloud-container", node.allocation().get().membership().cluster().id().value()); assertEquals("container", node.allocation().get().membership().cluster().type().name()); assertEquals(0, node.allocation().get().membership().cluster().group().get().index()); - Node copy = nodeSerializer.fromJson(Node.State.provisioned, nodeSerializer.toJson(node)); + Node copy = nodeSerializer.fromJson(nodeSerializer.toJson(node)); assertEquals(0, copy.history().events().size()); } @@ -237,7 +241,7 @@ public class NodeSerializerTest { clock.instant()); node = node.with(node.status().withFailCount(0)); - Node copy2 = nodeSerializer.fromJson(Node.State.provisioned, nodeSerializer.toJson(node)); + Node copy2 = nodeSerializer.fromJson(nodeSerializer.toJson(node)); assertEquals(0, copy2.status().failCount()); } @@ -249,7 +253,7 @@ public class NodeSerializerTest { .parentHostname(parentHostname) .build(); - Node deserializedNode = nodeSerializer.fromJson(State.provisioned, nodeSerializer.toJson(node)); + Node deserializedNode = nodeSerializer.fromJson(nodeSerializer.toJson(node)); assertEquals(parentHostname, deserializedNode.parentHostname().get()); } @@ -257,7 +261,7 @@ public class NodeSerializerTest { @Test public void serializes_multiple_ip_addresses() { byte[] nodeWithMultipleIps = createNodeJson("node4.yahoo.tld", "127.0.0.4", "::4"); - Node deserializedNode = nodeSerializer.fromJson(State.provisioned, nodeWithMultipleIps); + Node deserializedNode = nodeSerializer.fromJson(nodeWithMultipleIps); assertEquals(ImmutableSet.of("127.0.0.4", "::4"), deserializedNode.ipConfig().primary()); } @@ -269,13 +273,13 @@ public class NodeSerializerTest { node = node.with(node.ipConfig().withPool(IP.Pool.of( Set.of("::1", "::2", "::3"), List.of(new Address("a"), new Address("b"), new Address("c"))))); - Node copy = nodeSerializer.fromJson(node.state(), nodeSerializer.toJson(node)); + Node copy = nodeSerializer.fromJson(nodeSerializer.toJson(node)); assertEquals(node.ipConfig().pool().ipSet(), copy.ipConfig().pool().ipSet()); assertEquals(Set.copyOf(node.ipConfig().pool().getAddressList()), Set.copyOf(copy.ipConfig().pool().getAddressList())); // Test round-trip without address pool (handle empty pool) node = createNode(); - copy = nodeSerializer.fromJson(node.state(), nodeSerializer.toJson(node)); + copy = nodeSerializer.fromJson(nodeSerializer.toJson(node)); assertEquals(node.ipConfig().pool().ipSet(), copy.ipConfig().pool().ipSet()); assertEquals(Set.copyOf(node.ipConfig().pool().getAddressList()), Set.copyOf(copy.ipConfig().pool().getAddressList())); } @@ -286,11 +290,12 @@ public class NodeSerializerTest { "{\n" + " \"type\" : \"tenant\",\n" + " \"flavor\" : \"large\",\n" + + " \"state\" : \"provisioned\",\n" + " \"openStackId\" : \"myId\",\n" + " \"hostname\" : \"myHostname\",\n" + " \"ipAddresses\" : [\"127.0.0.1\"]\n" + "}"; - Node node = nodeSerializer.fromJson(State.provisioned, Utf8.toBytes(nodeData)); + Node node = nodeSerializer.fromJson(Utf8.toBytes(nodeData)); assertFalse(node.status().wantToRetire()); } @@ -301,7 +306,7 @@ public class NodeSerializerTest { node = node.with(node.flavor().with(FlavorOverrides.ofDisk(1234)), Agent.system, clock.instant()); assertEquals(1234, node.flavor().resources().diskGb(), 0); - Node copy = nodeSerializer.fromJson(Node.State.provisioned, nodeSerializer.toJson(node)); + Node copy = nodeSerializer.fromJson(nodeSerializer.toJson(node)); assertEquals(1234, copy.flavor().resources().diskGb(), 0); assertEquals(node, copy); assertTrue(node.history().event(History.Event.Type.resized).isPresent()); @@ -312,21 +317,22 @@ public class NodeSerializerTest { String nodeData = "{\n" + " \"type\" : \"tenant\",\n" + + " \"state\" : \"provisioned\",\n" + " \"flavor\" : \"large\",\n" + " \"openStackId\" : \"myId\",\n" + " \"hostname\" : \"myHostname\",\n" + " \"ipAddresses\" : [\"127.0.0.1\"]\n" + "}"; - Node node = nodeSerializer.fromJson(State.provisioned, Utf8.toBytes(nodeData)); + Node node = nodeSerializer.fromJson(Utf8.toBytes(nodeData)); assertFalse(node.status().wantToDeprovision()); } @Test public void want_to_rebuild() { - Node node = nodeSerializer.fromJson(State.active, nodeSerializer.toJson(createNode())); + Node node = nodeSerializer.fromJson(nodeSerializer.toJson(createNode())); assertFalse(node.status().wantToRebuild()); node = node.with(node.status().withWantToRetire(true, false, true)); - node = nodeSerializer.fromJson(State.active, nodeSerializer.toJson(node)); + node = nodeSerializer.fromJson(nodeSerializer.toJson(node)); assertTrue(node.status().wantToRetire()); assertFalse(node.status().wantToDeprovision()); assertTrue(node.status().wantToRebuild()); @@ -337,6 +343,7 @@ public class NodeSerializerTest { String nodeWithWantedVespaVersion = "{\n" + " \"type\" : \"tenant\",\n" + + " \"state\" : \"provisioned\",\n" + " \"flavor\" : \"large\",\n" + " \"openStackId\" : \"myId\",\n" + " \"hostname\" : \"myHostname\",\n" + @@ -349,13 +356,13 @@ public class NodeSerializerTest { " \"wantedVespaVersion\": \"6.42.2\"\n" + " }\n" + "}"; - Node node = nodeSerializer.fromJson(State.active, Utf8.toBytes(nodeWithWantedVespaVersion)); + Node node = nodeSerializer.fromJson(Utf8.toBytes(nodeWithWantedVespaVersion)); assertEquals("6.42.2", node.allocation().get().membership().cluster().vespaVersion().toString()); } @Test public void os_version_serialization() { - Node serialized = nodeSerializer.fromJson(State.provisioned, nodeSerializer.toJson(createNode())); + Node serialized = nodeSerializer.fromJson(nodeSerializer.toJson(createNode())); assertFalse(serialized.status().osVersion().current().isPresent()); // Update OS version @@ -364,7 +371,7 @@ public class NodeSerializerTest { serialized.history().event(History.Event.Type.osUpgraded).isPresent()); serialized = serialized.withCurrentOsVersion(Version.fromString("7.2"), Instant.ofEpochMilli(123)) .withCurrentOsVersion(Version.fromString("7.2"), Instant.ofEpochMilli(456)); - serialized = nodeSerializer.fromJson(State.provisioned, nodeSerializer.toJson(serialized)); + serialized = nodeSerializer.fromJson(nodeSerializer.toJson(serialized)); assertEquals(Version.fromString("7.2"), serialized.status().osVersion().current().get()); var osUpgradedEvents = serialized.history().events().stream() .filter(event -> event.type() == History.Event.Type.osUpgraded) @@ -376,11 +383,11 @@ public class NodeSerializerTest { @Test public void firmware_check_serialization() { - Node node = nodeSerializer.fromJson(State.active, nodeSerializer.toJson(createNode())); + Node node = nodeSerializer.fromJson(nodeSerializer.toJson(createNode())); assertFalse(node.status().firmwareVerifiedAt().isPresent()); node = node.withFirmwareVerifiedAt(Instant.ofEpochMilli(100)); - node = nodeSerializer.fromJson(State.active, nodeSerializer.toJson(node)); + node = nodeSerializer.fromJson(nodeSerializer.toJson(node)); assertEquals(100, node.status().firmwareVerifiedAt().get().toEpochMilli()); assertEquals(Instant.ofEpochMilli(100), node.history().event(History.Event.Type.firmwareVerified).get().at()); } @@ -394,7 +401,7 @@ public class NodeSerializerTest { @Test public void reports_serialization() { - Node node = nodeSerializer.fromJson(State.active, nodeSerializer.toJson(createNode())); + Node node = nodeSerializer.fromJson(nodeSerializer.toJson(createNode())); assertTrue(node.reports().isEmpty()); var slime = new Slime(); @@ -407,7 +414,7 @@ public class NodeSerializerTest { var reports = new Reports.Builder().setReport(report).build(); node = node.with(reports); - node = nodeSerializer.fromJson(State.active, nodeSerializer.toJson(node)); + node = nodeSerializer.fromJson(nodeSerializer.toJson(node)); reports = node.reports(); assertFalse(reports.isEmpty()); @@ -422,11 +429,11 @@ public class NodeSerializerTest { @Test public void model_id_serialization() { - Node node = nodeSerializer.fromJson(State.active, nodeSerializer.toJson(createNode())); + Node node = nodeSerializer.fromJson(nodeSerializer.toJson(createNode())); assertFalse(node.modelName().isPresent()); node = node.withModelName("some model"); - node = nodeSerializer.fromJson(State.active, nodeSerializer.toJson(node)); + node = nodeSerializer.fromJson(nodeSerializer.toJson(node)); assertEquals("some model", node.modelName().get()); } @@ -447,7 +454,7 @@ public class NodeSerializerTest { node = node.with(node.allocation().get().withNetworkPorts(ports)); assertTrue(node.allocation().isPresent()); assertTrue(node.allocation().get().networkPorts().isPresent()); - Node copy = nodeSerializer.fromJson(Node.State.provisioned, nodeSerializer.toJson(node)); + Node copy = nodeSerializer.fromJson(nodeSerializer.toJson(node)); assertTrue(copy.allocation().isPresent()); assertTrue(copy.allocation().get().networkPorts().isPresent()); NetworkPorts portsCopy = node.allocation().get().networkPorts().get(); @@ -457,11 +464,11 @@ public class NodeSerializerTest { @Test public void switch_hostname_serialization() { - Node node = nodeSerializer.fromJson(State.active, nodeSerializer.toJson(createNode())); + Node node = nodeSerializer.fromJson(nodeSerializer.toJson(createNode())); assertFalse(node.switchHostname().isPresent()); String switchHostname = "switch0.example.com"; node = node.withSwitchHostname(switchHostname); - node = nodeSerializer.fromJson(State.active, nodeSerializer.toJson(node)); + node = nodeSerializer.fromJson(nodeSerializer.toJson(node)); assertEquals(switchHostname, node.switchHostname().get()); } @@ -469,25 +476,25 @@ public class NodeSerializerTest { public void exclusive_to_serialization() { Node.Builder builder = Node.create("myId", IP.Config.EMPTY, "myHostname", nodeFlavors.getFlavorOrThrow("default"), NodeType.host); - Node node = nodeSerializer.fromJson(State.provisioned, nodeSerializer.toJson(builder.build())); + Node node = nodeSerializer.fromJson(nodeSerializer.toJson(builder.build())); assertFalse(node.exclusiveToApplicationId().isPresent()); assertFalse(node.exclusiveToClusterType().isPresent()); ApplicationId exclusiveToApp = ApplicationId.from("tenant1", "app1", "instance1"); ClusterSpec.Type exclusiveToCluster = ClusterSpec.Type.admin; node = builder.exclusiveToApplicationId(exclusiveToApp).exclusiveToClusterType(exclusiveToCluster).build(); - node = nodeSerializer.fromJson(State.provisioned, nodeSerializer.toJson(node)); + node = nodeSerializer.fromJson(nodeSerializer.toJson(node)); assertEquals(exclusiveToApp, node.exclusiveToApplicationId().get()); assertEquals(exclusiveToCluster, node.exclusiveToClusterType().get()); } @Test public void truststore_serialization() { - Node node = nodeSerializer.fromJson(State.active, nodeSerializer.toJson(createNode())); + Node node = nodeSerializer.fromJson(nodeSerializer.toJson(createNode())); assertEquals(List.of(), node.trustedCertificates()); List<TrustStoreItem> trustStoreItems = List.of(new TrustStoreItem("foo", Instant.parse("2023-09-01T23:59:59Z")), new TrustStoreItem("bar", Instant.parse("2025-05-20T23:59:59Z"))); node = node.with(trustStoreItems); - node = nodeSerializer.fromJson(State.active, nodeSerializer.toJson(node)); + node = nodeSerializer.fromJson(nodeSerializer.toJson(node)); assertEquals(trustStoreItems, node.trustedCertificates()); } @@ -498,7 +505,7 @@ public class NodeSerializerTest { .cloudAccount(account) .exclusiveToApplicationId(ApplicationId.defaultId()) .build(); - node = nodeSerializer.fromJson(State.provisioned, nodeSerializer.toJson(node)); + node = nodeSerializer.fromJson(nodeSerializer.toJson(node)); assertEquals(account, node.cloudAccount()); } @@ -514,6 +521,7 @@ public class NodeSerializerTest { return ("{\"hostname\":\"" + hostname + "\"," + ipAddressJsonPart + "\"openStackId\":\"myId\"," + + "\"state\":\"provisioned\"," + "\"flavor\":\"default\",\"rebootGeneration\":0," + "\"currentRebootGeneration\":0,\"failCount\":0,\"history\":[],\"type\":\"tenant\"}") .getBytes(StandardCharsets.UTF_8); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AclProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AclProvisioningTest.java index d26ac4d3916..30a49a89e12 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AclProvisioningTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AclProvisioningTest.java @@ -196,11 +196,16 @@ public class AclProvisioningTest { // ACL for nodes with allocation trust their respective load balancer networks, if any for (var host : hosts) { - var acls = tester.nodeRepository().getChildAcls(host); + List<NodeAcl> acls = tester.nodeRepository().getChildAcls(host); assertEquals(2, acls.size()); - assertEquals(Set.of(), acls.get(0).trustedNetworks()); - assertEquals(application, acls.get(1).node().allocation().get().owner()); - assertEquals(lbNetworks, acls.get(1).trustedNetworks()); + for (var acl : acls) { + if (acl.node().allocation().isPresent()) { + assertEquals(lbNetworks, acl.trustedNetworks()); + assertEquals(application, acl.node().allocation().get().owner()); + } else { + assertEquals(Set.of(), acl.trustedNetworks()); + } + } } } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java index 30bd1250430..e32643860f5 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java @@ -85,8 +85,8 @@ public class LoadBalancerProvisionerTest { assertEquals(4443, get(lbApp1.get().get(0).instance().get().reals(), 1).port()); // A container is failed - Supplier<List<Node>> containers = () -> tester.getNodes(app1).container().asList(); - Node toFail = containers.get().get(0); + Supplier<NodeList> containers = () -> tester.getNodes(app1).container(); + Node toFail = containers.get().first().get(); tester.nodeRepository().nodes().fail(toFail.hostname(), Agent.system, this.getClass().getSimpleName()); // Redeploying replaces failed node and removes it from load balancer @@ -99,8 +99,8 @@ public class LoadBalancerProvisionerTest { .map(Real::hostname) .map(DomainName::value) .noneMatch(hostname -> hostname.equals(toFail.hostname()))); - assertEquals(containers.get().get(0).hostname(), get(loadBalancer.instance().get().reals(), 0).hostname().value()); - assertEquals(containers.get().get(1).hostname(), get(loadBalancer.instance().get().reals(), 1).hostname().value()); + assertEquals(containers.get().state(Node.State.active).hostnames(), + loadBalancer.instance().get().reals().stream().map(r -> r.hostname().value()).collect(Collectors.toSet())); assertSame("State is unchanged", LoadBalancer.State.active, loadBalancer.state()); // Add another container cluster to first app diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java index 110569a371a..5e9549aafbb 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java @@ -328,7 +328,7 @@ public class ProvisioningTester { } public void fail(String hostname) { - int beforeFailCount = nodeRepository.nodes().node(hostname, Node.State.active).get().status().failCount(); + int beforeFailCount = nodeRepository.nodes().node(hostname).get().status().failCount(); Node failedNode = nodeRepository.nodes().fail(hostname, Agent.system, "Failing to unit test"); assertTrue(nodeRepository.nodes().list(Node.State.failed).nodeType(NodeType.tenant).asList().contains(failedNode)); assertEquals(beforeFailCount + 1, failedNode.status().failCount()); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiTest.java index f5fe0e6aafd..9cc7d81055d 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiTest.java @@ -796,8 +796,8 @@ public class NodesV2ApiTest { // Filter nodes by osVersion assertResponse(new Request("http://localhost:8080/nodes/v2/node/?osVersion=7.5.2"), "{\"nodes\":[" + - "{\"url\":\"http://localhost:8080/nodes/v2/node/dockerhost2.yahoo.com\"}," + - "{\"url\":\"http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com\"}" + + "{\"url\":\"http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com\"}," + + "{\"url\":\"http://localhost:8080/nodes/v2/node/dockerhost2.yahoo.com\"}" + "]}"); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/active-nodes.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/active-nodes.json index c46bc6acbd2..15e93ce40a0 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/active-nodes.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/active-nodes.json @@ -1,19 +1,19 @@ { "nodes": [ + @include(cfg1.json), + @include(cfg2.json), + @include(docker-node1.json), + @include(docker-node2.json), + @include(docker-node3.json), + @include(docker-node4.json), + @include(docker-node5.json), + @include(node1.json), @include(node10.json), + @include(node13.json), @include(node14.json), + @include(node2.json), @include(node4.json), @include(node6.json), - @include(docker-container1.json), - @include(node13.json), - @include(node2.json), - @include(node1.json), - @include(docker-node3.json), - @include(docker-node4.json), - @include(docker-node5.json), - @include(docker-node2.json), - @include(docker-node1.json), - @include(cfg1.json), - @include(cfg2.json) + @include(docker-container1.json) ] } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/application2-nodes.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/application2-nodes.json index 4581ecba73d..6ffdb05fa67 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/application2-nodes.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/application2-nodes.json @@ -1,6 +1,6 @@ { "nodes": [ - @include(node6.json), - @include(node2.json) + @include(node2.json), + @include(node6.json) ] } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/content-nodes.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/content-nodes.json index 134d27f5717..cbf79f5b55d 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/content-nodes.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/content-nodes.json @@ -1,8 +1,8 @@ { "nodes": [ + @include(node2.json), @include(node4.json), @include(node6.json), - @include(docker-container1.json), - @include(node2.json) + @include(docker-container1.json) ] } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/enclave-nodes-recursive.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/enclave-nodes-recursive.json index 5c728d77920..540a0086cbf 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/enclave-nodes-recursive.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/enclave-nodes-recursive.json @@ -1,6 +1,6 @@ { "nodes": [ - @include(node3.json), - @include(docker-node2.json) + @include(docker-node2.json), + @include(node3.json) ] } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/enclave-nodes.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/enclave-nodes.json index 5bbc683c92a..33fd4daa699 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/enclave-nodes.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/enclave-nodes.json @@ -1,10 +1,10 @@ { "nodes":[ { - "url":"http://localhost:8080/nodes/v2/node/host3.yahoo.com" + "url":"http://localhost:8080/nodes/v2/node/dockerhost2.yahoo.com" }, { - "url":"http://localhost:8080/nodes/v2/node/dockerhost2.yahoo.com" + "url":"http://localhost:8080/nodes/v2/node/host3.yahoo.com" } ] } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/nodes-recursive-include-deprovisioned.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/nodes-recursive-include-deprovisioned.json index 66b44726e7e..dc6fd3a317d 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/nodes-recursive-include-deprovisioned.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/nodes-recursive-include-deprovisioned.json @@ -1,24 +1,24 @@ { "nodes": [ - @include(node7.json), - @include(node3.json), - @include(node10.json), @include(cfg1.json), @include(cfg2.json), + @include(docker-node1.json), + @include(docker-node2.json), @include(docker-node3.json), - @include(node14.json), - @include(node4.json), @include(docker-node4.json), - @include(node6.json), - @include(docker-container1.json), @include(docker-node5.json), - @include(docker-node2.json), + @include(dockerhost6.json), + @include(node1.json), + @include(node10.json), @include(node13.json), + @include(node14.json), @include(node2.json), - @include(docker-node1.json), - @include(node1.json), - @include(node55.json), + @include(node3.json), + @include(node4.json), @include(node5.json), - @include(dockerhost6.json) + @include(node55.json), + @include(node6.json), + @include(node7.json), + @include(docker-container1.json) ] } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/nodes-recursive.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/nodes-recursive.json index 7b52bc576ae..bd3b743e98e 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/nodes-recursive.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/nodes-recursive.json @@ -1,23 +1,23 @@ { "nodes": [ - @include(node7.json), - @include(node3.json), - @include(node10.json), @include(cfg1.json), @include(cfg2.json), + @include(docker-node1.json), + @include(docker-node2.json), @include(docker-node3.json), - @include(node14.json), - @include(node4.json), @include(docker-node4.json), - @include(node6.json), - @include(docker-container1.json), @include(docker-node5.json), - @include(docker-node2.json), + @include(node1.json), + @include(node10.json), @include(node13.json), + @include(node14.json), @include(node2.json), - @include(docker-node1.json), - @include(node1.json), + @include(node3.json), + @include(node4.json), + @include(node5.json), @include(node55.json), - @include(node5.json) + @include(node6.json), + @include(node7.json), + @include(docker-container1.json) ] } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/nodes.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/nodes.json index 86da5fb6e62..8147f92df31 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/nodes.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/nodes.json @@ -1,61 +1,61 @@ { "nodes": [ { - "url": "http://localhost:8080/nodes/v2/node/host7.yahoo.com" + "url": "http://localhost:8080/nodes/v2/node/cfg1.yahoo.com" }, { - "url": "http://localhost:8080/nodes/v2/node/host3.yahoo.com" + "url": "http://localhost:8080/nodes/v2/node/cfg2.yahoo.com" }, { - "url": "http://localhost:8080/nodes/v2/node/host10.yahoo.com" + "url": "http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com" }, { - "url": "http://localhost:8080/nodes/v2/node/cfg1.yahoo.com" + "url": "http://localhost:8080/nodes/v2/node/dockerhost2.yahoo.com" }, { - "url": "http://localhost:8080/nodes/v2/node/cfg2.yahoo.com" + "url": "http://localhost:8080/nodes/v2/node/dockerhost3.yahoo.com" }, { - "url": "http://localhost:8080/nodes/v2/node/dockerhost3.yahoo.com" + "url": "http://localhost:8080/nodes/v2/node/dockerhost4.yahoo.com" }, { - "url": "http://localhost:8080/nodes/v2/node/host14.yahoo.com" + "url": "http://localhost:8080/nodes/v2/node/dockerhost5.yahoo.com" }, { - "url": "http://localhost:8080/nodes/v2/node/host4.yahoo.com" + "url": "http://localhost:8080/nodes/v2/node/host1.yahoo.com" }, { - "url": "http://localhost:8080/nodes/v2/node/dockerhost4.yahoo.com" + "url": "http://localhost:8080/nodes/v2/node/host10.yahoo.com" }, { - "url": "http://localhost:8080/nodes/v2/node/host6.yahoo.com" + "url": "http://localhost:8080/nodes/v2/node/host13.yahoo.com" }, { - "url": "http://localhost:8080/nodes/v2/node/test-node-pool-102-2" + "url": "http://localhost:8080/nodes/v2/node/host14.yahoo.com" }, { - "url": "http://localhost:8080/nodes/v2/node/dockerhost5.yahoo.com" + "url": "http://localhost:8080/nodes/v2/node/host2.yahoo.com" }, { - "url": "http://localhost:8080/nodes/v2/node/dockerhost2.yahoo.com" + "url": "http://localhost:8080/nodes/v2/node/host3.yahoo.com" }, { - "url": "http://localhost:8080/nodes/v2/node/host13.yahoo.com" + "url": "http://localhost:8080/nodes/v2/node/host4.yahoo.com" }, { - "url": "http://localhost:8080/nodes/v2/node/host2.yahoo.com" + "url": "http://localhost:8080/nodes/v2/node/host5.yahoo.com" }, { - "url": "http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com" + "url": "http://localhost:8080/nodes/v2/node/host55.yahoo.com" }, { - "url": "http://localhost:8080/nodes/v2/node/host1.yahoo.com" + "url": "http://localhost:8080/nodes/v2/node/host6.yahoo.com" }, { - "url": "http://localhost:8080/nodes/v2/node/host55.yahoo.com" + "url": "http://localhost:8080/nodes/v2/node/host7.yahoo.com" }, { - "url": "http://localhost:8080/nodes/v2/node/host5.yahoo.com" + "url": "http://localhost:8080/nodes/v2/node/test-node-pool-102-2" } ] } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/states-recursive.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/states-recursive.json index 5ece0e642f1..2bdffea108a 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/states-recursive.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/states-recursive.json @@ -20,21 +20,21 @@ "active": { "url": "http://localhost:8080/nodes/v2/state/active", "nodes": [ + @include(cfg1.json), + @include(cfg2.json), + @include(docker-node1.json), + @include(docker-node2.json), + @include(docker-node3.json), + @include(docker-node4.json), + @include(docker-node5.json), + @include(node1.json), @include(node10.json), + @include(node13.json), @include(node14.json), + @include(node2.json), @include(node4.json), @include(node6.json), - @include(docker-container1.json), - @include(node13.json), - @include(node2.json), - @include(node1.json), - @include(docker-node3.json), - @include(docker-node4.json), - @include(docker-node5.json), - @include(docker-node2.json), - @include(docker-node1.json), - @include(cfg1.json), - @include(cfg2.json) + @include(docker-container1.json) ] }, "inactive": { diff --git a/screwdriver/build-vespa.sh b/screwdriver/build-vespa.sh index 6a93474c620..1a9288c007f 100755 --- a/screwdriver/build-vespa.sh +++ b/screwdriver/build-vespa.sh @@ -18,7 +18,7 @@ ccache -p if ! source $SOURCE_DIR/screwdriver/detect-what-to-build.sh; then echo "Could not detect what to build." - SHOULD_BUILD=all + SHOULD_BUILD=systemtest fi echo "Building: $SHOULD_BUILD" diff --git a/screwdriver/detect-what-to-build.sh b/screwdriver/detect-what-to-build.sh index 358c855f7bd..9666f058942 100755 --- a/screwdriver/detect-what-to-build.sh +++ b/screwdriver/detect-what-to-build.sh @@ -19,7 +19,7 @@ COMMITS=$(jq -re '.[].sha' <<< "$JSON") FILES=$(for C in $COMMITS; do JSON=$(curl -sLf https://api.github.com/repos/vespa-engine/vespa/commits/$C); jq -re '.files[].filename' <<< "$JSON"; done) -if [[ $PR_TITLE =~ \[run-systemtest\] ]]; then +if ! [[ $PR_TITLE =~ \[skip-systemtest\] ]]; then SHOULD_BUILD=systemtest elif [[ -z $FILES ]]; then SHOULD_BUILD=all diff --git a/screwdriver/release-container-image.sh b/screwdriver/release-container-image.sh index 6a9db770c82..6d8babe3dcc 100755 --- a/screwdriver/release-container-image.sh +++ b/screwdriver/release-container-image.sh @@ -49,7 +49,10 @@ docker context use vespa-context docker buildx create --name vespa-builder --driver docker-container --use docker buildx inspect --bootstrap -for data in "Dockerfile vespa" "Dockerfile.minimal vespa-minimal"; do +#The minimal image seem to have issues building on cd.screwdriver.cd. Needs investigation. +#for data in "Dockerfile vespa" "Dockerfile.minimal vespa-minimal"; do + +for data in "Dockerfile vespa"; do set -- $data DOCKER_FILE=$1 IMAGE_NAME=$2 diff --git a/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp b/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp index 584335803a8..d00342d0e51 100644 --- a/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp +++ b/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp @@ -976,8 +976,8 @@ TEST_F("Populates address space usage", DenseTensorAttributeHnswIndex) const auto& all = usage.get_all(); EXPECT_EQUAL(3u, all.size()); EXPECT_EQUAL(1u, all.count("tensor-store")); - EXPECT_EQUAL(1u, all.count("hnsw-node-store")); - EXPECT_EQUAL(1u, all.count("hnsw-link-store")); + EXPECT_EQUAL(1u, all.count("hnsw-levels-store")); + EXPECT_EQUAL(1u, all.count("hnsw-links-store")); } diff --git a/searchlib/src/tests/tensor/hnsw_index/hnsw_index_test.cpp b/searchlib/src/tests/tensor/hnsw_index/hnsw_index_test.cpp index d8e189717c3..7fbfe19b42b 100644 --- a/searchlib/src/tests/tensor/hnsw_index/hnsw_index_test.cpp +++ b/searchlib/src/tests/tensor/hnsw_index/hnsw_index_test.cpp @@ -522,7 +522,7 @@ TYPED_TEST(HnswIndexTest, memory_is_reclaimed_when_doing_changes_to_graph) EXPECT_EQ(0, mem_2.allocatedBytesOnHold()); this->remove_document(2); - size_t node_ref_growth = 0; + size_t nodes_growth = 0; if constexpr (TestFixture::is_single) { this->expect_level_0(1, {3}); this->expect_empty_level_0(2); @@ -532,11 +532,11 @@ TYPED_TEST(HnswIndexTest, memory_is_reclaimed_when_doing_changes_to_graph) this->expect_level_0(1, {2}); this->expect_empty_level_0(3); this->expect_level_0(2, {1}); - node_ref_growth = sizeof(HnswNode); // Entry for nodeid 3 added when adding doc 2 + nodes_growth = sizeof(HnswNode); // Entry for nodeid 3 added when adding doc 2 } auto mem_3 = this->memory_usage(); // We end up in the same state as before document 2 was added and effectively use the same amount of memory. - EXPECT_EQ((mem_1.usedBytes() - mem_1.deadBytes() + node_ref_growth), (mem_3.usedBytes() - mem_3.deadBytes())); + EXPECT_EQ((mem_1.usedBytes() - mem_1.deadBytes() + nodes_growth), (mem_3.usedBytes() - mem_3.deadBytes())); EXPECT_EQ(0, mem_3.allocatedBytesOnHold()); } @@ -642,14 +642,14 @@ make_graph_helper(HnswIndex<type>& index) using LinkArrayRef = typename HnswGraph<type>::LinkArrayRef; auto& graph = index.get_graph(); ResultGraph result(graph.size()); - assert(!graph.get_node_ref(0).valid()); + assert(!graph.get_levels_ref(0).valid()); for (uint32_t doc_id = 1; doc_id < graph.size(); ++doc_id) { auto& node = result[doc_id]; - auto node_ref = graph.get_node_ref(doc_id); + auto levels_ref = graph.get_levels_ref(doc_id); if constexpr (std::is_same_v<std::remove_reference_t<decltype(node)>, uint32_t>) { - node = node_ref.ref(); + node = levels_ref.ref(); } else { - LevelArrayRef level_array(graph.get_level_array(node_ref)); + LevelArrayRef level_array(graph.get_level_array(levels_ref)); for (uint32_t level = 0; level < level_array.size(); ++level) { if constexpr (std::is_same_v<std::remove_reference_t<decltype(node)>, std::vector<uint32_t>>) { node.emplace_back(level_array[level].load_relaxed().ref()); diff --git a/searchlib/src/tests/tensor/hnsw_nodeid_mapping/hnsw_nodeid_mapping_test.cpp b/searchlib/src/tests/tensor/hnsw_nodeid_mapping/hnsw_nodeid_mapping_test.cpp index ac8b21d6136..3dba1aa44ac 100644 --- a/searchlib/src/tests/tensor/hnsw_nodeid_mapping/hnsw_nodeid_mapping_test.cpp +++ b/searchlib/src/tests/tensor/hnsw_nodeid_mapping/hnsw_nodeid_mapping_test.cpp @@ -79,13 +79,13 @@ TEST_F(HnswNodeidMappingTest, free_ids_puts_nodeids_on_hold_list_and_then_free_l TEST_F(HnswNodeidMappingTest, on_load_populates_mapping) { std::vector<HnswNode> nodes(10); - nodes[1].ref().store_relaxed(EntryRef(1)); + nodes[1].levels_ref().store_relaxed(EntryRef(1)); nodes[1].store_docid(7); nodes[1].store_subspace(0); - nodes[2].ref().store_relaxed(EntryRef(2)); + nodes[2].levels_ref().store_relaxed(EntryRef(2)); nodes[2].store_docid(4); nodes[2].store_subspace(0); - nodes[7].ref().store_relaxed(EntryRef(3)); + nodes[7].levels_ref().store_relaxed(EntryRef(3)); nodes[7].store_docid(4); nodes[7].store_subspace(1); mapping.on_load(vespalib::ConstArrayRef(nodes.data(), nodes.size())); diff --git a/searchlib/src/tests/tensor/hnsw_saver/hnsw_save_load_test.cpp b/searchlib/src/tests/tensor/hnsw_saver/hnsw_save_load_test.cpp index bf4abdd7cf8..21ee88a46fe 100644 --- a/searchlib/src/tests/tensor/hnsw_saver/hnsw_save_load_test.cpp +++ b/searchlib/src/tests/tensor/hnsw_saver/hnsw_save_load_test.cpp @@ -113,7 +113,7 @@ void modify(HnswGraph<type> &graph) { graph.set_link_array(4, 1, V{7}); graph.set_link_array(7, 1, V{4}); - graph.set_entry_node({4, graph.get_node_ref(4), 1}); + graph.set_entry_node({4, graph.get_levels_ref(4), 1}); } @@ -124,7 +124,7 @@ public: GraphType copy; void expect_empty_d(uint32_t nodeid) const { - EXPECT_FALSE(copy.acquire_node_ref(nodeid).valid()); + EXPECT_FALSE(copy.acquire_levels_ref(nodeid).valid()); } void expect_level_0(uint32_t nodeid, const V& exp_links) const { @@ -160,7 +160,7 @@ public: } void expect_docid_and_subspace(uint32_t nodeid) const { - auto& node = copy.node_refs.get_elem_ref(nodeid); + auto& node = copy.nodes.get_elem_ref(nodeid); EXPECT_EQ(fake_docid<GraphType::index_type>(nodeid), fake_get_docid(node, nodeid)); EXPECT_EQ(fake_subspace<GraphType::index_type>(nodeid), node.acquire_subspace()); } diff --git a/searchlib/src/vespa/searchlib/attribute/address_space_components.cpp b/searchlib/src/vespa/searchlib/attribute/address_space_components.cpp index c8f2f93832a..244e01a3874 100644 --- a/searchlib/src/vespa/searchlib/attribute/address_space_components.cpp +++ b/searchlib/src/vespa/searchlib/attribute/address_space_components.cpp @@ -19,7 +19,7 @@ const vespalib::string AddressSpaceComponents::enum_store = "enum-store"; const vespalib::string AddressSpaceComponents::multi_value = "multi-value"; const vespalib::string AddressSpaceComponents::tensor_store = "tensor-store"; const vespalib::string AddressSpaceComponents::shared_string_repo = "shared-string-repo"; -const vespalib::string AddressSpaceComponents::hnsw_node_store = "hnsw-node-store"; -const vespalib::string AddressSpaceComponents::hnsw_link_store = "hnsw-link-store"; +const vespalib::string AddressSpaceComponents::hnsw_levels_store = "hnsw-levels-store"; +const vespalib::string AddressSpaceComponents::hnsw_links_store = "hnsw-links-store"; } diff --git a/searchlib/src/vespa/searchlib/attribute/address_space_components.h b/searchlib/src/vespa/searchlib/attribute/address_space_components.h index 3d409ece973..a8a41a024f8 100644 --- a/searchlib/src/vespa/searchlib/attribute/address_space_components.h +++ b/searchlib/src/vespa/searchlib/attribute/address_space_components.h @@ -18,8 +18,8 @@ public: static const vespalib::string multi_value; static const vespalib::string tensor_store; static const vespalib::string shared_string_repo; - static const vespalib::string hnsw_node_store; - static const vespalib::string hnsw_link_store; + static const vespalib::string hnsw_levels_store; + static const vespalib::string hnsw_links_store; }; } diff --git a/searchlib/src/vespa/searchlib/memoryindex/field_index_base.h b/searchlib/src/vespa/searchlib/memoryindex/field_index_base.h index 93fe77ba50b..2d6d367af3b 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/field_index_base.h +++ b/searchlib/src/vespa/searchlib/memoryindex/field_index_base.h @@ -78,7 +78,7 @@ protected: GenerationHandler _generationHandler; DictionaryTree _dict; FeatureStore _featureStore; - uint32_t _fieldId; + const uint32_t _fieldId; FieldIndexRemover _remover; std::unique_ptr<IOrderedFieldIndexInserter> _inserter; index::FieldLengthCalculator _calculator; diff --git a/searchlib/src/vespa/searchlib/memoryindex/field_index_collection.h b/searchlib/src/vespa/searchlib/memoryindex/field_index_collection.h index 7a65195e843..a9f597e6296 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/field_index_collection.h +++ b/searchlib/src/vespa/searchlib/memoryindex/field_index_collection.h @@ -28,7 +28,7 @@ private: using GenerationHandler = vespalib::GenerationHandler; std::vector<std::unique_ptr<IFieldIndex>> _fieldIndexes; - uint32_t _numFields; + const uint32_t _numFields; public: FieldIndexCollection(const index::Schema& schema, const index::IFieldLengthInspector& inspector); diff --git a/searchlib/src/vespa/searchlib/memoryindex/field_inverter.h b/searchlib/src/vespa/searchlib/memoryindex/field_inverter.h index e68f664603e..7995dc56de8 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/field_inverter.h +++ b/searchlib/src/vespa/searchlib/memoryindex/field_inverter.h @@ -97,7 +97,7 @@ private: class ElemInfo { public: - int32_t _weight; + const int32_t _weight; uint32_t _len; uint32_t _field_length; @@ -161,7 +161,7 @@ private: using UInt32Vector = std::vector<uint32_t, vespalib::allocator_large<uint32_t>>; // Current field state. - uint32_t _fieldId; // current field id + const uint32_t _fieldId; // current field id uint32_t _elem; // current element uint32_t _wpos; // current word pos uint32_t _docId; diff --git a/searchlib/src/vespa/searchlib/memoryindex/invert_task.h b/searchlib/src/vespa/searchlib/memoryindex/invert_task.h index b898deb5c16..a351fd2a10f 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/invert_task.h +++ b/searchlib/src/vespa/searchlib/memoryindex/invert_task.h @@ -27,7 +27,7 @@ class InvertTask : public vespalib::Executor::Task const std::vector<std::unique_ptr<FieldInverter>>& _inverters; const std::vector<std::unique_ptr<UrlFieldInverter>>& _uri_inverters; const document::Document& _doc; - uint32_t _lid; + const uint32_t _lid; std::remove_reference_t<OnWriteDoneType> _on_write_done; public: InvertTask(const DocumentInverterContext& inv_context, const InvertContext& context, const std::vector<std::unique_ptr<FieldInverter>>& inverters, const std::vector<std::unique_ptr<UrlFieldInverter>>& uri_inverters, uint32_t lid, const document::Document& doc, OnWriteDoneType on_write_done); diff --git a/searchlib/src/vespa/searchlib/memoryindex/posting_list_entry.h b/searchlib/src/vespa/searchlib/memoryindex/posting_list_entry.h index ef0866e957b..a8cc7fce1f2 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/posting_list_entry.h +++ b/searchlib/src/vespa/searchlib/memoryindex/posting_list_entry.h @@ -13,7 +13,6 @@ class InterleavedFeatures { protected: uint16_t _num_occs; uint16_t _field_length; - public: InterleavedFeatures() : _num_occs(0), diff --git a/searchlib/src/vespa/searchlib/tensor/hnsw_graph.cpp b/searchlib/src/vespa/searchlib/tensor/hnsw_graph.cpp index 4be7a3e135c..e707b464049 100644 --- a/searchlib/src/vespa/searchlib/tensor/hnsw_graph.cpp +++ b/searchlib/src/vespa/searchlib/tensor/hnsw_graph.cpp @@ -9,13 +9,13 @@ namespace search::tensor { template <HnswIndexType type> HnswGraph<type>::HnswGraph() - : node_refs(), - node_refs_size(1u), - nodes(HnswIndex<type>::make_default_node_store_config(), {}), - links(HnswIndex<type>::make_default_link_store_config(), {}), + : nodes(), + nodes_size(1u), + levels_store(HnswIndex<type>::make_default_level_array_store_config(), {}), + links_store(HnswIndex<type>::make_default_link_array_store_config(), {}), entry_nodeid_and_level() { - node_refs.ensure_size(1, NodeType()); + nodes.ensure_size(1, NodeType()); EntryNode entry; set_entry_node(entry); } @@ -24,68 +24,68 @@ template <HnswIndexType type> HnswGraph<type>::~HnswGraph() = default; template <HnswIndexType type> -typename HnswGraph<type>::NodeRef +typename HnswGraph<type>::LevelsRef HnswGraph<type>::make_node(uint32_t nodeid, uint32_t docid, uint32_t subspace, uint32_t num_levels) { - node_refs.ensure_size(nodeid + 1, NodeType()); + nodes.ensure_size(nodeid + 1, NodeType()); // A document cannot be added twice. - assert(!get_node_ref(nodeid).valid()); + assert(!get_levels_ref(nodeid).valid()); // Note: The level array instance lives as long as the document is present in the index. std::vector<AtomicEntryRef> levels(num_levels, AtomicEntryRef()); - auto node_ref = nodes.add(levels); - auto& node = node_refs[nodeid]; - node.ref().store_release(node_ref); + auto levels_ref = levels_store.add(levels); + auto& node = nodes[nodeid]; + node.levels_ref().store_release(levels_ref); node.store_docid(docid); node.store_subspace(subspace); - if (nodeid >= node_refs_size.load(std::memory_order_relaxed)) { - node_refs_size.store(nodeid + 1, std::memory_order_release); + if (nodeid >= nodes_size.load(std::memory_order_relaxed)) { + nodes_size.store(nodeid + 1, std::memory_order_release); } - return node_ref; + return levels_ref; } template <HnswIndexType type> void HnswGraph<type>::remove_node(uint32_t nodeid) { - auto node_ref = get_node_ref(nodeid); - assert(node_ref.valid()); - auto levels = nodes.get(node_ref); + auto levels_ref = get_levels_ref(nodeid); + assert(levels_ref.valid()); + auto levels = levels_store.get(levels_ref); vespalib::datastore::EntryRef invalid; - node_refs[nodeid].ref().store_release(invalid); + nodes[nodeid].levels_ref().store_release(invalid); // Ensure data referenced through the old ref can be recycled: - nodes.remove(node_ref); + levels_store.remove(levels_ref); for (size_t i = 0; i < levels.size(); ++i) { auto old_links_ref = levels[i].load_relaxed(); - links.remove(old_links_ref); + links_store.remove(old_links_ref); } - if (nodeid + 1 == node_refs_size.load(std::memory_order_relaxed)) { - trim_node_refs_size(); + if (nodeid + 1 == nodes_size.load(std::memory_order_relaxed)) { + trim_nodes_size(); } } template <HnswIndexType type> void -HnswGraph<type>::trim_node_refs_size() +HnswGraph<type>::trim_nodes_size() { - uint32_t check_nodeid = node_refs_size.load(std::memory_order_relaxed) - 1; - while (check_nodeid > 0u && !get_node_ref(check_nodeid).valid()) { + uint32_t check_nodeid = nodes_size.load(std::memory_order_relaxed) - 1; + while (check_nodeid > 0u && !get_levels_ref(check_nodeid).valid()) { --check_nodeid; } - node_refs_size.store(check_nodeid + 1, std::memory_order_release); + nodes_size.store(check_nodeid + 1, std::memory_order_release); } template <HnswIndexType type> void HnswGraph<type>::set_link_array(uint32_t nodeid, uint32_t level, const LinkArrayRef& new_links) { - auto new_links_ref = links.add(new_links); - auto node_ref = get_node_ref(nodeid); - assert(node_ref.valid()); - auto levels = nodes.get_writable(node_ref); + auto new_links_ref = links_store.add(new_links); + auto levels_ref = get_levels_ref(nodeid); + assert(levels_ref.valid()); + auto levels = levels_store.get_writable(levels_ref); assert(level < levels.size()); auto old_links_ref = levels[level].load_relaxed(); levels[level].store_release(new_links_ref); - links.remove(old_links_ref); + links_store.remove(old_links_ref); } template <HnswIndexType type> @@ -93,17 +93,17 @@ typename HnswGraph<type>::Histograms HnswGraph<type>::histograms() const { Histograms result; - size_t num_nodes = node_refs_size.load(std::memory_order_acquire); + size_t num_nodes = nodes_size.load(std::memory_order_acquire); for (size_t i = 0; i < num_nodes; ++i) { - auto node_ref = acquire_node_ref(i); - if (node_ref.valid()) { + auto levels_ref = acquire_levels_ref(i); + if (levels_ref.valid()) { uint32_t levels = 0; uint32_t l0links = 0; - auto level_array = nodes.get(node_ref); + auto level_array = levels_store.get(levels_ref); levels = level_array.size(); if (levels > 0) { auto links_ref = level_array[0].load_acquire(); - auto link_array = links.get(links_ref); + auto link_array = links_store.get(links_ref); l0links = link_array.size(); } while (result.level_histogram.size() <= levels) { @@ -125,7 +125,7 @@ HnswGraph<type>::set_entry_node(EntryNode node) { uint64_t value = node.level; value <<= 32; value |= node.nodeid; - if (node.node_ref.valid()) { + if (node.levels_ref.valid()) { assert(node.level >= 0); assert(node.nodeid > 0); } else { diff --git a/searchlib/src/vespa/searchlib/tensor/hnsw_graph.h b/searchlib/src/vespa/searchlib/tensor/hnsw_graph.h index 3fcb1f09eeb..f56d6e60145 100644 --- a/searchlib/src/vespa/searchlib/tensor/hnsw_graph.h +++ b/searchlib/src/vespa/searchlib/tensor/hnsw_graph.h @@ -31,74 +31,74 @@ struct HnswGraph { using NodeType = typename HnswIndexTraits<type>::NodeType; // Provides mapping from document id -> node reference. - // The reference is used to lookup the node data in NodeStore. - using NodeRefVector = vespalib::RcuVector<NodeType>; - using NodeRef = vespalib::datastore::EntryRef; + // The reference is used to lookup the node data in LevelArrayStore. + using NodeVector = vespalib::RcuVector<NodeType>; + using LevelsRef = vespalib::datastore::EntryRef; // This stores the level arrays for all nodes. // Each node consists of an array of levels (from level 0 to n) where each entry is a reference to the link array at that level. - using NodeStore = vespalib::datastore::ArrayStore<AtomicEntryRef, LevelArrayEntryRefType>; - using LevelArrayRef = NodeStore::ConstArrayRef; + using LevelArrayStore = vespalib::datastore::ArrayStore<AtomicEntryRef, LevelArrayEntryRefType>; + using LevelArrayRef = LevelArrayStore::ConstArrayRef; // This stores all link arrays. // A link array consists of the document ids of the nodes a particular node is linked to. - using LinkStore = vespalib::datastore::ArrayStore<uint32_t, LinkArrayEntryRefType>; - using LinkArrayRef = LinkStore::ConstArrayRef; + using LinkArrayStore = vespalib::datastore::ArrayStore<uint32_t, LinkArrayEntryRefType>; + using LinkArrayRef = LinkArrayStore::ConstArrayRef; - NodeRefVector node_refs; - std::atomic<uint32_t> node_refs_size; - NodeStore nodes; - LinkStore links; + NodeVector nodes; + std::atomic<uint32_t> nodes_size; + LevelArrayStore levels_store; + LinkArrayStore links_store; std::atomic<uint64_t> entry_nodeid_and_level; HnswGraph(); ~HnswGraph(); - NodeRef make_node(uint32_t nodeid, uint32_t docid, uint32_t subspace, uint32_t num_levels); + LevelsRef make_node(uint32_t nodeid, uint32_t docid, uint32_t subspace, uint32_t num_levels); void remove_node(uint32_t nodeid); - void trim_node_refs_size(); + void trim_nodes_size(); - NodeRef get_node_ref(uint32_t nodeid) const { - return node_refs.get_elem_ref(nodeid).ref().load_relaxed(); // Called from writer only + LevelsRef get_levels_ref(uint32_t nodeid) const { + return nodes.get_elem_ref(nodeid).levels_ref().load_relaxed(); // Called from writer only } - const NodeType& acquire_node_refs_elem_ref(uint32_t nodeid) const { - return node_refs.acquire_elem_ref(nodeid); + const NodeType& acquire_node(uint32_t nodeid) const { + return nodes.acquire_elem_ref(nodeid); } - NodeRef acquire_node_ref(uint32_t nodeid) const { - return node_refs.acquire_elem_ref(nodeid).ref().load_acquire(); + LevelsRef acquire_levels_ref(uint32_t nodeid) const { + return nodes.acquire_elem_ref(nodeid).levels_ref().load_acquire(); } - bool still_valid(uint32_t nodeid, NodeRef node_ref) const { - return node_ref.valid() && (acquire_node_ref(nodeid) == node_ref); + bool still_valid(uint32_t nodeid, LevelsRef levels_ref) const { + return levels_ref.valid() && (acquire_levels_ref(nodeid) == levels_ref); } - LevelArrayRef get_level_array(NodeRef node_ref) const { - if (node_ref.valid()) { - return nodes.get(node_ref); + LevelArrayRef get_level_array(LevelsRef levels_ref) const { + if (levels_ref.valid()) { + return levels_store.get(levels_ref); } return LevelArrayRef(); } LevelArrayRef get_level_array(uint32_t nodeid) const { - auto node_ref = get_node_ref(nodeid); - return get_level_array(node_ref); + auto levels_ref = get_levels_ref(nodeid); + return get_level_array(levels_ref); } LevelArrayRef acquire_level_array(uint32_t nodeid) const { - auto node_ref = acquire_node_ref(nodeid); - return get_level_array(node_ref); + auto levels_ref = acquire_levels_ref(nodeid); + return get_level_array(levels_ref); } LinkArrayRef get_link_array(LevelArrayRef levels, uint32_t level) const { if (level < levels.size()) { auto links_ref = levels[level].load_acquire(); if (links_ref.valid()) { - return links.get(links_ref); + return links_store.get(links_ref); } } return LinkArrayRef(); @@ -114,8 +114,8 @@ struct HnswGraph { return get_link_array(levels, level); } - LinkArrayRef get_link_array(NodeRef node_ref, uint32_t level) const { - auto levels = get_level_array(node_ref); + LinkArrayRef get_link_array(LevelsRef levels_ref, uint32_t level) const { + auto levels = get_level_array(levels_ref); return get_link_array(levels, level); } @@ -123,16 +123,16 @@ struct HnswGraph { struct EntryNode { uint32_t nodeid; - NodeRef node_ref; + LevelsRef levels_ref; int32_t level; EntryNode() : nodeid(0), // Note that nodeid 0 is reserved and never used - node_ref(), + levels_ref(), level(-1) {} - EntryNode(uint32_t nodeid_in, NodeRef node_ref_in, int32_t level_in) + EntryNode(uint32_t nodeid_in, LevelsRef levels_ref_in, int32_t level_in) : nodeid(nodeid_in), - node_ref(node_ref_in), + levels_ref(levels_ref_in), level(level_in) {} }; @@ -148,18 +148,18 @@ struct HnswGraph { while (true) { uint64_t value = get_entry_atomic(); entry.nodeid = (uint32_t)value; - entry.node_ref = acquire_node_ref(entry.nodeid); + entry.levels_ref = acquire_levels_ref(entry.nodeid); entry.level = (int32_t)(value >> 32); if ((entry.nodeid == 0) && (entry.level == -1) - && ! entry.node_ref.valid()) + && ! entry.levels_ref.valid()) { // invalid in every way return entry; } if ((entry.nodeid > 0) && (entry.level > -1) - && entry.node_ref.valid() + && entry.levels_ref.valid() && (get_entry_atomic() == value)) { // valid in every way @@ -168,7 +168,7 @@ struct HnswGraph { } } - size_t size() const { return node_refs_size.load(std::memory_order_acquire); } + size_t size() const { return nodes_size.load(std::memory_order_acquire); } struct Histograms { std::vector<uint32_t> level_histogram; diff --git a/searchlib/src/vespa/searchlib/tensor/hnsw_index.cpp b/searchlib/src/vespa/searchlib/tensor/hnsw_index.cpp index 32087e0e731..abec443e13e 100644 --- a/searchlib/src/vespa/searchlib/tensor/hnsw_index.cpp +++ b/searchlib/src/vespa/searchlib/tensor/hnsw_index.cpp @@ -63,9 +63,9 @@ bool operator< (const PairDist &a, const PairDist &b) { template <HnswIndexType type> vespalib::datastore::ArrayStoreConfig -HnswIndex<type>::make_default_node_store_config() +HnswIndex<type>::make_default_level_array_store_config() { - return NodeStore::optimizedConfigForHugePage(max_level_array_size, + return LevelArrayStore::optimizedConfigForHugePage(max_level_array_size, vespalib::alloc::MemoryAllocator::HUGEPAGE_SIZE, vespalib::alloc::MemoryAllocator::PAGE_SIZE, min_num_arrays_for_new_buffer, @@ -74,9 +74,9 @@ HnswIndex<type>::make_default_node_store_config() template <HnswIndexType type> vespalib::datastore::ArrayStoreConfig -HnswIndex<type>::make_default_link_store_config() +HnswIndex<type>::make_default_link_array_store_config() { - return LinkStore::optimizedConfigForHugePage(max_link_array_size, + return LinkArrayStore::optimizedConfigForHugePage(max_link_array_size, vespalib::alloc::MemoryAllocator::HUGEPAGE_SIZE, vespalib::alloc::MemoryAllocator::PAGE_SIZE, min_num_arrays_for_new_buffer, @@ -272,9 +272,9 @@ HnswIndex<type>::find_nearest_in_layer(const TypedCells& input, const HnswCandid bool keep_searching = true; while (keep_searching) { keep_searching = false; - for (uint32_t neighbor_nodeid : _graph.get_link_array(nearest.node_ref, level)) { - auto& neighbor_node = _graph.acquire_node_refs_elem_ref(neighbor_nodeid); - auto neighbor_ref = neighbor_node.ref().load_acquire(); + for (uint32_t neighbor_nodeid : _graph.get_link_array(nearest.levels_ref, level)) { + auto& neighbor_node = _graph.acquire_node(neighbor_nodeid); + auto neighbor_ref = neighbor_node.levels_ref().load_acquire(); uint32_t neighbor_docid = acquire_docid(neighbor_node, neighbor_nodeid); uint32_t neighbor_subspace = neighbor_node.acquire_subspace(); double dist = calc_distance(input, neighbor_docid, neighbor_subspace); @@ -317,12 +317,12 @@ HnswIndex<type>::search_layer_helper(const TypedCells& input, uint32_t neighbors break; } candidates.pop(); - for (uint32_t neighbor_nodeid : _graph.get_link_array(cand.node_ref, level)) { + for (uint32_t neighbor_nodeid : _graph.get_link_array(cand.levels_ref, level)) { if (neighbor_nodeid >= nodeid_limit) { continue; } - auto& neighbor_node = _graph.acquire_node_refs_elem_ref(neighbor_nodeid); - auto neighbor_ref = neighbor_node.ref().load_acquire(); + auto& neighbor_node = _graph.acquire_node(neighbor_nodeid); + auto neighbor_ref = neighbor_node.levels_ref().load_acquire(); if ((! neighbor_ref.valid()) || ! visited.try_mark(neighbor_nodeid)) { @@ -351,7 +351,7 @@ void HnswIndex<type>::search_layer(const TypedCells& input, uint32_t neighbors_to_find, BestNeighbors& best_neighbors, uint32_t level, const GlobalFilter *filter) const { - uint32_t nodeid_limit = _graph.node_refs_size.load(std::memory_order_acquire); + uint32_t nodeid_limit = _graph.nodes_size.load(std::memory_order_acquire); if (filter) { nodeid_limit = std::min(filter->size(), nodeid_limit); } @@ -386,7 +386,7 @@ HnswIndex<type>::add_document(uint32_t docid) { vespalib::GenerationHandler::Guard no_guard_needed; PreparedAddDoc op(docid, std::move(no_guard_needed)); - auto input_vectors = get_vector_by_docid(docid); + auto input_vectors = get_vectors(docid); auto subspaces = input_vectors.subspaces(); op.nodes.reserve(subspaces); auto nodeids = _id_mapping.allocate_ids(docid, subspaces); @@ -427,8 +427,8 @@ HnswIndex<type>::internal_prepare_add_node(typename HnswIndex::PreparedAddDoc& o int search_level = entry.level; double entry_dist = calc_distance(input_vector, entry.nodeid); uint32_t entry_docid = get_docid(entry.nodeid); - // TODO: check if entry nodeid/node_ref is still valid here - HnswCandidate entry_point(entry.nodeid, entry_docid, entry.node_ref, entry_dist); + // TODO: check if entry nodeid/levels_ref is still valid here + HnswCandidate entry_point(entry.nodeid, entry_docid, entry.levels_ref, entry_dist); while (search_level > node_max_level) { entry_point = find_nearest_in_layer(input_vector, entry_point, search_level); --search_level; @@ -444,9 +444,9 @@ HnswIndex<type>::internal_prepare_add_node(typename HnswIndex::PreparedAddDoc& o auto& links = connections[search_level]; links.reserve(neighbors.used.size()); for (const auto & neighbor : neighbors.used) { - auto neighbor_levels = _graph.get_level_array(neighbor.node_ref); + auto neighbor_levels = _graph.get_level_array(neighbor.levels_ref); if (size_t(search_level) < neighbor_levels.size()) { - links.emplace_back(neighbor.nodeid, neighbor.node_ref); + links.emplace_back(neighbor.nodeid, neighbor.levels_ref); } else { LOG(warning, "in prepare_add(%u), selected neighbor %u is missing level %d (has %zu levels)", op.docid, neighbor.nodeid, search_level, neighbor_levels.size()); @@ -465,10 +465,10 @@ HnswIndex<type>::filter_valid_nodeids(uint32_t level, const typename PreparedAdd valid.reserve(neighbors.size()); for (const auto & neighbor : neighbors) { uint32_t nodeid = neighbor.first; - vespalib::datastore::EntryRef node_ref = neighbor.second; - if (_graph.still_valid(nodeid, node_ref)) { + vespalib::datastore::EntryRef levels_ref = neighbor.second; + if (_graph.still_valid(nodeid, levels_ref)) { assert(nodeid != self_nodeid); - auto levels = _graph.get_level_array(node_ref); + auto levels = _graph.get_level_array(levels_ref); if (level < levels.size()) { valid.push_back(nodeid); } @@ -495,13 +495,13 @@ void HnswIndex<type>::internal_complete_add_node(uint32_t nodeid, uint32_t docid, uint32_t subspace, PreparedAddNode &prepared_node) { int32_t num_levels = prepared_node.connections.size(); - auto node_ref = _graph.make_node(nodeid, docid, subspace, num_levels); + auto levels_ref = _graph.make_node(nodeid, docid, subspace, num_levels); for (int level = 0; level < num_levels; ++level) { auto neighbors = filter_valid_nodeids(level, prepared_node.connections[level], nodeid); connect_new_node(nodeid, neighbors, level); } if (num_levels - 1 > get_entry_level()) { - _graph.set_entry_node({nodeid, node_ref, num_levels - 1}); + _graph.set_entry_node({nodeid, levels_ref, num_levels - 1}); } } @@ -511,7 +511,7 @@ HnswIndex<type>::prepare_add_document(uint32_t docid, VectorBundle vectors, vespalib::GenerationHandler::Guard read_guard) const { - uint32_t max_nodes = _graph.node_refs_size.load(std::memory_order_acquire); + uint32_t max_nodes = _graph.nodes_size.load(std::memory_order_acquire); if (max_nodes < _cfg.min_size_before_two_phase()) { // the first documents added will do all work in write thread // to ensure they are linked together: @@ -530,7 +530,7 @@ HnswIndex<type>::complete_add_document(uint32_t docid, std::unique_ptr<PrepareRe internal_complete_add(docid, *prepared); } else { // we expect this for the first documents added, so no warning for them - if (_graph.node_refs.size() > 1.25 * _cfg.min_size_before_two_phase()) { + if (_graph.nodes.size() > 1.25 * _cfg.min_size_before_two_phase()) { LOG(warning, "complete_add_document(%u) called with invalid prepare_result %s/%u", docid, (prepared ? "valid ptr" : "nullptr"), (prepared ? prepared->docid : 0u)); } @@ -576,8 +576,8 @@ HnswIndex<type>::remove_node(uint32_t nodeid) LinkArrayRef my_links = _graph.get_link_array(nodeid, level); for (uint32_t neighbor_id : my_links) { if (need_new_entrypoint) { - auto entry_node_ref = _graph.get_node_ref(neighbor_id); - _graph.set_entry_node({neighbor_id, entry_node_ref, level}); + auto entry_levels_ref = _graph.get_levels_ref(neighbor_id); + _graph.set_entry_node({neighbor_id, entry_levels_ref, level}); need_new_entrypoint = false; } remove_link_to(neighbor_id, nodeid, level); @@ -608,9 +608,9 @@ HnswIndex<type>::assign_generation(generation_t current_gen) { // Note: RcuVector transfers hold lists as part of reallocation based on current generation. // We need to set the next generation here, as it is incremented on a higher level right after this call. - _graph.node_refs.setGeneration(current_gen + 1); - _graph.nodes.assign_generation(current_gen); - _graph.links.assign_generation(current_gen); + _graph.nodes.setGeneration(current_gen + 1); + _graph.levels_store.assign_generation(current_gen); + _graph.links_store.assign_generation(current_gen); _id_mapping.assign_generation(current_gen); } @@ -618,9 +618,9 @@ template <HnswIndexType type> void HnswIndex<type>::reclaim_memory(generation_t oldest_used_gen) { - _graph.node_refs.reclaim_memory(oldest_used_gen); _graph.nodes.reclaim_memory(oldest_used_gen); - _graph.links.reclaim_memory(oldest_used_gen); + _graph.levels_store.reclaim_memory(oldest_used_gen); + _graph.links_store.reclaim_memory(oldest_used_gen); _id_mapping.reclaim_memory(oldest_used_gen); } @@ -628,15 +628,15 @@ template <HnswIndexType type> void HnswIndex<type>::compact_level_arrays(CompactionSpec compaction_spec, const CompactionStrategy& compaction_strategy) { - auto compacting_buffers = _graph.nodes.start_compact_worst_buffers(compaction_spec, compaction_strategy); - uint32_t nodeid_limit = _graph.node_refs.size(); + auto compacting_buffers = _graph.levels_store.start_compact_worst_buffers(compaction_spec, compaction_strategy); + uint32_t nodeid_limit = _graph.nodes.size(); auto filter = compacting_buffers->make_entry_ref_filter(); - vespalib::ArrayRef<NodeType> refs(&_graph.node_refs[0], nodeid_limit); - for (auto& ref : refs) { - auto node_ref = ref.ref().load_relaxed(); - if (node_ref.valid() && filter.has(node_ref)) { - EntryRef new_node_ref = _graph.nodes.move_on_compact(node_ref); - ref.ref().store_release(new_node_ref); + vespalib::ArrayRef<NodeType> nodes(&_graph.nodes[0], nodeid_limit); + for (auto& node : nodes) { + auto levels_ref = node.levels_ref().load_relaxed(); + if (levels_ref.valid() && filter.has(levels_ref)) { + EntryRef new_levels_ref = _graph.levels_store.move_on_compact(levels_ref); + node.levels_ref().store_release(new_levels_ref); } } compacting_buffers->finish(); @@ -646,12 +646,12 @@ template <HnswIndexType type> void HnswIndex<type>::compact_link_arrays(CompactionSpec compaction_spec, const CompactionStrategy& compaction_strategy) { - auto context = _graph.links.compactWorst(compaction_spec, compaction_strategy); - uint32_t nodeid_limit = _graph.node_refs.size(); + auto context = _graph.links_store.compactWorst(compaction_spec, compaction_strategy); + uint32_t nodeid_limit = _graph.nodes.size(); for (uint32_t nodeid = 1; nodeid < nodeid_limit; ++nodeid) { - EntryRef level_ref = _graph.get_node_ref(nodeid); - if (level_ref.valid()) { - vespalib::ArrayRef<AtomicEntryRef> refs(_graph.nodes.get_writable(level_ref)); + EntryRef levels_ref = _graph.get_levels_ref(nodeid); + if (levels_ref.valid()) { + vespalib::ArrayRef<AtomicEntryRef> refs(_graph.levels_store.get_writable(levels_ref)); context->compact(refs); } } @@ -661,7 +661,7 @@ template <HnswIndexType type> bool HnswIndex<type>::consider_compact_level_arrays(const CompactionStrategy& compaction_strategy) { - if (!_graph.nodes.has_held_buffers() && _compaction_spec.level_arrays().compact()) { + if (!_graph.levels_store.has_held_buffers() && _compaction_spec.level_arrays().compact()) { compact_level_arrays(_compaction_spec.level_arrays(), compaction_strategy); return true; } @@ -672,7 +672,7 @@ template <HnswIndexType type> bool HnswIndex<type>::consider_compact_link_arrays(const CompactionStrategy& compaction_strategy) { - if (!_graph.links.has_held_buffers() && _compaction_spec.link_arrays().compact()) { + if (!_graph.links_store.has_held_buffers() && _compaction_spec.link_arrays().compact()) { compact_link_arrays(_compaction_spec.link_arrays(), compaction_strategy); return true; } @@ -698,12 +698,12 @@ vespalib::MemoryUsage HnswIndex<type>::update_stat(const CompactionStrategy& compaction_strategy) { vespalib::MemoryUsage result; - result.merge(_graph.node_refs.getMemoryUsage()); - auto level_arrays_memory_usage = _graph.nodes.getMemoryUsage(); - auto level_arrays_address_space_usage = _graph.nodes.addressSpaceUsage(); + result.merge(_graph.nodes.getMemoryUsage()); + auto level_arrays_memory_usage = _graph.levels_store.getMemoryUsage(); + auto level_arrays_address_space_usage = _graph.levels_store.addressSpaceUsage(); result.merge(level_arrays_memory_usage); - auto link_arrays_memory_usage = _graph.links.getMemoryUsage(); - auto link_arrays_address_space_usage = _graph.links.addressSpaceUsage(); + auto link_arrays_memory_usage = _graph.links_store.getMemoryUsage(); + auto link_arrays_address_space_usage = _graph.links_store.addressSpaceUsage(); _compaction_spec = HnswIndexCompactionSpec(compaction_strategy.should_compact(level_arrays_memory_usage, level_arrays_address_space_usage), compaction_strategy.should_compact(link_arrays_memory_usage, link_arrays_address_space_usage)); result.merge(link_arrays_memory_usage); @@ -715,9 +715,9 @@ vespalib::MemoryUsage HnswIndex<type>::memory_usage() const { vespalib::MemoryUsage result; - result.merge(_graph.node_refs.getMemoryUsage()); result.merge(_graph.nodes.getMemoryUsage()); - result.merge(_graph.links.getMemoryUsage()); + result.merge(_graph.levels_store.getMemoryUsage()); + result.merge(_graph.links_store.getMemoryUsage()); result.merge(_id_mapping.memory_usage()); return result; } @@ -726,8 +726,8 @@ template <HnswIndexType type> void HnswIndex<type>::populate_address_space_usage(search::AddressSpaceUsage& usage) const { - usage.set(AddressSpaceComponents::hnsw_node_store, _graph.nodes.addressSpaceUsage()); - usage.set(AddressSpaceComponents::hnsw_link_store, _graph.links.addressSpaceUsage()); + usage.set(AddressSpaceComponents::hnsw_levels_store, _graph.levels_store.addressSpaceUsage()); + usage.set(AddressSpaceComponents::hnsw_links_store, _graph.links_store.addressSpaceUsage()); } template <HnswIndexType type> @@ -737,9 +737,9 @@ HnswIndex<type>::get_state(const vespalib::slime::Inserter& inserter) const auto& object = inserter.insertObject(); auto& memUsageObj = object.setObject("memory_usage"); StateExplorerUtils::memory_usage_to_slime(memory_usage(), memUsageObj.setObject("all")); - StateExplorerUtils::memory_usage_to_slime(_graph.node_refs.getMemoryUsage(), memUsageObj.setObject("node_refs")); StateExplorerUtils::memory_usage_to_slime(_graph.nodes.getMemoryUsage(), memUsageObj.setObject("nodes")); - StateExplorerUtils::memory_usage_to_slime(_graph.links.getMemoryUsage(), memUsageObj.setObject("links")); + StateExplorerUtils::memory_usage_to_slime(_graph.levels_store.getMemoryUsage(), memUsageObj.setObject("levels")); + StateExplorerUtils::memory_usage_to_slime(_graph.links_store.getMemoryUsage(), memUsageObj.setObject("links")); object.setLong("nodes", _graph.size()); auto& histogram_array = object.setArray("level_histogram"); auto& links_hst_array = object.setArray("level_0_links_histogram"); @@ -775,12 +775,12 @@ void HnswIndex<type>::shrink_lid_space(uint32_t doc_id_limit) { assert(doc_id_limit >= 1u); - assert(doc_id_limit >= _graph.node_refs_size.load(std::memory_order_relaxed)); - uint32_t old_doc_id_limit = _graph.node_refs.size(); + assert(doc_id_limit >= _graph.nodes_size.load(std::memory_order_relaxed)); + uint32_t old_doc_id_limit = _graph.nodes.size(); if (doc_id_limit >= old_doc_id_limit) { return; } - _graph.node_refs.shrink(doc_id_limit); + _graph.nodes.shrink(doc_id_limit); } template <HnswIndexType type> @@ -850,8 +850,8 @@ HnswIndex<type>::top_k_candidates(const TypedCells &vector, uint32_t k, const Gl int search_level = entry.level; double entry_dist = calc_distance(vector, entry.nodeid); uint32_t entry_docid = get_docid(entry.nodeid); - // TODO: check if entry docid/node_ref is still valid here - HnswCandidate entry_point(entry.nodeid, entry_docid, entry.node_ref, entry_dist); + // TODO: check if entry docid/levels_ref is still valid here + HnswCandidate entry_point(entry.nodeid, entry_docid, entry.levels_ref, entry_dist); while (search_level > 0) { entry_point = find_nearest_in_layer(vector, entry_point, search_level); --search_level; @@ -865,14 +865,14 @@ template <HnswIndexType type> HnswTestNode HnswIndex<type>::get_node(uint32_t nodeid) const { - auto node_ref = _graph.acquire_node_ref(nodeid); - if (!node_ref.valid()) { + auto levels_ref = _graph.acquire_levels_ref(nodeid); + if (!levels_ref.valid()) { return HnswTestNode(); } - auto levels = _graph.nodes.get(node_ref); + auto levels = _graph.levels_store.get(levels_ref); HnswTestNode::LevelArray result; for (const auto& links_ref : levels) { - auto links = _graph.links.get(links_ref.load_acquire()); + auto links = _graph.links_store.get(links_ref.load_acquire()); HnswTestNode::LinkArray result_links(links.begin(), links.end()); std::sort(result_links.begin(), result_links.end()); result.push_back(result_links); @@ -886,13 +886,13 @@ HnswIndex<type>::set_node(uint32_t nodeid, const HnswTestNode &node) { size_t num_levels = node.size(); assert(num_levels > 0); - auto node_ref = _graph.make_node(nodeid, nodeid, 0, num_levels); + auto levels_ref = _graph.make_node(nodeid, nodeid, 0, num_levels); for (size_t level = 0; level < num_levels; ++level) { connect_new_node(nodeid, node.level(level), level); } int max_level = num_levels - 1; if (get_entry_level() < max_level) { - _graph.set_entry_node({nodeid, node_ref, max_level}); + _graph.set_entry_node({nodeid, levels_ref, max_level}); } } @@ -903,12 +903,12 @@ HnswIndex<type>::check_link_symmetry() const bool all_sym = true; size_t nodeid_limit = _graph.size(); for (size_t nodeid = 0; nodeid < nodeid_limit; ++nodeid) { - auto node_ref = _graph.acquire_node_ref(nodeid); - if (node_ref.valid()) { - auto levels = _graph.nodes.get(node_ref); + auto levels_ref = _graph.acquire_levels_ref(nodeid); + if (levels_ref.valid()) { + auto levels = _graph.levels_store.get(levels_ref); uint32_t level = 0; for (const auto& links_ref : levels) { - auto links = _graph.links.get(links_ref.load_acquire()); + auto links = _graph.links_store.get(links_ref.load_acquire()); for (auto neighbor_nodeid : links) { auto neighbor_links = _graph.acquire_link_array(neighbor_nodeid, level); if (! has_link_to(neighbor_links, nodeid)) { diff --git a/searchlib/src/vespa/searchlib/tensor/hnsw_index.h b/searchlib/src/vespa/searchlib/tensor/hnsw_index.h index 32058e9ac44..38b2c69faf2 100644 --- a/searchlib/src/vespa/searchlib/tensor/hnsw_index.h +++ b/searchlib/src/vespa/searchlib/tensor/hnsw_index.h @@ -65,7 +65,7 @@ public: if constexpr (NodeType::identity_mapping) { return nodeid; } else { - return _graph.node_refs.acquire_elem_ref(nodeid).acquire_docid(); + return _graph.nodes.acquire_elem_ref(nodeid).acquire_docid(); } } @@ -76,9 +76,9 @@ protected: using GraphType = HnswGraph<type>; using NodeType = typename GraphType::NodeType; using AtomicEntryRef = vespalib::datastore::AtomicEntryRef; - using NodeStore = typename GraphType::NodeStore; + using LevelArrayStore = typename GraphType::LevelArrayStore; - using LinkStore = typename GraphType::LinkStore; + using LinkArrayStore = typename GraphType::LinkArrayStore; using LinkArrayRef = typename GraphType::LinkArrayRef; using LinkArray = std::vector<uint32_t, vespalib::allocator_large<uint32_t>>; @@ -137,7 +137,7 @@ protected: if constexpr (NodeType::identity_mapping) { return _vectors.get_vector(nodeid, 0); } else { - auto& ref = _graph.node_refs.acquire_elem_ref(nodeid); + auto& ref = _graph.nodes.acquire_elem_ref(nodeid); uint32_t docid = ref.acquire_docid(); uint32_t subspace = ref.acquire_subspace(); return _vectors.get_vector(docid, subspace); @@ -146,7 +146,7 @@ protected: inline TypedCells get_vector(uint32_t docid, uint32_t subspace) const { return _vectors.get_vector(docid, subspace); } - inline VectorBundle get_vector_by_docid(uint32_t docid) const { + inline VectorBundle get_vectors(uint32_t docid) const { return _vectors.get_vectors(docid); } @@ -259,8 +259,8 @@ public: GraphType& get_graph() { return _graph; } IdMapping& get_id_mapping() { return _id_mapping; } - static vespalib::datastore::ArrayStoreConfig make_default_node_store_config(); - static vespalib::datastore::ArrayStoreConfig make_default_link_store_config(); + static vespalib::datastore::ArrayStoreConfig make_default_level_array_store_config(); + static vespalib::datastore::ArrayStoreConfig make_default_link_array_store_config(); }; } diff --git a/searchlib/src/vespa/searchlib/tensor/hnsw_index_loader.hpp b/searchlib/src/vespa/searchlib/tensor/hnsw_index_loader.hpp index de9cc760fec..279adbac559 100644 --- a/searchlib/src/vespa/searchlib/tensor/hnsw_index_loader.hpp +++ b/searchlib/src/vespa/searchlib/tensor/hnsw_index_loader.hpp @@ -61,12 +61,12 @@ HnswIndexLoader<ReaderType, type>::load_next() if (++_nodeid < _num_nodes) { return true; } else { - _graph.node_refs.ensure_size(std::max(_num_nodes, 1u)); - _graph.node_refs_size.store(std::max(_num_nodes, 1u), std::memory_order_release); - _graph.trim_node_refs_size(); - auto entry_node_ref = _graph.get_node_ref(_entry_nodeid); - _graph.set_entry_node({_entry_nodeid, entry_node_ref, _entry_level}); - _id_mapping.on_load(_graph.node_refs.make_read_view(_graph.node_refs.size())); + _graph.nodes.ensure_size(std::max(_num_nodes, 1u)); + _graph.nodes_size.store(std::max(_num_nodes, 1u), std::memory_order_release); + _graph.trim_nodes_size(); + auto entry_levels_ref = _graph.get_levels_ref(_entry_nodeid); + _graph.set_entry_node({_entry_nodeid, entry_levels_ref, _entry_level}); + _id_mapping.on_load(_graph.nodes.make_read_view(_graph.nodes.size())); _complete = true; return false; } diff --git a/searchlib/src/vespa/searchlib/tensor/hnsw_index_saver.cpp b/searchlib/src/vespa/searchlib/tensor/hnsw_index_saver.cpp index 370e2ddf92a..70daeb34f5c 100644 --- a/searchlib/src/vespa/searchlib/tensor/hnsw_index_saver.cpp +++ b/searchlib/src/vespa/searchlib/tensor/hnsw_index_saver.cpp @@ -14,11 +14,11 @@ template <HnswIndexType type> size_t count_valid_link_arrays(const HnswGraph<type> & graph) { size_t count(0); - size_t num_nodes = graph.node_refs.get_size(); // Called from writer only + size_t num_nodes = graph.nodes.get_size(); // Called from writer only for (size_t i = 0; i < num_nodes; ++i) { - auto node_ref = graph.get_node_ref(i); - if (node_ref.valid()) { - count += graph.nodes.get(node_ref).size(); + auto levels_ref = graph.get_levels_ref(i); + if (levels_ref.valid()) { + count += graph.levels_store.get(levels_ref).size(); } } return count; @@ -42,23 +42,23 @@ HnswIndexSaver<type>::~HnswIndexSaver() = default; template <HnswIndexType type> HnswIndexSaver<type>::HnswIndexSaver(const HnswGraph<type> &graph) - : _graph_links(graph.links), _meta_data() + : _graph_links(graph.links_store), _meta_data() { auto entry = graph.get_entry_node(); _meta_data.entry_nodeid = entry.nodeid; _meta_data.entry_level = entry.level; - size_t num_nodes = graph.node_refs.get_size(); // Called from writer only + size_t num_nodes = graph.nodes.get_size(); // Called from writer only assert (num_nodes <= (std::numeric_limits<uint32_t>::max() - 1)); size_t link_array_count = count_valid_link_arrays(graph); assert (link_array_count <= std::numeric_limits<uint32_t>::max()); _meta_data.refs.reserve(link_array_count); _meta_data.nodes.reserve(num_nodes+1); for (size_t i = 0; i < num_nodes; ++i) { - auto& node = graph.node_refs.get_elem_ref(i); + auto& node = graph.nodes.get_elem_ref(i); _meta_data.nodes.emplace_back(_meta_data.refs.size(), node); - auto node_ref = node.ref().load_relaxed(); - if (node_ref.valid()) { - auto levels = graph.nodes.get(node_ref); + auto levels_ref = node.levels_ref().load_relaxed(); + if (levels_ref.valid()) { + auto levels = graph.levels_store.get(levels_ref); for (const auto& links_ref : levels) { _meta_data.refs.push_back(links_ref.load_relaxed()); } diff --git a/searchlib/src/vespa/searchlib/tensor/hnsw_index_saver.h b/searchlib/src/vespa/searchlib/tensor/hnsw_index_saver.h index 2884ee9b494..76a97274cc6 100644 --- a/searchlib/src/vespa/searchlib/tensor/hnsw_index_saver.h +++ b/searchlib/src/vespa/searchlib/tensor/hnsw_index_saver.h @@ -34,7 +34,7 @@ private: MetaData(); ~MetaData(); }; - const typename HnswGraph<type>::LinkStore &_graph_links; + const typename HnswGraph<type>::LinkArrayStore &_graph_links; MetaData _meta_data; }; diff --git a/searchlib/src/vespa/searchlib/tensor/hnsw_index_utils.h b/searchlib/src/vespa/searchlib/tensor/hnsw_index_utils.h index a88b805f198..95aace19e69 100644 --- a/searchlib/src/vespa/searchlib/tensor/hnsw_index_utils.h +++ b/searchlib/src/vespa/searchlib/tensor/hnsw_index_utils.h @@ -15,14 +15,14 @@ namespace search::tensor { */ struct HnswTraversalCandidate { uint32_t nodeid; - vespalib::datastore::EntryRef node_ref; + vespalib::datastore::EntryRef levels_ref; double distance; HnswTraversalCandidate(uint32_t nodeid_in, double distance_in) noexcept - : nodeid(nodeid_in), node_ref(), distance(distance_in) {} - HnswTraversalCandidate(uint32_t nodeid_in, vespalib::datastore::EntryRef node_ref_in, double distance_in) noexcept - : nodeid(nodeid_in), node_ref(node_ref_in), distance(distance_in) {} - HnswTraversalCandidate(uint32_t nodeid_in, uint32_t docid_in, vespalib::datastore::EntryRef node_ref_in, double distance_in) noexcept - : nodeid(nodeid_in), node_ref(node_ref_in), distance(distance_in) + : nodeid(nodeid_in), levels_ref(), distance(distance_in) {} + HnswTraversalCandidate(uint32_t nodeid_in, vespalib::datastore::EntryRef levels_ref_in, double distance_in) noexcept + : nodeid(nodeid_in), levels_ref(levels_ref_in), distance(distance_in) {} + HnswTraversalCandidate(uint32_t nodeid_in, uint32_t docid_in, vespalib::datastore::EntryRef levels_ref_in, double distance_in) noexcept + : nodeid(nodeid_in), levels_ref(levels_ref_in), distance(distance_in) { (void) docid_in; } @@ -35,8 +35,8 @@ struct HnswTraversalCandidate { struct HnswCandidate : public HnswTraversalCandidate { uint32_t docid; - HnswCandidate(uint32_t nodeid_in, uint32_t docid_in, vespalib::datastore::EntryRef node_ref_in, double distance_in) noexcept - : HnswTraversalCandidate(nodeid_in, docid_in, node_ref_in, distance_in), + HnswCandidate(uint32_t nodeid_in, uint32_t docid_in, vespalib::datastore::EntryRef levels_ref_in, double distance_in) noexcept + : HnswTraversalCandidate(nodeid_in, docid_in, levels_ref_in, distance_in), docid(docid_in) { } diff --git a/searchlib/src/vespa/searchlib/tensor/hnsw_node.h b/searchlib/src/vespa/searchlib/tensor/hnsw_node.h index 2e14f363bba..e5e2910fe81 100644 --- a/searchlib/src/vespa/searchlib/tensor/hnsw_node.h +++ b/searchlib/src/vespa/searchlib/tensor/hnsw_node.h @@ -14,19 +14,19 @@ class HnswNode { using AtomicEntryRef = vespalib::datastore::AtomicEntryRef; using EntryRef = vespalib::datastore::EntryRef; - AtomicEntryRef _ref; + AtomicEntryRef _levels_ref; vespalib::datastore::AtomicValueWrapper<uint32_t> _docid; vespalib::datastore::AtomicValueWrapper<uint32_t> _subspace; public: HnswNode() noexcept - : _ref(), + : _levels_ref(), _docid(), _subspace() { } - AtomicEntryRef& ref() noexcept { return _ref; } - const AtomicEntryRef& ref() const noexcept { return _ref; } + AtomicEntryRef& levels_ref() noexcept { return _levels_ref; } + const AtomicEntryRef& levels_ref() const noexcept { return _levels_ref; } void store_docid(uint32_t docid) noexcept { _docid.store_release(docid); } void store_subspace(uint32_t subspace) noexcept { _subspace.store_release(subspace); } // Mapping from nodeid to docid and subspace. diff --git a/searchlib/src/vespa/searchlib/tensor/hnsw_nodeid_mapping.cpp b/searchlib/src/vespa/searchlib/tensor/hnsw_nodeid_mapping.cpp index 9983bf6d97b..787abf4ad14 100644 --- a/searchlib/src/vespa/searchlib/tensor/hnsw_nodeid_mapping.cpp +++ b/searchlib/src/vespa/searchlib/tensor/hnsw_nodeid_mapping.cpp @@ -122,7 +122,7 @@ get_docid_limit(vespalib::ConstArrayRef<HnswNode> nodes) { uint32_t max_docid = 0; for (auto& node : nodes) { - if (node.ref().load_relaxed().valid()) { + if (node.levels_ref().load_relaxed().valid()) { max_docid = std::max(node.acquire_docid(), max_docid); } } @@ -135,7 +135,7 @@ make_subspaces_histogram(vespalib::ConstArrayRef<HnswNode> nodes, uint32_t docid // Make histogram std::vector<uint32_t> histogram(docid_limit); for (auto& node : nodes) { - if (node.ref().load_relaxed().valid()) { + if (node.levels_ref().load_relaxed().valid()) { auto docid = node.acquire_docid(); auto subspace = node.acquire_subspace(); auto &num_subspaces = histogram[docid]; @@ -171,7 +171,7 @@ HnswNodeidMapping::populate_docid_to_nodeids_mapping_and_free_list(vespalib::Con { uint32_t nodeid = 0; for (auto& node : nodes) { - if (node.ref().load_relaxed().valid()) { + if (node.levels_ref().load_relaxed().valid()) { auto docid = node.acquire_docid(); auto subspace = node.acquire_subspace(); auto nodeids = _nodeids.get_writable(_refs[docid]); @@ -207,7 +207,7 @@ HnswNodeidMapping::on_load(vespalib::ConstArrayRef<HnswNode> nodes) return; } // Check that reserved nodeid is not used - assert(!nodes[0].ref().load_relaxed().valid()); + assert(!nodes[0].levels_ref().load_relaxed().valid()); auto docid_limit = get_docid_limit(nodes); auto histogram = make_subspaces_histogram(nodes, docid_limit); // Allocate mapping from docid to nodeids allocate_docid_to_nodeids_mapping(std::move(histogram)); diff --git a/searchlib/src/vespa/searchlib/tensor/hnsw_simple_node.h b/searchlib/src/vespa/searchlib/tensor/hnsw_simple_node.h index 9740cab5c31..4d775f61a8d 100644 --- a/searchlib/src/vespa/searchlib/tensor/hnsw_simple_node.h +++ b/searchlib/src/vespa/searchlib/tensor/hnsw_simple_node.h @@ -13,15 +13,15 @@ class HnswSimpleNode { using AtomicEntryRef = vespalib::datastore::AtomicEntryRef; using EntryRef = vespalib::datastore::EntryRef; - AtomicEntryRef _ref; + AtomicEntryRef _levels_ref; public: HnswSimpleNode() - : _ref() + : _levels_ref() { } - AtomicEntryRef& ref() noexcept { return _ref; } - const AtomicEntryRef& ref() const noexcept { return _ref; } + AtomicEntryRef& levels_ref() noexcept { return _levels_ref; } + const AtomicEntryRef& levels_ref() const noexcept { return _levels_ref; } void store_docid(uint32_t docid) noexcept { (void) docid; } void store_subspace(uint32_t subspace) noexcept { (void) subspace; } // Mapping from nodeid to docid and subspace. diff --git a/slobrok/src/vespa/slobrok/server/exchange_manager.cpp b/slobrok/src/vespa/slobrok/server/exchange_manager.cpp index b5460460a4a..4b8c1c02252 100644 --- a/slobrok/src/vespa/slobrok/server/exchange_manager.cpp +++ b/slobrok/src/vespa/slobrok/server/exchange_manager.cpp @@ -118,8 +118,8 @@ ExchangeManager::healthCheck() if (remoteList.size() != 0) { vespalib::string diff = diffLists(newWorldList, remoteList); if (! diff.empty()) { - LOG(warning, "Diff from consensus map to peer slobrok mirror: %s", - diff.c_str()); + LOG(warning, "Peer slobrok at %s may have problems, differences from consensus map: %s", + partner->getName().c_str(), diff.c_str()); someBad = true; } } diff --git a/slobrok/src/vespa/slobrok/server/remote_check.cpp b/slobrok/src/vespa/slobrok/server/remote_check.cpp index 40a43a8946b..dccf0934a91 100644 --- a/slobrok/src/vespa/slobrok/server/remote_check.cpp +++ b/slobrok/src/vespa/slobrok/server/remote_check.cpp @@ -15,7 +15,7 @@ RemoteCheck::RemoteCheck(FNET_Scheduler *sched, ExchangeManager& exch) : FNET_Task(sched), _exchanger(exch) { - double seconds = randomIn(15.3, 27.9); + double seconds = randomIn(5.3, 9.7); Schedule(seconds); } @@ -31,7 +31,7 @@ RemoteCheck::PerformTask() { LOG(debug, "asking exchanger to health check"); _exchanger.healthCheck(); - double seconds = randomIn(151.3, 179.7); + double seconds = randomIn(15.3, 17.7); Schedule(seconds); } diff --git a/vespa-dependencies-enforcer/allowed-maven-dependencies.txt b/vespa-dependencies-enforcer/allowed-maven-dependencies.txt index 9cd6ed787b3..1e38b6029ba 100644 --- a/vespa-dependencies-enforcer/allowed-maven-dependencies.txt +++ b/vespa-dependencies-enforcer/allowed-maven-dependencies.txt @@ -240,4 +240,3 @@ org.junit.vintage:junit-vintage-engine:5.8.1 org.mockito:mockito-core:4.0.0 org.mockito:mockito-junit-jupiter:4.0.0 org.objenesis:objenesis:3.2 -xmlunit:xmlunit:1.5 diff --git a/vespalib/CMakeLists.txt b/vespalib/CMakeLists.txt index a3d5054973f..8509d5fc382 100644 --- a/vespalib/CMakeLists.txt +++ b/vespalib/CMakeLists.txt @@ -21,6 +21,7 @@ vespa_define_module( src/apps/vespa-drop-file-from-cache src/apps/vespa-probe-io-uring src/apps/vespa-resource-limits + src/apps/vespa-stress-and-validate-memory src/apps/vespa-tsan-digest src/apps/vespa-validate-hostname @@ -193,6 +194,7 @@ vespa_define_module( src/tests/util/generationhandler_stress src/tests/util/hamming src/tests/util/md5 + src/tests/util/memory_trap src/tests/util/mmap_file_allocator src/tests/util/mmap_file_allocator_factory src/tests/util/rcuvector diff --git a/vespalib/src/apps/vespa-stress-and-validate-memory/.gitignore b/vespalib/src/apps/vespa-stress-and-validate-memory/.gitignore new file mode 100644 index 00000000000..77cf05d77d5 --- /dev/null +++ b/vespalib/src/apps/vespa-stress-and-validate-memory/.gitignore @@ -0,0 +1 @@ +vespa-stress-and-validate-memory diff --git a/vespalib/src/apps/vespa-stress-and-validate-memory/CMakeLists.txt b/vespalib/src/apps/vespa-stress-and-validate-memory/CMakeLists.txt new file mode 100644 index 00000000000..17ea9d709df --- /dev/null +++ b/vespalib/src/apps/vespa-stress-and-validate-memory/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(vespalib_stress-and-validate-memory_app + SOURCES + stress_and_validate_memory.cpp + OUTPUT_NAME vespa-stress-and-validate-memory + INSTALL bin + DEPENDS + vespalib +) diff --git a/vespalib/src/apps/vespa-stress-and-validate-memory/stress_and_validate_memory.cpp b/vespalib/src/apps/vespa-stress-and-validate-memory/stress_and_validate_memory.cpp new file mode 100644 index 00000000000..a7a227e45b3 --- /dev/null +++ b/vespalib/src/apps/vespa-stress-and-validate-memory/stress_and_validate_memory.cpp @@ -0,0 +1,262 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/vespalib/util/mmap_file_allocator.h> +#include <vespa/vespalib/util/size_literals.h> +#include <vespa/vespalib/util/time.h> +#include <thread> +#include <vector> +#include <atomic> +#include <cstring> +#include <mutex> +#include <filesystem> +#include <iostream> + +std::atomic<bool> stopped = false; +std::mutex log_mutex; +using namespace vespalib; + +const char * description = + "Runs stress test of memory by slowly growing a heap filled with 0.\n" + "Each core on the node will then continously read back and verify random memory sections still being zero.\n" + "-h heap_in_GB(1) and -t run_time_in_seconds(10) are the options available.\n" + "Memory will grow slowly during the first half of the test and then stay put.\n" + "There is also the option to include stress testing of swap files by using -s <directory>.\n" + "The swap will grow to twice the heap size in the same manner.\n" + "Swap memory is stressed by constant random writing from all cores.\n"; + +class Config { +public: + Config(size_t heap_size, size_t nprocs, size_t allocs_per_thread, duration alloc_time) + : _heap_size(heap_size), + _nprocs(nprocs), + _allocs_per_thread(allocs_per_thread), + _alloc_time(alloc_time) + {} + size_t allocs_per_thread() const { return _allocs_per_thread; } + duration alloc_time() const { return _alloc_time; } + size_t alloc_size() const { return _heap_size / _nprocs / _allocs_per_thread; } + size_t nprocs() const { return _nprocs; } + size_t heap_size() const { return _heap_size; } +private: + const size_t _heap_size; + const size_t _nprocs; + const size_t _allocs_per_thread; + const duration _alloc_time; +}; + +class Allocations { +public: + Allocations(const Config & config); + ~Allocations(); + size_t make_and_load_alloc_per_thread(); + size_t verify_random_allocation(unsigned int *seed) const; + const Config & cfg() const { return _cfg; } + size_t verify_and_report_errors() const { + std::lock_guard guard(_mutex); + for (const auto & alloc : _allocations) { + _total_errors += verify_allocation(alloc.get()); + } + return _total_errors; + } +private: + size_t verify_allocation(const char *) const; + const Config & _cfg; + mutable std::mutex _mutex; + mutable size_t _total_errors; + std::vector<std::unique_ptr<char[]>> _allocations; +}; + +Allocations::Allocations(const Config & config) + : _cfg(config), + _mutex(), + _total_errors(0), + _allocations() +{ + _allocations.reserve(config.nprocs() * config.allocs_per_thread()); + std::cout << "Starting memory stress with " << config.nprocs() << " threads and heap size " << (config.heap_size()/1_Mi) << " mb. Allocation size = " << config.alloc_size() << std::endl; +} + +Allocations::~Allocations() = default; + +size_t +Allocations::make_and_load_alloc_per_thread() { + auto alloc = std::make_unique<char[]>(cfg().alloc_size()); + memset(alloc.get(), 0, cfg().alloc_size()); + std::lock_guard guard(_mutex); + _allocations.push_back(std::move(alloc)); + return 1; +} + +size_t +Allocations::verify_random_allocation(unsigned int *seed) const { + const char * alloc; + { + std::lock_guard guard(_mutex); + alloc = _allocations[rand_r(seed) % _allocations.size()].get(); + } + size_t error_count = verify_allocation(alloc); + std::lock_guard guard(_mutex); + _total_errors += error_count; + return error_count; +} + +size_t +Allocations::verify_allocation(const char * alloc) const { + size_t error_count = 0; + for (size_t i = 0; i < cfg().alloc_size(); i++) { + if (alloc[i] != 0) { + error_count++; + std::lock_guard guard(log_mutex); + std::cout << "Thread " << std::this_thread::get_id() << ": Unexpected byte(" << std::hex << int(alloc[i]) << ") at " << static_cast<const void *>(alloc + i) << std::endl; + } + } + return error_count; +} + +class FileBackedMemory { +public: + FileBackedMemory(const Config & config, std::string dir); + ~FileBackedMemory(); + const Config & cfg() const { return _cfg; } + size_t make_and_load_alloc_per_thread(); + void random_write(unsigned int *seed); +private: + using PtrAndSize = std::pair<void *, size_t>; + const Config & _cfg; + mutable std::mutex _mutex; + alloc::MmapFileAllocator _allocator; + std::vector<PtrAndSize> _allocations; +}; + +FileBackedMemory::FileBackedMemory(const Config & config, std::string dir) + : _cfg(config), + _mutex(), + _allocator(dir), + _allocations() +{ + _allocations.reserve(config.nprocs() * config.allocs_per_thread()); + std::cout << "Starting mmapped stress in '" << dir << "' with " << config.nprocs() << " threads and heap size " << (config.heap_size()/1_Mi) << " mb. Allocation size = " << config.alloc_size() << std::endl; +} + +FileBackedMemory::~FileBackedMemory() { + std::lock_guard guard(_mutex); + for (auto ptrAndSize : _allocations) { + _allocator.free(ptrAndSize); + } +} + + +size_t +FileBackedMemory::make_and_load_alloc_per_thread() { + PtrAndSize alloc; + { + std::lock_guard guard(_mutex); + alloc = _allocator.alloc(cfg().alloc_size()); + } + memset(alloc.first, 0, cfg().alloc_size()); + std::lock_guard guard(_mutex); + _allocations.push_back(std::move(alloc)); + return 1; +} + +void +FileBackedMemory::random_write(unsigned int *seed) { + PtrAndSize ptrAndSize; + { + std::lock_guard guard(_mutex); + ptrAndSize = _allocations[rand_r(seed) % _allocations.size()]; + } + memset(ptrAndSize.first, rand_r(seed)%256, ptrAndSize.second); +} + +void +stress_and_validate_heap(Allocations *allocs) { + size_t num_verifications = 0; + size_t num_errors = 0; + size_t num_allocs = allocs->make_and_load_alloc_per_thread(); + const size_t max_allocs = allocs->cfg().allocs_per_thread(); + const double alloc_time = to_s(allocs->cfg().alloc_time()); + steady_time start = steady_clock::now(); + unsigned int seed = start.time_since_epoch().count()%4294967291ul; + for (;!stopped; num_verifications++) { + num_errors += allocs->verify_random_allocation(&seed); + double ratio = to_s(steady_clock::now() - start) / alloc_time; + if (num_allocs < std::min(size_t(ratio*max_allocs), max_allocs)) { + num_allocs += allocs->make_and_load_alloc_per_thread(); + } + } + std::lock_guard guard(log_mutex); + std::cout << "Thread " << std::this_thread::get_id() << ": Completed " << num_verifications << " verifications with " << num_errors << std::endl; +} + +void +stress_file_backed_memory(FileBackedMemory * mmapped) { + size_t num_writes = 0; + size_t num_allocs = mmapped->make_and_load_alloc_per_thread(); + const size_t max_allocs = mmapped->cfg().allocs_per_thread(); + const double alloc_time = to_s(mmapped->cfg().alloc_time()); + steady_time start = steady_clock::now(); + unsigned int seed = start.time_since_epoch().count()%4294967291ul; + for (;!stopped; num_writes++) { + mmapped->random_write(&seed); + double ratio = to_s(steady_clock::now() - start) / alloc_time; + if (num_allocs < std::min(size_t(ratio*max_allocs), max_allocs)) { + num_allocs += mmapped->make_and_load_alloc_per_thread(); + } + } + std::lock_guard guard(log_mutex); + std::cout << "Thread " << std::this_thread::get_id() << ": Completed " << num_writes << " writes" << std::endl; +} + +int +main(int argc, char *argv[]) { + size_t heapSize = 1_Gi; + duration runTime = 10s; + std::string swap_dir; + std::cout << description << std::endl; + for (int i = 1; i+2 <= argc; i+=2) { + char option = argv[i][strlen(argv[i]) - 1]; + char *arg = argv[i+1]; + switch (option) { + case 'h': heapSize = atof(arg) * 1_Gi; break; + case 's': swap_dir = arg; break; + case 't': runTime = from_s(atof(arg)); break; + default: + std::cerr << "Option " << option << " not in allowed set [h,s,t]" << std::endl; + break; + } + } + size_t nprocs = std::thread::hardware_concurrency(); + size_t allocations_per_thread = 1024; + + Config cfgHeap(heapSize, nprocs, allocations_per_thread, runTime/2); + Config cfgFile(heapSize*2, nprocs, allocations_per_thread, runTime/2); + Allocations allocations(cfgHeap); + std::unique_ptr<FileBackedMemory> filebackedMemory; + + std::vector<std::thread> heapValidators; + heapValidators.reserve(nprocs*2); + for (unsigned int i = 0; i < nprocs; i++) { + heapValidators.emplace_back(stress_and_validate_heap, &allocations); + } + if ( ! swap_dir.empty()) { + std::filesystem::create_directories(swap_dir); + filebackedMemory = std::make_unique<FileBackedMemory>(cfgFile, swap_dir); + for (unsigned int i = 0; i < nprocs; i++) { + heapValidators.emplace_back(stress_file_backed_memory, filebackedMemory.get()); + } + } + std::cout << "Running memory stresstest for " << to_s(runTime) << " seconds" << std::endl; + steady_time eot = steady_clock::now() + runTime; + while (steady_clock::now() < eot) { + std::this_thread::sleep_for(1s); + } + stopped = true; + for (auto & th : heapValidators) { + th.join(); + } + heapValidators.clear(); + size_t num_errors = allocations.verify_and_report_errors(); + std::cout << "Completed stresstest with " << num_errors << " errors" << std::endl; + return num_errors == 0 ? 0 : 1; +} diff --git a/vespalib/src/tests/data/smart_buffer/smart_buffer_test.cpp b/vespalib/src/tests/data/smart_buffer/smart_buffer_test.cpp index c461da1fcf7..b860aa3326a 100644 --- a/vespalib/src/tests/data/smart_buffer/smart_buffer_test.cpp +++ b/vespalib/src/tests/data/smart_buffer/smart_buffer_test.cpp @@ -24,6 +24,7 @@ TEST("require that basic read/write works") { SmartBuffer buf(3); TEST_DO(checkBuffer("", buf)); { // read from empty buffer + EXPECT_TRUE(buf.empty()); EXPECT_EQUAL(0u, buf.obtain().size); } { // write to buffer @@ -34,6 +35,7 @@ TEST("require that basic read/write works") { mem.data[1] = 'b'; mem.data[2] = 'c'; EXPECT_EQUAL(&buf, &buf.commit(3)); + EXPECT_FALSE(buf.empty()); mem = buf.reserve(0); TEST_DO(checkBuffer("abc", buf)); EXPECT_LESS_EQUAL(0u, mem.size); @@ -61,6 +63,7 @@ TEST("require that basic read/write works") { EXPECT_LESS_EQUAL(5u, mem.size); } { // read until end + EXPECT_FALSE(buf.empty()); Memory mem = buf.obtain(); TEST_DO(checkBuffer("cd", buf)); TEST_DO(checkMemory("cd", mem)); @@ -69,6 +72,7 @@ TEST("require that basic read/write works") { TEST_DO(checkBuffer("d", buf)); TEST_DO(checkMemory("d", mem)); EXPECT_EQUAL(&buf, &buf.evict(1)); + EXPECT_TRUE(buf.empty()); mem = buf.obtain(); TEST_DO(checkBuffer("", buf)); TEST_DO(checkMemory("", mem)); @@ -83,16 +87,21 @@ TEST("require that requested initial size is not adjusted") { TEST("require that buffer auto-resets when empty") { SmartBuffer buf(64); EXPECT_EQUAL(buf.reserve(10).size, 64u); + EXPECT_TRUE(buf.empty()); write_buf("abc", buf); + EXPECT_FALSE(buf.empty()); EXPECT_EQUAL(buf.reserve(10).size, 61u); buf.evict(3); + EXPECT_TRUE(buf.empty()); EXPECT_EQUAL(buf.reserve(10).size, 64u); } TEST("require that buffer can grow") { SmartBuffer buf(64); EXPECT_EQUAL(buf.capacity(), 64u); + EXPECT_TRUE(buf.empty()); write_buf("abc", buf); + EXPECT_FALSE(buf.empty()); write_buf("abc", buf); buf.evict(3); EXPECT_EQUAL(buf.reserve(70).size, size_t(128 - 3)); @@ -103,7 +112,9 @@ TEST("require that buffer can grow") { TEST("require that buffer can grow more than 2x") { SmartBuffer buf(64); EXPECT_EQUAL(buf.capacity(), 64u); + EXPECT_TRUE(buf.empty()); write_buf("abc", buf); + EXPECT_FALSE(buf.empty()); write_buf("abc", buf); buf.evict(3); EXPECT_EQUAL(buf.reserve(170).size, 170u); @@ -114,7 +125,9 @@ TEST("require that buffer can grow more than 2x") { TEST("require that buffer can be compacted") { SmartBuffer buf(16); EXPECT_EQUAL(buf.capacity(), 16u); + EXPECT_TRUE(buf.empty()); write_buf("abc", buf); + EXPECT_FALSE(buf.empty()); write_buf("abc", buf); buf.evict(3); write_buf("abc", buf); @@ -133,6 +146,7 @@ TEST("require that buffer can be compacted") { TEST("require that a completely empty buffer can be created") { SmartBuffer buf(0); EXPECT_EQUAL(buf.capacity(), 0u); + EXPECT_TRUE(buf.empty()); EXPECT_TRUE(buf.obtain().data == nullptr); } diff --git a/vespalib/src/tests/util/memory_trap/CMakeLists.txt b/vespalib/src/tests/util/memory_trap/CMakeLists.txt new file mode 100644 index 00000000000..c3241b0ad93 --- /dev/null +++ b/vespalib/src/tests/util/memory_trap/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(vespalib_util_memory_trap_test_app TEST + SOURCES + memory_trap_test.cpp + DEPENDS + vespalib + GTest::GTest +) +vespa_add_test(NAME vespalib_util_memory_trap_test_app COMMAND vespalib_util_memory_trap_test_app) diff --git a/vespalib/src/tests/util/memory_trap/memory_trap_test.cpp b/vespalib/src/tests/util/memory_trap/memory_trap_test.cpp new file mode 100644 index 00000000000..ee26231c546 --- /dev/null +++ b/vespalib/src/tests/util/memory_trap/memory_trap_test.cpp @@ -0,0 +1,61 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/vespalib/util/memory_trap.h> +#include <vespa/vespalib/gtest/gtest.h> +#include <cstdlib> + +using namespace vespalib; +using namespace ::testing; + +template <typename T> +void do_not_optimize_away(T&& t) noexcept { + asm volatile("" : : "m"(t) : "memory"); // Clobber the value to avoid losing it to compiler optimizations +} + +struct MemoryTrapTest : Test { + static void SetUpTestSuite() { + // Don't overwrite env var if already set; we'll assume it's done for a good reason. + setenv("VESPA_USE_MPROTECT_TRAP", "yes", 0); + } +}; + +TEST_F(MemoryTrapTest, untouched_memory_traps_do_not_trigger) { + InlineMemoryTrap<2> stack_trap; + HeapMemoryTrap heap_trap(4); + // No touching == no crashing. Good times. +} + +TEST_F(MemoryTrapTest, write_to_stack_trap_eventually_discovered) { + // We don't explicitly test death messages since the way the process dies depends on + // whether mprotect is enabled, whether ASAN instrumentation is enabled etc. + ASSERT_DEATH({ + InlineMemoryTrap<2> stack_trap; + // This may trigger immediately or on destruction. Either way it eventually kills the process. + stack_trap.trapper().buffer()[0] = 0x01; + },""); +} + +TEST_F(MemoryTrapTest, write_to_heap_trap_eventually_discovered) { + ASSERT_DEATH({ + HeapMemoryTrap heap_trap(4); + // This may trigger immediately or on destruction. Either way it eventually kills the process. + heap_trap.trapper().buffer()[heap_trap.trapper().size() - 1] = 0x01; + },""); +} + +TEST_F(MemoryTrapTest, read_from_hw_backed_trap_crashes_process) { + if (!MemoryRangeTrapper::hw_trapping_enabled()) { + return; + } + ASSERT_DEATH({ + HeapMemoryTrap heap_trap(4); // Entire buffer should always be covered + // Clobber trap just in case the compiler is clever enough to look into the trap implementation + // and see that we memset everything to zero and `dummy` can thus be constant-promoted to 0 + // (probably won't dare to do this anyway due to opaque mprotect() that touches buffer pointer). + do_not_optimize_away(heap_trap); + char dummy = heap_trap.trapper().buffer()[0]; + do_not_optimize_away(dummy); // never reached + },""); +} + +GTEST_MAIN_RUN_ALL_TESTS() diff --git a/vespalib/src/vespa/vespalib/data/smart_buffer.h b/vespalib/src/vespa/vespalib/data/smart_buffer.h index 17fb7614f0e..fc7042c5eea 100644 --- a/vespalib/src/vespa/vespalib/data/smart_buffer.h +++ b/vespalib/src/vespa/vespalib/data/smart_buffer.h @@ -33,9 +33,10 @@ private: public: SmartBuffer(size_t initial_size); ~SmartBuffer(); + bool empty() const { return (read_len() == 0); } size_t capacity() const { return _data.size(); } void drop_if_empty() { - if ((read_len() == 0) && (_data.size() > 0)) { + if (empty() && (_data.size() > 0)) { drop(); } } diff --git a/vespalib/src/vespa/vespalib/net/tls/auto_reloading_tls_crypto_engine.cpp b/vespalib/src/vespa/vespalib/net/tls/auto_reloading_tls_crypto_engine.cpp index 7d0d94b3627..4b287c1c86b 100644 --- a/vespalib/src/vespa/vespalib/net/tls/auto_reloading_tls_crypto_engine.cpp +++ b/vespalib/src/vespa/vespalib/net/tls/auto_reloading_tls_crypto_engine.cpp @@ -6,6 +6,7 @@ #include "tls_crypto_engine.h" #include "transport_security_options.h" #include "transport_security_options_reading.h" +#include "crypto_codec.h" #include <functional> #include <stdexcept> @@ -111,14 +112,14 @@ AutoReloadingTlsCryptoEngine::always_use_tls_when_server() const return acquire_current_engine()->always_use_tls_when_server(); } -std::unique_ptr<TlsCryptoSocket> -AutoReloadingTlsCryptoEngine::create_tls_client_crypto_socket(SocketHandle socket, const SocketSpec &spec) { - return acquire_current_engine()->create_tls_client_crypto_socket(std::move(socket), spec); +std::unique_ptr<CryptoCodec> +AutoReloadingTlsCryptoEngine::create_tls_client_crypto_codec(const SocketHandle &socket, const SocketSpec &spec) { + return acquire_current_engine()->create_tls_client_crypto_codec(socket, spec); } -std::unique_ptr<TlsCryptoSocket> -AutoReloadingTlsCryptoEngine::create_tls_server_crypto_socket(SocketHandle socket) { - return acquire_current_engine()->create_tls_server_crypto_socket(std::move(socket)); +std::unique_ptr<CryptoCodec> +AutoReloadingTlsCryptoEngine::create_tls_server_crypto_codec(const SocketHandle &socket) { + return acquire_current_engine()->create_tls_server_crypto_codec(socket); } } diff --git a/vespalib/src/vespa/vespalib/net/tls/auto_reloading_tls_crypto_engine.h b/vespalib/src/vespa/vespalib/net/tls/auto_reloading_tls_crypto_engine.h index b379fd75b99..e642d93bfac 100644 --- a/vespalib/src/vespa/vespalib/net/tls/auto_reloading_tls_crypto_engine.h +++ b/vespalib/src/vespa/vespalib/net/tls/auto_reloading_tls_crypto_engine.h @@ -49,8 +49,8 @@ public: CryptoSocket::UP create_server_crypto_socket(SocketHandle socket) override; bool use_tls_when_client() const override; bool always_use_tls_when_server() const override; - std::unique_ptr<TlsCryptoSocket> create_tls_client_crypto_socket(SocketHandle socket, const SocketSpec &spec) override; - std::unique_ptr<TlsCryptoSocket> create_tls_server_crypto_socket(SocketHandle socket) override; + std::unique_ptr<CryptoCodec> create_tls_client_crypto_codec(const SocketHandle &socket, const SocketSpec &spec) override; + std::unique_ptr<CryptoCodec> create_tls_server_crypto_codec(const SocketHandle &socket) override; }; } diff --git a/vespalib/src/vespa/vespalib/net/tls/maybe_tls_crypto_engine.cpp b/vespalib/src/vespa/vespalib/net/tls/maybe_tls_crypto_engine.cpp index 832d52c0383..971256ff402 100644 --- a/vespalib/src/vespa/vespalib/net/tls/maybe_tls_crypto_engine.cpp +++ b/vespalib/src/vespa/vespalib/net/tls/maybe_tls_crypto_engine.cpp @@ -2,11 +2,22 @@ #include "maybe_tls_crypto_engine.h" #include "maybe_tls_crypto_socket.h" +#include "crypto_codec.h" namespace vespalib { MaybeTlsCryptoEngine::~MaybeTlsCryptoEngine() = default; +std::unique_ptr<net::tls::CryptoCodec> +MaybeTlsCryptoEngine::create_tls_client_crypto_codec(const SocketHandle &socket, const SocketSpec &spec) { + return _tls_engine->create_tls_client_crypto_codec(socket, spec); +} + +std::unique_ptr<net::tls::CryptoCodec> +MaybeTlsCryptoEngine::create_tls_server_crypto_codec(const SocketHandle &socket) { + return _tls_engine->create_tls_server_crypto_codec(socket); +} + CryptoSocket::UP MaybeTlsCryptoEngine::create_client_crypto_socket(SocketHandle socket, const SocketSpec &spec) { diff --git a/vespalib/src/vespa/vespalib/net/tls/maybe_tls_crypto_engine.h b/vespalib/src/vespa/vespalib/net/tls/maybe_tls_crypto_engine.h index 2b82d6eb8bc..e69d5858eab 100644 --- a/vespalib/src/vespa/vespalib/net/tls/maybe_tls_crypto_engine.h +++ b/vespalib/src/vespa/vespalib/net/tls/maybe_tls_crypto_engine.h @@ -15,7 +15,7 @@ namespace vespalib { * connections is controlled by the use_tls_when_client flag given to * the constructor. **/ -class MaybeTlsCryptoEngine : public CryptoEngine +class MaybeTlsCryptoEngine : public AbstractTlsCryptoEngine { private: std::shared_ptr<NullCryptoEngine> _null_engine; @@ -29,6 +29,8 @@ public: _tls_engine(std::move(tls_engine)), _use_tls_when_client(use_tls_when_client) {} ~MaybeTlsCryptoEngine() override; + std::unique_ptr<CryptoCodec> create_tls_client_crypto_codec(const SocketHandle &socket, const SocketSpec &spec) override; + std::unique_ptr<CryptoCodec> create_tls_server_crypto_codec(const SocketHandle &socket) override; bool use_tls_when_client() const override { return _use_tls_when_client; } bool always_use_tls_when_server() const override { return false; } CryptoSocket::UP create_client_crypto_socket(SocketHandle socket, const SocketSpec &spec) override; diff --git a/vespalib/src/vespa/vespalib/net/tls/maybe_tls_crypto_socket.cpp b/vespalib/src/vespa/vespalib/net/tls/maybe_tls_crypto_socket.cpp index 04613cb3a65..0d00ab51309 100644 --- a/vespalib/src/vespa/vespalib/net/tls/maybe_tls_crypto_socket.cpp +++ b/vespalib/src/vespa/vespalib/net/tls/maybe_tls_crypto_socket.cpp @@ -4,6 +4,7 @@ #include "statistics.h" #include "tls_crypto_socket.h" #include "protocol_snooping.h" +#include "crypto_codec_adapter.h" #include <vespa/vespalib/data/smart_buffer.h> #include <vespa/vespalib/net/connection_auth_context.h> #include <vespa/vespalib/util/size_literals.h> @@ -52,7 +53,8 @@ public: } if (looksLikeTlsToMe(src.data)) { CryptoSocket::UP &self = _self; // need copy due to self destruction - auto tls_socket = _factory->create_tls_server_crypto_socket(std::move(_socket)); + auto tls_codec = _factory->create_tls_server_crypto_codec(_socket); + auto tls_socket = std::make_unique<net::tls::CryptoCodecAdapter>(std::move(_socket), std::move(tls_codec)); tls_socket->inject_read_data(src.data, src.size); self = std::move(tls_socket); return self->handshake(); diff --git a/vespalib/src/vespa/vespalib/net/tls/tls_crypto_engine.cpp b/vespalib/src/vespa/vespalib/net/tls/tls_crypto_engine.cpp index 9ae270780b5..6f148568d80 100644 --- a/vespalib/src/vespa/vespalib/net/tls/tls_crypto_engine.cpp +++ b/vespalib/src/vespa/vespalib/net/tls/tls_crypto_engine.cpp @@ -13,17 +13,27 @@ TlsCryptoEngine::TlsCryptoEngine(net::tls::TransportSecurityOptions tls_opts, ne { } -std::unique_ptr<TlsCryptoSocket> -TlsCryptoEngine::create_tls_client_crypto_socket(SocketHandle socket, const SocketSpec &peer_spec) +std::unique_ptr<net::tls::CryptoCodec> +TlsCryptoEngine::create_tls_client_crypto_codec(const SocketHandle &socket, const SocketSpec &peer_spec) { - auto codec = net::tls::CryptoCodec::create_default_client_codec(_tls_ctx, peer_spec, SocketAddress::peer_address(socket.get())); - return std::make_unique<net::tls::CryptoCodecAdapter>(std::move(socket), std::move(codec)); + return net::tls::CryptoCodec::create_default_client_codec(_tls_ctx, peer_spec, SocketAddress::peer_address(socket.get())); } -std::unique_ptr<TlsCryptoSocket> -TlsCryptoEngine::create_tls_server_crypto_socket(SocketHandle socket) +std::unique_ptr<net::tls::CryptoCodec> +TlsCryptoEngine::create_tls_server_crypto_codec(const SocketHandle &socket) { - auto codec = net::tls::CryptoCodec::create_default_server_codec(_tls_ctx, SocketAddress::peer_address(socket.get())); + return net::tls::CryptoCodec::create_default_server_codec(_tls_ctx, SocketAddress::peer_address(socket.get())); +} + +CryptoSocket::UP +TlsCryptoEngine::create_client_crypto_socket(SocketHandle socket, const SocketSpec &peer_spec) { + auto codec = create_tls_client_crypto_codec(socket, peer_spec); + return std::make_unique<net::tls::CryptoCodecAdapter>(std::move(socket), std::move(codec)); +} + +CryptoSocket::UP +TlsCryptoEngine::create_server_crypto_socket(SocketHandle socket) { + auto codec = create_tls_server_crypto_codec(socket); return std::make_unique<net::tls::CryptoCodecAdapter>(std::move(socket), std::move(codec)); } diff --git a/vespalib/src/vespa/vespalib/net/tls/tls_crypto_engine.h b/vespalib/src/vespa/vespalib/net/tls/tls_crypto_engine.h index 0e05363ab1b..1ee4cf07559 100644 --- a/vespalib/src/vespa/vespalib/net/tls/tls_crypto_engine.h +++ b/vespalib/src/vespa/vespalib/net/tls/tls_crypto_engine.h @@ -9,10 +9,13 @@ namespace vespalib { +namespace net { namespace tls { class CryptoCodec; }} + class AbstractTlsCryptoEngine : public CryptoEngine { public: - virtual std::unique_ptr<TlsCryptoSocket> create_tls_client_crypto_socket(SocketHandle socket, const SocketSpec &spec) = 0; - virtual std::unique_ptr<TlsCryptoSocket> create_tls_server_crypto_socket(SocketHandle socket) = 0; + using CryptoCodec = net::tls::CryptoCodec; + virtual std::unique_ptr<CryptoCodec> create_tls_client_crypto_codec(const SocketHandle &socket, const SocketSpec &spec) = 0; + virtual std::unique_ptr<CryptoCodec> create_tls_server_crypto_codec(const SocketHandle &socket) = 0; }; /** @@ -26,17 +29,12 @@ public: explicit TlsCryptoEngine(net::tls::TransportSecurityOptions tls_opts, net::tls::AuthorizationMode authz_mode = net::tls::AuthorizationMode::Enforce); ~TlsCryptoEngine() override; - std::unique_ptr<TlsCryptoSocket> create_tls_client_crypto_socket(SocketHandle socket, const SocketSpec &spec) override; - std::unique_ptr<TlsCryptoSocket> create_tls_server_crypto_socket(SocketHandle socket) override; + std::unique_ptr<CryptoCodec> create_tls_client_crypto_codec(const SocketHandle &socket, const SocketSpec &spec) override; + std::unique_ptr<CryptoCodec> create_tls_server_crypto_codec(const SocketHandle &socket) override; bool use_tls_when_client() const override { return true; } bool always_use_tls_when_server() const override { return true; } - CryptoSocket::UP create_client_crypto_socket(SocketHandle socket, const SocketSpec &spec) override { - return create_tls_client_crypto_socket(std::move(socket), spec); - } - CryptoSocket::UP create_server_crypto_socket(SocketHandle socket) override { - return create_tls_server_crypto_socket(std::move(socket)); - } - + CryptoSocket::UP create_client_crypto_socket(SocketHandle socket, const SocketSpec &spec) override; + CryptoSocket::UP create_server_crypto_socket(SocketHandle socket) override; std::shared_ptr<net::tls::TlsContext> tls_context() const noexcept { return _tls_ctx; }; }; diff --git a/vespalib/src/vespa/vespalib/util/CMakeLists.txt b/vespalib/src/vespa/vespalib/util/CMakeLists.txt index 3812fda4bdf..ad2db89288c 100644 --- a/vespalib/src/vespa/vespalib/util/CMakeLists.txt +++ b/vespalib/src/vespa/vespalib/util/CMakeLists.txt @@ -54,6 +54,7 @@ vespa_add_library(vespalib_vespalib_util OBJECT lz4compressor.cpp malloc_mmap_guard.cpp md5.c + memory_trap.cpp memoryusage.cpp mmap_file_allocator.cpp mmap_file_allocator_factory.cpp diff --git a/vespalib/src/vespa/vespalib/util/memory_trap.cpp b/vespalib/src/vespa/vespalib/util/memory_trap.cpp new file mode 100644 index 00000000000..d3b666d9a6e --- /dev/null +++ b/vespalib/src/vespa/vespalib/util/memory_trap.cpp @@ -0,0 +1,166 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "memory_trap.h" +#include <string_view> +#include <cassert> +#include <cerrno> +#include <cstdint> +#include <cstdlib> +#include <cstring> +#include <malloc.h> +#include <unistd.h> +#include <sys/mman.h> + +#include <vespa/log/log.h> +LOG_SETUP(".vespalib.util.memory_trap"); + +using namespace std::string_view_literals; + +namespace vespalib { + +namespace { + +// Have some symbols that provide immediate context in a crash backtrace +[[noreturn]] void abort_due_to_guard_bits_tampered_with() __attribute__((noinline)); +[[noreturn]] void abort_due_to_guard_bits_tampered_with() { + abort(); +} + +[[noreturn]] void abort_due_to_PROTECTED_guard_bits_tampered_with() __attribute__((noinline)); +[[noreturn]] void abort_due_to_PROTECTED_guard_bits_tampered_with() { + abort(); +} + +} // anon ns + +MemoryRangeTrapper::MemoryRangeTrapper(char* trap_buf, size_t buf_len) noexcept + : _trap_buf(trap_buf), + _buf_len(buf_len), + _trap_offset(0), + _trap_len(0) +{ + if (_buf_len > 0) { + memset(trap_buf, 0, _buf_len); + } + rw_protect_buffer_if_possible(); +} + +MemoryRangeTrapper::~MemoryRangeTrapper() { + check_and_release(); +} + +void MemoryRangeTrapper::check_and_release() noexcept { + unprotect_buffer_to_read_only(); // Make sure sanity check can't race with writes + verify_buffer_is_all_zeros(); + unprotect_buffer_to_read_and_write(); + _trap_len = _buf_len = 0; +} + +void MemoryRangeTrapper::verify_buffer_is_all_zeros() { + for (size_t i = 0; i < _buf_len; ++i) { + if (_trap_buf[i] != 0) { + const bool in_protected_area = ((i >= _trap_offset) && (i < _trap_offset + _trap_len)); + LOG(error, "Memory corruption detected! Offset %zu into buffer %p: 0x%.2x != 0x00%s", + i, _trap_buf, static_cast<unsigned int>(_trap_buf[i]), + in_protected_area ? ". CORRUPTION IN R/W PROTECTED MEMORY!" : ""); + if (in_protected_area) { + abort_due_to_PROTECTED_guard_bits_tampered_with(); + } else { + abort_due_to_guard_bits_tampered_with(); + } + } + } +} + +#ifdef __linux__ + +namespace { + +bool has_4k_pages() noexcept { + return (sysconf(_SC_PAGESIZE) == 4096); +} + +constexpr bool is_4k_aligned(size_t v) noexcept { + return (v % 4096) == 0; +} + +constexpr size_t align_up_4k(size_t v) noexcept { + return (v + 4095) & ~4095ULL; +} + +constexpr size_t align_down_4k(size_t v) noexcept { + return v & ~4095ULL; +} + +bool env_var_is_yes(const char *env_var) noexcept { + const char *ev = getenv(env_var); + return ((ev != nullptr) && ("yes"sv == ev)); +} + +bool mprotect_trapping_is_enabled() noexcept { + static const bool enabled = (has_4k_pages() && env_var_is_yes("VESPA_USE_MPROTECT_TRAP")); + return enabled; +} + +} // anon ns + +void MemoryRangeTrapper::rw_protect_buffer_if_possible() { + static_assert(std::is_same_v<size_t, uintptr_t>); + const auto aligned_start = align_up_4k(reinterpret_cast<uintptr_t>(_trap_buf)); + const auto aligned_end = align_down_4k(reinterpret_cast<uintptr_t>(_trap_buf + _buf_len)); + if ((aligned_end > aligned_start) && mprotect_trapping_is_enabled()) { + _trap_offset = aligned_start - reinterpret_cast<uintptr_t>(_trap_buf); + _trap_len = aligned_end - aligned_start; + assert(is_4k_aligned(_trap_len)); + + LOG(info, "attempting mprotect(%p + %zu = %p, %zu, PROT_NONE)", + _trap_buf, _trap_offset, _trap_buf + _trap_offset, _trap_len); + int ret = mprotect(_trap_buf + _trap_offset, _trap_len, PROT_NONE); + if (ret != 0) { + LOG(warning, "Failed to mprotect(%p + %zu, %zu, PROT_NONE). errno = %d. " + "Falling back to unprotected mode.", + _trap_buf, _trap_offset, _trap_len, errno); + _trap_offset = _trap_len = 0; + } + } +} + +bool MemoryRangeTrapper::hw_trapping_enabled() noexcept { + return mprotect_trapping_is_enabled(); +} + +void MemoryRangeTrapper::unprotect_buffer_to_read_only() { + if (_trap_len > 0) { + int ret = mprotect(_trap_buf + _trap_offset, _trap_len, PROT_READ); + assert(ret == 0 && "failed to un-protect memory region to PROT_READ"); + } +} + +void MemoryRangeTrapper::unprotect_buffer_to_read_and_write() { + if (_trap_len > 0) { + int ret = mprotect(_trap_buf + _trap_offset, _trap_len, PROT_READ | PROT_WRITE); + assert(ret == 0 && "failed to un-protect memory region to PROT_READ | PROT_WRITE"); + } +} + +#else // Not on Linux, fall back to no-ops + +void MemoryRangeTrapper::rw_protect_buffer_if_possible() { /* no-op */ } +bool MemoryRangeTrapper::hw_trapping_enabled() noexcept { return false; } +void MemoryRangeTrapper::unprotect_buffer_to_read_only() { /* no-op */ } +void MemoryRangeTrapper::unprotect_buffer_to_read_and_write() { /* no-op */ } + +#endif + +HeapMemoryTrap::HeapMemoryTrap(size_t trap_4k_pages) + : _trap_buf(static_cast<char*>(memalign(4096, trap_4k_pages * 4096))), + _trapper(_trap_buf, _trap_buf ? trap_4k_pages * 4096 : 0) +{ +} + +HeapMemoryTrap::~HeapMemoryTrap() { + _trapper.check_and_release(); + free(_trap_buf); +} + +} diff --git a/vespalib/src/vespa/vespalib/util/memory_trap.h b/vespalib/src/vespa/vespalib/util/memory_trap.h new file mode 100644 index 00000000000..f9c8e8458fd --- /dev/null +++ b/vespalib/src/vespa/vespalib/util/memory_trap.h @@ -0,0 +1,101 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <cstddef> + +namespace vespalib { + +/** + * Guard for attempting to detect spurious writes (and if possible; reads) to a memory region. + * + * If supported by the OS+HW, as much as possible of the buffer will be mapped + * as non-readable and non-writable. This immediately triggers a SIGSEGV for any + * spurious read or write to the mapped buffer sub-range. + * + * For memory map-backed trapping to be used, all of the following must hold: + * - The process must be running on Linux and on hardware with a page size of 4 KiB + * - The environment variable VESPA_USE_MPROTECT_TRAP must be set and have the value 'yes' + * - The trap buffer must be long enough to fit at least one whole 4 KiB-aligned page + * - The buffer passed to the trapper must originally have been allocated via mmap(). + * This should hold for any reasonable implementation of malloc(). + * + * Regardless of whether memory map-backed trapping is used, the buffer will always be + * filled with all zeroes upon construction. If any buffer byte is non-zero upon + * destruction, the process will be terminated with a corruption error in the logs. + * + * If buffer mapping fails during construction, the trapper falls back to just checking + * buffer contents. This may happen if the kernel has exhausted the bookkeeping-structures + * for keeping track of separate virtual memory ranges. + * + * Note that due to possible interference with things like hugepages etc, VESPA_USE_MPROTECT_TRAP + * should only be selectively enabled. + */ +class MemoryRangeTrapper { + char* _trap_buf; + size_t _buf_len; + size_t _trap_offset; + size_t _trap_len; +public: + MemoryRangeTrapper(char* trap_buf, size_t buf_len) noexcept; + ~MemoryRangeTrapper(); + + MemoryRangeTrapper(const MemoryRangeTrapper&) = delete; + MemoryRangeTrapper(MemoryRangeTrapper&&) noexcept = delete; + + // Exposed for testing only + char* buffer() const noexcept { return _trap_buf; } + size_t size() const noexcept { return _buf_len; } + + void check_and_release() noexcept; + + [[nodiscard]] static bool hw_trapping_enabled() noexcept; +private: + void rw_protect_buffer_if_possible(); + void unprotect_buffer_to_read_only(); + void unprotect_buffer_to_read_and_write(); + void verify_buffer_is_all_zeros(); +}; + +/** + * Places a memory trap "inline" with other variables in an object. I.e. the trap will + * be in a memory range that is a sub-range of that taken up by the owning object. + * + * Always takes up at least 8 KiB of space. + */ +template <size_t Guard4KPages> +class InlineMemoryTrap { + static_assert(Guard4KPages > 0); + constexpr static size_t BufSize = 4096 * (Guard4KPages + 1); + char _trap_buf[BufSize]; + MemoryRangeTrapper _trapper; +public: + InlineMemoryTrap() noexcept : _trap_buf(), _trapper(_trap_buf, BufSize) {} + ~InlineMemoryTrap() = default; + + InlineMemoryTrap(const InlineMemoryTrap&) = delete; + InlineMemoryTrap(InlineMemoryTrap&&) noexcept = delete; + + // Exposed for testing only + const MemoryRangeTrapper& trapper() const noexcept { return _trapper; } +}; + +/** + * Allocates a 4 KiB-aligned heap buffer and watches it for spurious access. + * Useful for distributing traps across various allocation size-classes. + */ +class HeapMemoryTrap { + char* _trap_buf; + MemoryRangeTrapper _trapper; +public: + explicit HeapMemoryTrap(size_t trap_4k_pages); + ~HeapMemoryTrap(); + + HeapMemoryTrap(const HeapMemoryTrap&) = delete; + HeapMemoryTrap(HeapMemoryTrap&&) noexcept = delete; + + // Exposed for testing only + const MemoryRangeTrapper& trapper() const noexcept { return _trapper; } +}; + +} diff --git a/vespalib/src/vespa/vespalib/util/mmap_file_allocator.h b/vespalib/src/vespa/vespalib/util/mmap_file_allocator.h index 3b7b0039fab..0a83bfb4e60 100644 --- a/vespalib/src/vespa/vespalib/util/mmap_file_allocator.h +++ b/vespalib/src/vespa/vespalib/util/mmap_file_allocator.h @@ -13,7 +13,7 @@ namespace vespalib::alloc { /* * Class handling memory allocations backed by one or more files. - * Not reentant. Should not be destructed before all allocations + * Not reentrant or thread safe. Should not be destructed before all allocations * have been freed. */ class MmapFileAllocator : public MemoryAllocator { @@ -30,8 +30,8 @@ class MmapFileAllocator : public MemoryAllocator { { } }; - vespalib::string _dir_name; - mutable File _file; + const vespalib::string _dir_name; + mutable File _file; mutable uint64_t _end_offset; mutable hash_map<void *, SizeAndOffset> _allocations; mutable FileAreaFreeList _freelist; |