diff options
18 files changed, 155 insertions, 1467 deletions
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomConfigPayloadBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomConfigPayloadBuilder.java index e9abcfd87a9..d07f6c4e6fd 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomConfigPayloadBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomConfigPayloadBuilder.java @@ -3,17 +3,16 @@ package com.yahoo.vespa.model.builder.xml.dom; import com.yahoo.collections.Tuple2; import com.yahoo.config.ConfigurationRuntimeException; -import com.yahoo.config.codegen.CNode; -import com.yahoo.log.LogLevel; +import com.yahoo.vespa.config.ConfigDefinition; +import com.yahoo.vespa.config.ConfigDefinitionKey; +import com.yahoo.vespa.config.ConfigPayloadBuilder; import com.yahoo.yolean.Exceptions; import com.yahoo.text.XML; -import com.yahoo.vespa.config.*; import com.yahoo.vespa.config.util.ConfigUtils; import org.w3c.dom.Element; -import java.util.*; -import java.util.logging.Logger; +import java.util.List; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -27,8 +26,6 @@ import java.util.regex.Pattern; */ public class DomConfigPayloadBuilder { - private static final Logger log = Logger.getLogger(DomConfigPayloadBuilder.class.getPackage().toString()); - private static final Pattern namePattern = ConfigDefinition.namePattern; private static final Pattern namespacePattern = ConfigDefinition.namespacePattern; @@ -57,40 +54,30 @@ public class DomConfigPayloadBuilder { public static ConfigDefinitionKey parseConfigName(Element configE) { if (!configE.getNodeName().equals("config")) { - throw new ConfigurationRuntimeException("The root element must be 'config', but was '" - + configE.getNodeName() + "'."); + throw new ConfigurationRuntimeException("The root element must be 'config', but was '" + configE.getNodeName() + "'."); } + if (!configE.hasAttribute("name")) { throw new ConfigurationRuntimeException ("The 'config' element must have a 'name' attribute that matches the name of the config definition."); } - String xmlName = configE.getAttribute("name"); - final boolean xmlNamespaceAttributeExists = configE.hasAttribute("namespace"); - - String xmlNamespace = null; - // If name contains dots, rewrite to name and namespace - if (xmlName.contains(".")) { - Tuple2<String, String> t = ConfigUtils.getNameAndNamespaceFromString(xmlName); - xmlName = t.first; - xmlNamespace = t.second; - } else { - if (!xmlNamespaceAttributeExists) { - log.log(LogLevel.WARNING, "No namespace in 'config name=" + xmlName + "', please specify one"); - } + String elementString = configE.getAttribute("name"); + if (!elementString.contains(".")) { + throw new ConfigurationRuntimeException("The config name '" + elementString + + "' contains illegal characters. Only names with the pattern " + + namespacePattern.pattern() + "." + namePattern.pattern() + " are legal."); } + Tuple2<String, String> t = ConfigUtils.getNameAndNamespaceFromString(elementString); + String xmlName = t.first; + String xmlNamespace = t.second; + if (!validName(xmlName)) { throw new ConfigurationRuntimeException("The config name '" + xmlName + "' contains illegal characters. Only names with the pattern " + namePattern.toString() + " are legal."); } - if (xmlNamespace == null) { - xmlNamespace = configE.getAttribute("namespace"); - if (xmlNamespace == null || xmlNamespace.isEmpty()) { - xmlNamespace = CNode.DEFAULT_NAMESPACE; - } - } if (!validNamespace(xmlNamespace)) { throw new ConfigurationRuntimeException("The config namespace '" + xmlNamespace + "' contains illegal characters. Only namespaces with the pattern " + namespacePattern.toString() + " are legal."); diff --git a/config-model/src/main/resources/schema/common.rnc b/config-model/src/main/resources/schema/common.rnc index 3e33227b064..c5690f9c915 100644 --- a/config-model/src/main/resources/schema/common.rnc +++ b/config-model/src/main/resources/schema/common.rnc @@ -41,7 +41,7 @@ OptionalDedicatedNodes = element nodes { GenericConfig = element config { attribute name { text }, - attribute namespace { text }?, + attribute namespace { text }?, # TODO: Remove in Vespa 8 attribute version { text }?, anyElement + } diff --git a/config-model/src/test/cfg/admin/userconfigs/functiontest-defaultvalues.xml b/config-model/src/test/cfg/admin/userconfigs/functiontest-defaultvalues.xml index 7da7d788ca1..298d6676354 100644 --- a/config-model/src/test/cfg/admin/userconfigs/functiontest-defaultvalues.xml +++ b/config-model/src/test/cfg/admin/userconfigs/functiontest-defaultvalues.xml @@ -1,6 +1,6 @@ <?xml version="1.0" encoding="utf-8" ?> <!-- Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> -<config name="function-test"> +<config name="test.function-test"> <bool_val>false</bool_val> <int_val>5</int_val> <long_val>1234567890123</long_val> diff --git a/config-model/src/test/cfg/admin/userconfigs/whitespace-test.xml b/config-model/src/test/cfg/admin/userconfigs/whitespace-test.xml index 68576786376..956cba1c9ff 100644 --- a/config-model/src/test/cfg/admin/userconfigs/whitespace-test.xml +++ b/config-model/src/test/cfg/admin/userconfigs/whitespace-test.xml @@ -1,6 +1,6 @@ <?xml version="1.0" encoding="utf-8" ?> <!-- Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> -<config name="function-test"> +<config name="test.function-test"> <stringVal> This is a string that contains different kinds of whitespace </stringVal> </config> diff --git a/config-model/src/test/cfg/application/app_genericservices/services.xml b/config-model/src/test/cfg/application/app_genericservices/services.xml index 6c7cad10e5d..9ae7f122e81 100644 --- a/config-model/src/test/cfg/application/app_genericservices/services.xml +++ b/config-model/src/test/cfg/application/app_genericservices/services.xml @@ -3,11 +3,11 @@ <services version="1.0"> <service version="1.0" name="myservice" command="mycmd1.sh"> - <config name="myconfig"> + <config name="a.myconfig"> <mysetting>bar</mysetting> </config> <node hostalias="node1"> - <config name="myconfig"> + <config name="a.myconfig"> <mysetting>baz</mysetting> </config> </node> @@ -17,11 +17,11 @@ </service> <service version="1.0" name="myotherservice" command="/home/vespa/bin/mycmd2.sh --ytest $FOO_BAR"> - <config name="myconfig"> + <config name="a.myconfig"> <mysetting>bar2</mysetting> </config> <node hostalias="node3"> - <config name="myconfig"> + <config name="a.myconfig"> <mysetting>baz2</mysetting> </config> </node> diff --git a/config-model/src/test/java/com/yahoo/vespa/model/builder/UserConfigBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/builder/UserConfigBuilderTest.java index f3eeec62cff..6bfd55ca5de 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/builder/UserConfigBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/builder/UserConfigBuilderTest.java @@ -8,7 +8,6 @@ import com.yahoo.config.model.deploy.ConfigDefinitionStore; import com.yahoo.test.SimpletypesConfig; import com.yahoo.config.model.producer.UserConfigRepo; import com.yahoo.config.model.builder.xml.XmlHelper; -import com.yahoo.vespa.config.ConfigDefinition; import com.yahoo.vespa.config.ConfigDefinitionKey; import com.yahoo.vespa.config.ConfigPayload; import com.yahoo.vespa.config.ConfigPayloadBuilder; @@ -18,7 +17,6 @@ import org.w3c.dom.Document; import org.w3c.dom.Element; import org.xml.sax.InputSource; -import javax.xml.parsers.ParserConfigurationException; import java.io.Reader; import java.io.StringReader; import java.util.Optional; @@ -28,47 +26,42 @@ import static org.junit.Assert.*; /** * @author Ulf Lilleengen - * @since 5.1 */ public class UserConfigBuilderTest { - private final ConfigDefinitionStore configDefinitionStore = new ConfigDefinitionStore() { - @Override - public Optional<ConfigDefinition> getConfigDefinition(ConfigDefinitionKey defKey) { return Optional.empty(); } - }; + private final ConfigDefinitionStore configDefinitionStore = defKey -> Optional.empty(); @Test - public void require_that_simple_config_is_resolved() throws ParserConfigurationException { - Element configRoot = getDocument("<config name=\"simpletypes\">" + + public void require_that_simple_config_is_resolved() { + Element configRoot = getDocument("<config name=\"test.simpletypes\">" + " <intval>13</intval>" + "</config>" + - "<config name=\"simpletypes\" version=\"1\">" + + "<config name=\"test.simpletypes\" version=\"1\">" + " <stringval>foolio</stringval>" + "</config>"); UserConfigRepo map = UserConfigBuilder.build(configRoot, configDefinitionStore, new BaseDeployLogger()); assertFalse(map.isEmpty()); - ConfigDefinitionKey key = new ConfigDefinitionKey("simpletypes", "config"); + ConfigDefinitionKey key = new ConfigDefinitionKey("simpletypes", "test"); assertNotNull(map.get(key)); SimpletypesConfig config = createConfig(SimpletypesConfig.class, map.get(key)); assertThat(config.intval(), is(13)); assertThat(config.stringval(), is("foolio")); } - public static <ConfigType extends ConfigInstance> ConfigType createConfig(Class<ConfigType> clazz, ConfigPayloadBuilder builder) { + private static <ConfigType extends ConfigInstance> ConfigType createConfig(Class<ConfigType> clazz, ConfigPayloadBuilder builder) { return ConfigPayload.fromBuilder(builder).toInstance(clazz, ""); } - @Test - public void require_that_arrays_config_is_resolved() throws ParserConfigurationException { - Element configRoot = getDocument("<config name=\"arraytypes\">" + + public void require_that_arrays_config_is_resolved() { + Element configRoot = getDocument("<config name=\"test.arraytypes\">" + " <intarr operation=\"append\">13</intarr>" + " <intarr operation=\"append\">10</intarr>" + " <intarr operation=\"append\">1337</intarr>" + "</config>"); UserConfigRepo map = UserConfigBuilder.build(configRoot, configDefinitionStore, new BaseDeployLogger()); assertFalse(map.isEmpty()); - ConfigDefinitionKey key = new ConfigDefinitionKey("arraytypes", "config"); + ConfigDefinitionKey key = new ConfigDefinitionKey("arraytypes", "test"); assertNotNull(map.get(key)); ArraytypesConfig config = createConfig(ArraytypesConfig.class, map.get(key)); assertThat(config.intarr().size(), is(3)); @@ -78,7 +71,7 @@ public class UserConfigBuilderTest { } @Test - public void require_that_arrays_of_structs_are_resolved() throws ParserConfigurationException { + public void require_that_arrays_of_structs_are_resolved() { Element configRoot = getDocument( " <config name='vespa.configdefinition.specialtokens'>" + " <tokenlist operation='append'>" + @@ -105,12 +98,12 @@ public class UserConfigBuilderTest { } @Test - public void no_exception_when_config_class_does_not_exist() throws ParserConfigurationException { - Element configRoot = getDocument("<config name=\"unknown\">" + + public void no_exception_when_config_class_does_not_exist() { + Element configRoot = getDocument("<config name=\"is.unknown\">" + " <foo>1</foo>" + "</config>"); UserConfigRepo repo = UserConfigBuilder.build(configRoot, configDefinitionStore, new BaseDeployLogger()); - ConfigPayloadBuilder builder = repo.get(new ConfigDefinitionKey("unknown", "config")); + ConfigPayloadBuilder builder = repo.get(new ConfigDefinitionKey("unknown", "is")); assertNotNull(builder); } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomConfigPayloadBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomConfigPayloadBuilderTest.java index 5359e09347e..a62255c4c5c 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomConfigPayloadBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomConfigPayloadBuilderTest.java @@ -16,7 +16,6 @@ import org.w3c.dom.Document; import org.w3c.dom.Element; import org.xml.sax.InputSource; -import javax.xml.parsers.ParserConfigurationException; import java.io.ByteArrayOutputStream; import java.io.File; import java.io.FileNotFoundException; @@ -37,7 +36,7 @@ import static org.junit.Assert.fail; public class DomConfigPayloadBuilderTest { @Test - public void testFunctionTest_DefaultValues() throws FileNotFoundException, ParserConfigurationException { + public void testFunctionTest_DefaultValues() throws FileNotFoundException { Element configRoot = getDocument(new FileReader(new File("src/test/cfg/admin/userconfigs/functiontest-defaultvalues.xml"))); ConfigPayload config = ConfigPayload.fromBuilder(new DomConfigPayloadBuilder(null).build(configRoot)); String expected = "" @@ -81,9 +80,9 @@ public class DomConfigPayloadBuilderTest { } @Test - public void put_to_leaf_map() throws Exception { + public void put_to_leaf_map() { Reader xmlConfig = new StringReader("<?xml version=\"1.0\" encoding=\"utf-8\" ?>" + - "<config name=\"foobar\">" + + "<config name=\"test.foobar\">" + " <intmap>" + " <item key=\"bar\">1338</item>" + " <item key=\"foo\">1337</item>" + @@ -94,9 +93,9 @@ public class DomConfigPayloadBuilderTest { } @Test - public void put_to_inner_map() throws Exception { + public void put_to_inner_map() { Reader xmlConfig = new StringReader("<?xml version=\"1.0\" encoding=\"utf-8\" ?>" + - "<config name=\"foobar\">" + + "<config name=\"test.foobar\">" + " <innermap>" + " <item key=\"bar\">" + " <foo>baz</foo>" + @@ -111,9 +110,9 @@ public class DomConfigPayloadBuilderTest { } @Test - public void put_to_nested_map() throws Exception { + public void put_to_nested_map() { Reader xmlConfig = new StringReader("<?xml version=\"1.0\" encoding=\"utf-8\" ?>" + - "<config name=\"foobar\">" + + "<config name=\"test.foobar\">" + " <nestedmap>" + " <item key=\"bar\">" + " <inner>" + @@ -136,10 +135,10 @@ public class DomConfigPayloadBuilderTest { } @Test - public void append_to_leaf_array() throws Exception { + public void append_to_leaf_array() { // Simulate user config from vespa-services.xml Reader xmlConfig = new StringReader("<?xml version=\"1.0\" encoding=\"utf-8\" ?>" + - "<config name=\"function-test\">" + + "<config name=\"a.function-test\">" + " <intarr operation=\"append\">1</intarr>" + " <intarr operation=\"append\">2</intarr>" + "</config> "); @@ -148,9 +147,9 @@ public class DomConfigPayloadBuilderTest { } @Test - public void camel_case_via_dashes() throws Exception { + public void camel_case_via_dashes() { Reader xmlConfig = new StringReader("<?xml version=\"1.0\" encoding=\"utf-8\" ?>" + - "<config name=\"function-test\">" + + "<config name=\"test.function-test\">" + " <some-struct> <any-value>17</any-value> </some-struct>" + "</config> "); ConfigPayload userConfig = ConfigPayload.fromBuilder(new DomConfigPayloadBuilder(null).build(getDocument(xmlConfig))); @@ -159,7 +158,7 @@ public class DomConfigPayloadBuilderTest { // Verifies that an exception is thrown when the root element is not 'config'. @Test - public void testFailWrongTagName() throws ParserConfigurationException { + public void testFailWrongTagName() { Element configRoot = getDocument(new StringReader("<configs name=\"foo\"/>")); try { new DomConfigPayloadBuilder(null).build(configRoot); @@ -172,7 +171,7 @@ public class DomConfigPayloadBuilderTest { // Verifies that an exception is thrown when the root element is not 'config'. @Test - public void testFailNoNameAttribute() throws ParserConfigurationException { + public void testFailNoNameAttribute() { Element configRoot = getDocument(new StringReader("<config/>")); try { new DomConfigPayloadBuilder(null).build(configRoot); @@ -184,53 +183,17 @@ public class DomConfigPayloadBuilderTest { } @Test - public void testNamespace() throws ParserConfigurationException { - Element configRoot = getDocument(new StringReader("<config name=\"function-test\" namespace=\"config\">" + - "<int_val>1</int_val> +" + - "</config>")); - ConfigPayload config = ConfigPayload.fromBuilder(new DomConfigPayloadBuilder(null).build(configRoot)); - assertPayload("{\"int_val\":\"1\"}", config); - - configRoot = getDocument(new StringReader("<config name=\"config.function-test\">" + - "<int_val>1</int_val> +" + - "</config>")); - config = ConfigPayload.fromBuilder(new DomConfigPayloadBuilder(null).build(configRoot)); - assertPayload("{\"int_val\":\"1\"}", config); - - configRoot = getDocument(new StringReader("<config name=\"config.function_test\">" + - "<int_val>1</int_val> +" + - "</config>")); - config = ConfigPayload.fromBuilder(new DomConfigPayloadBuilder(null).build(configRoot)); - assertPayload("{\"int_val\":\"1\"}", config); - } - - @Test - public void testNameParsing() throws ParserConfigurationException { - Element configRoot = getDocument(new StringReader("<config name=\"function-test\" version=\"1\" namespace=\"config\">" + + public void testNameParsing() { + Element configRoot = getDocument(new StringReader("<config name=\"test.function-test\" version=\"1\">" + "<int_val>1</int_val> +" + "</config>")); ConfigDefinitionKey key = DomConfigPayloadBuilder.parseConfigName(configRoot); assertThat(key.getName(), is("function-test")); - assertThat(key.getNamespace(), is("config")); - - configRoot = getDocument(new StringReader("<config name=\"function_test\" version=\"1\">" + - "<int_val>1</int_val> +" + - "</config>")); - key = DomConfigPayloadBuilder.parseConfigName(configRoot); - assertThat(key.getName(), is("function_test")); - assertThat(key.getNamespace(), is("config")); - - // Both namespace and name in name attribute - configRoot = getDocument(new StringReader("<config name=\"config.function-test\" version=\"1\">" + - "<int_val>1</int_val> +" + - "</config>")); - key = DomConfigPayloadBuilder.parseConfigName(configRoot); - assertThat(key.getName(), is("function-test")); - assertThat(key.getNamespace(), is("config")); + assertThat(key.getNamespace(), is("test")); } @Test(expected = ConfigurationRuntimeException.class) - public void testNameParsingInvalidName() throws ParserConfigurationException { + public void testNameParsingInvalidName() { Element configRoot = getDocument(new StringReader("<config name=\" function-test\" version=\"1\">" + "<int_val>1</int_val> +" + "</config>")); @@ -238,17 +201,17 @@ public class DomConfigPayloadBuilderTest { } @Test(expected = ConfigurationRuntimeException.class) - public void testNameParsingInvalidNamespace() throws ParserConfigurationException { - Element configRoot = getDocument(new StringReader("<config name=\"function-test\" namespace=\"_foo\" version=\"1\">" + + public void testNameParsingInvalidNamespace() { + Element configRoot = getDocument(new StringReader("<config name=\"_foo.function-test\" version=\"1\">" + "<int_val>1</int_val> +" + "</config>")); DomConfigPayloadBuilder.parseConfigName(configRoot); } @Test - public void require_that_item_syntax_works_with_leaf() throws ParserConfigurationException { + public void require_that_item_syntax_works_with_leaf() { Element configRoot = getDocument( - "<config name=\"arraytypes\" version=\"1\">" + + "<config name=\"test.arraytypes\" version=\"1\">" + " <intarr>" + " <item>13</item>" + " <item>10</item>" + @@ -261,9 +224,9 @@ public class DomConfigPayloadBuilderTest { } @Test - public void require_that_item_syntax_works_with_struct() throws ParserConfigurationException { + public void require_that_item_syntax_works_with_struct() { Element configRoot = getDocument( - "<config name=\"arraytypes\" version=\"1\">" + + "<config name=\"test.arraytypes\" version=\"1\">" + " <lolarray>" + " <item><foo>hei</foo><bar>hei2</bar></item>" + " <item><foo>hoo</foo><bar>hoo2</bar></item>" + @@ -277,9 +240,9 @@ public class DomConfigPayloadBuilderTest { } @Test - public void require_that_item_syntax_works_with_struct_array() throws ParserConfigurationException { + public void require_that_item_syntax_works_with_struct_array() { Element configRoot = getDocument( - "<config name=\"arraytypes\" version=\"1\">" + + "<config name=\"test.arraytypes\" version=\"1\">" + " <lolarray>" + " <item><fooarray><item>13</item></fooarray></item>" + " <item><fooarray><item>10</item></fooarray></item>" + @@ -292,18 +255,18 @@ public class DomConfigPayloadBuilderTest { } @Test(expected = ConfigurationRuntimeException.class) - public void require_that_item_is_reserved_in_root() throws ParserConfigurationException { + public void require_that_item_is_reserved_in_root() { Element configRoot = getDocument( - "<config name=\"arraytypes\" version=\"1\">" + + "<config name=\"test.arraytypes\" version=\"1\">" + " <item>13</item>" + "</config>"); new DomConfigPayloadBuilder(null).build(configRoot); } @Test(expected=ConfigurationRuntimeException.class) - public void require_that_exceptions_are_issued() throws ParserConfigurationException, FileNotFoundException { + public void require_that_exceptions_are_issued() throws FileNotFoundException { Element configRoot = getDocument( - "<config name=\"simpletypes\">" + + "<config name=\"test.simpletypes\">" + "<longval>invalid</longval>" + "</config>"); DefParser defParser = new DefParser("simpletypes", @@ -323,7 +286,7 @@ public class DomConfigPayloadBuilderTest { return doc.getDocumentElement(); } - private Element getDocument(String xml) throws ParserConfigurationException { + private Element getDocument(String xml) { Reader xmlReader = new StringReader(xml); return getDocument(xmlReader); } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/LegacyConfigModelBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/LegacyConfigModelBuilderTest.java index bef2b37d65d..fa4c4d313a4 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/LegacyConfigModelBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/LegacyConfigModelBuilderTest.java @@ -3,15 +3,14 @@ package com.yahoo.vespa.model.builder.xml.dom; import com.yahoo.config.model.ConfigModel; import com.yahoo.config.model.ConfigModelContext; -import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.builder.xml.ConfigModelId; +import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.test.MockApplicationPackage; import com.yahoo.config.model.test.MockRoot; import com.yahoo.text.XML; import org.junit.Test; import org.w3c.dom.Element; -import java.util.Arrays; import java.util.List; import static org.hamcrest.CoreMatchers.is; @@ -19,12 +18,12 @@ import static org.junit.Assert.assertThat; /** * @author Ulf Lilleengen - * @since 5.1 */ public class LegacyConfigModelBuilderTest { + @Test public void testThatProducerIsInserted() { - String services = "<foo><config name=\"bar\"><key>value</key></config></foo>"; + String services = "<foo><config name=\"bar.foo\"><key>value</key></config></foo>"; ModelBuilder builder = new ModelBuilder(); Model model = builder.build(DeployState.createTestState(new MockApplicationPackage.Builder().withServices(services).build()), null, null, new MockRoot(), XML.getDocument(services).getDocumentElement()); @@ -51,7 +50,7 @@ public class LegacyConfigModelBuilderTest { } private static class ModelBuilder extends LegacyConfigModelBuilder<Model> { - public ModelBuilder() { + ModelBuilder() { super(Model.class); } @@ -61,7 +60,7 @@ public class LegacyConfigModelBuilderTest { @Override public List<ConfigModelId> handlesElements() { - return Arrays.asList(ConfigModelId.fromName("foo")); + return List.of(ConfigModelId.fromName("foo")); } } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/VespaDomBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/VespaDomBuilderTest.java index 09b7acafefd..b8d9ad59ae0 100755 --- a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/VespaDomBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/VespaDomBuilderTest.java @@ -25,7 +25,7 @@ import static org.junit.Assert.assertThat; */ public class VespaDomBuilderTest { - final static String hosts = "<?xml version=\"1.0\" encoding=\"utf-8\" ?>" + + private final static String hosts = "<?xml version=\"1.0\" encoding=\"utf-8\" ?>" + "<hosts>" + " <host name=\"localhost\">" + " <alias>node1</alias>" + @@ -33,9 +33,9 @@ public class VespaDomBuilderTest { " </host>" + "</hosts>"; - final static String services = "<?xml version=\"1.0\" encoding=\"utf-8\" ?>" + + private final static String services = "<?xml version=\"1.0\" encoding=\"utf-8\" ?>" + "<services>" + - " <config name=\"standard\">" + + " <config name=\"a.standard\">" + " <basicStruct>" + " <stringVal>default</stringVal>" + " </basicStruct>" + @@ -45,7 +45,7 @@ public class VespaDomBuilderTest { " <adminserver hostalias=\"node1\" />" + " </admin>" + " <container version=\"1.0\">" + - " <config name=\"standard\">" + + " <config name=\"a.standard\">" + " <basicStruct>" + " <stringVal>qrservers</stringVal>" + " </basicStruct>" + @@ -56,19 +56,7 @@ public class VespaDomBuilderTest { " </container>\n" + "</services>"; - final static String servicesWithNamespace = "<?xml version=\"1.0\" encoding=\"utf-8\" ?>" + - "<services>" + - " <config name=\"testnamespace\" namespace=\"foo\">" + - " <basicStruct>" + - " <stringVal>default</stringVal>" + - " </basicStruct>" + - " </config> " + - " <admin version=\"2.0\">" + - " <adminserver hostalias=\"node1\" />" + - " </admin>" + - "</services>"; - - final static String servicesWithNamespace2 = "<?xml version=\"1.0\" encoding=\"utf-8\" ?>" + + private final static String servicesWithNamespace = "<?xml version=\"1.0\" encoding=\"utf-8\" ?>" + "<services>" + " <config name=\"foo.testnamespace\">" + " <basicStruct>" + @@ -93,16 +81,6 @@ public class VespaDomBuilderTest { " \"stringVal\": \"default\"\n" + " }\n" + "}\n"); - - model = createModel(hosts, servicesWithNamespace2); - - builder = new GenericConfig.GenericConfigBuilder(new ConfigDefinitionKey("testnamespace", "foo"), new ConfigPayloadBuilder()); - model.getConfig(builder, "admin"); - assertEquals(builder.getPayload().toString(), "{\n" + - " \"basicStruct\": {\n" + - " \"stringVal\": \"default\"\n" + - " }\n" + - "}\n"); } @Test diff --git a/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTestCase.java index 333bb3728f6..ac81c1a7b7e 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTestCase.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTestCase.java @@ -66,13 +66,9 @@ public class VespaModelTestCase { "</host>" + "</hosts>"; - public static VespaModel getVespaModel(String configPath) { - return getVespaModel(configPath, true); - } - - public static VespaModel getVespaModel(String configPath, boolean validateXml) { + private static VespaModel getVespaModel(String configPath) { VespaModelCreatorWithFilePkg creator = new VespaModelCreatorWithFilePkg(configPath); - return creator.create(validateXml); + return creator.create(true); } // Debugging @@ -232,7 +228,7 @@ public class VespaModelTestCase { public void testDeployLogger() throws IOException, SAXException { final String services = "<?xml version=\"1.0\" encoding=\"utf-8\" ?>" + "<services version=\"1.0\">" + - "<config name=\"unknsownfoo\">" + + "<config name=\"bar.unknsownfoo\">" + "<logserver><host>foo</host></logserver>" + "</config>" + "<admin version=\"2.0\">" + diff --git a/configgen/bin/make-config.pl b/configgen/bin/make-config.pl deleted file mode 100755 index 13017a679e3..00000000000 --- a/configgen/bin/make-config.pl +++ /dev/null @@ -1,151 +0,0 @@ -#!/usr/bin/env perl -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -# -# This script transforms a .def file into a .h and .cpp file for config -# -# TODO: Remove this script and use the java code directly. BTW, why -# does this script have the same limitations as the old make-config.pl -# in that the .def-file must reside in the root directory? The java -# code supports reading the .def-file from any directory. This script -# should have the same input parameters as the java code, and just -# map them directly to the java system properties - -# BEGIN perl environment bootstrap section -# Do not edit between here and END as this section should stay identical in all scripts - -use File::Basename; -use File::Path; - -sub findpath { - my $myfullname = ${0}; - my($myname, $mypath) = fileparse($myfullname); - - return $mypath if ( $mypath && -d $mypath ); - $mypath=`pwd`; - - my $pwdfullname = $mypath . "/" . $myname; - return $mypath if ( -f $pwdfullname ); - return 0; -} - -# Returns the argument path if it seems to point to VESPA_HOME, 0 otherwise -sub is_vespa_home { - my($VESPA_HOME) = shift; - my $COMMON_ENV="libexec/vespa/common-env.sh"; - if ( $VESPA_HOME && -d $VESPA_HOME ) { - my $common_env = $VESPA_HOME . "/" . $COMMON_ENV; - return $VESPA_HOME if -f $common_env; - } - return 0; -} - -# Returns the home of Vespa, or dies if it cannot -sub findhome { - # Try the VESPA_HOME env variable - return $ENV{'VESPA_HOME'} if is_vespa_home($ENV{'VESPA_HOME'}); - if ( $ENV{'VESPA_HOME'} ) { # was set, but not correctly - die "FATAL: bad VESPA_HOME value '" . $ENV{'VESPA_HOME'} . "'\n"; - } - - # Try the ROOT env variable - $ROOT = $ENV{'ROOT'}; - return $ROOT if is_vespa_home($ROOT); - - # Try the script location or current dir - my $mypath = findpath(); - if ($mypath) { - while ( $mypath =~ s|/[^/]*$|| ) { - return $mypath if is_vespa_home($mypath); - } - } - die "FATAL: Missing VESPA_HOME environment variable\n"; -} - -sub findhost { - my $tmp = $ENV{'VESPA_HOSTNAME'}; - my $bin = $ENV{'VESPA_HOME'} . "/bin"; - if (!defined $tmp) { - $tmp = `${bin}/vespa-detect-hostname || hostname -f || hostname || echo "localhost"`; - chomp $tmp; - } - my $validate = "${bin}/vespa-validate-hostname"; - if (-f "${validate}") { - system("${validate} $tmp"); - ( $? == 0 ) or die "Could not validate hostname\n"; - } - return $tmp; -} - -BEGIN { - my $tmp = findhome(); - $ENV{'VESPA_HOME'} = $tmp; - $tmp = findhost(); - $ENV{'VESPA_HOSTNAME'} = $tmp; -} -my $VESPA_HOME = $ENV{'VESPA_HOME'}; - -# END perl environment bootstrap section - -use lib $ENV{'VESPA_HOME'} . '/lib/perl5/site_perl'; -use lib $ENV{'VESPA_HOME'} . '/lib64/perl5/site_perl'; -use Yahoo::Vespa::Defaults; -readConfFile(); - -require 5.006_001; -use strict; -use warnings; - -use Cwd 'abs_path'; - -# Now this uses the new java codegen library. But the script still exist to -# map be able to call java the right way, setting the necessary properties - -my ($root, $def) = @ARGV; - -if (!defined $root || !defined $def) { - print "Usage make-config.pl <source root dir> <def file>\n"; - exit(1); -} - -#print "Root: $root\n" -# . "Def: $def\n"; - -my $subdir = &getRelativePath($root, &getPath($def)); - -my $cmd = "java" - . " -Dconfig.spec=$def" - . " -Dconfig.dest=$root" - . " -Dconfig.lang=cppng" - . " -Dconfig.requireNamespace=false" - . " -Dconfig.subdir=$subdir" - . " -Dconfig.dumpTree=false" - . " -Xms64m -Xmx64m" - . " -jar $VESPA_HOME/lib/jars/configgen.jar"; - -print "Generating config: $cmd\n"; -exec($cmd); - -exit(0); # Will never be called due to exec above, but just to indicate end - -sub getRelativePath { - my ($from, $to) = @_; - - $from = abs_path($from); - $to = abs_path($to); - - # Escape $from so it can contain regex special characters in path - $from =~ s/([\+\*\(\)\{\}\.\?\[\]\$\&])/\\$1/g; - - $to =~ /^$from\/(.*)$/ - or die "The def file must be contained within the root"; - return $1; -} - -sub getPath { - my $file = $_[0]; - if ($file =~ /^(.*)\/[^\/]*$/) { - return $1; - } else { - return "."; - } -} diff --git a/configgen/bin/make-configold.pl b/configgen/bin/make-configold.pl deleted file mode 100755 index d299d09d3be..00000000000 --- a/configgen/bin/make-configold.pl +++ /dev/null @@ -1,150 +0,0 @@ -#!/usr/bin/env perl -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -# -# This script transforms a .def file into a .h and .cpp file for config -# -# TODO: Remove this script and use the java code directly. BTW, why -# does this script have the same limitations as the old make-config.pl -# in that the .def-file must reside in the root directory? The java -# code supports reading the .def-file from any directory. This script -# should have the same input parameters as the java code, and just -# map them directly to the java system properties - -# BEGIN perl environment bootstrap section -# Do not edit between here and END as this section should stay identical in all scripts - -use File::Basename; -use File::Path; - -sub findpath { - my $myfullname = ${0}; - my($myname, $mypath) = fileparse($myfullname); - - return $mypath if ( $mypath && -d $mypath ); - $mypath=`pwd`; - - my $pwdfullname = $mypath . "/" . $myname; - return $mypath if ( -f $pwdfullname ); - return 0; -} - -# Returns the argument path if it seems to point to VESPA_HOME, 0 otherwise -sub is_vespa_home { - my($VESPA_HOME) = shift; - my $COMMON_ENV="libexec/vespa/common-env.sh"; - if ( $VESPA_HOME && -d $VESPA_HOME ) { - my $common_env = $VESPA_HOME . "/" . $COMMON_ENV; - return $VESPA_HOME if -f $common_env; - } - return 0; -} - -# Returns the home of Vespa, or dies if it cannot -sub findhome { - # Try the VESPA_HOME env variable - return $ENV{'VESPA_HOME'} if is_vespa_home($ENV{'VESPA_HOME'}); - if ( $ENV{'VESPA_HOME'} ) { # was set, but not correctly - die "FATAL: bad VESPA_HOME value '" . $ENV{'VESPA_HOME'} . "'\n"; - } - - # Try the ROOT env variable - $ROOT = $ENV{'ROOT'}; - return $ROOT if is_vespa_home($ROOT); - - # Try the script location or current dir - my $mypath = findpath(); - if ($mypath) { - while ( $mypath =~ s|/[^/]*$|| ) { - return $mypath if is_vespa_home($mypath); - } - } - die "FATAL: Missing VESPA_HOME environment variable\n"; -} - -sub findhost { - my $tmp = $ENV{'VESPA_HOSTNAME'}; - my $bin = $ENV{'VESPA_HOME'} . "/bin"; - if (!defined $tmp) { - $tmp = `${bin}/vespa-detect-hostname || hostname -f || hostname || echo "localhost"`; - chomp $tmp; - } - my $validate = "${bin}/vespa-validate-hostname"; - if (-f "${validate}") { - system("${validate} $tmp"); - ( $? == 0 ) or die "Could not validate hostname\n"; - } - return $tmp; -} - -BEGIN { - my $tmp = findhome(); - $ENV{'VESPA_HOME'} = $tmp; - $tmp = findhost(); - $ENV{'VESPA_HOSTNAME'} = $tmp; -} -my $VESPA_HOME = $ENV{'VESPA_HOME'}; - -# END perl environment bootstrap section - -use lib $ENV{'VESPA_HOME'} . '/lib/perl5/site_perl'; -use lib $ENV{'VESPA_HOME'} . '/lib64/perl5/site_perl'; -use Yahoo::Vespa::Defaults; -readConfFile(); - -require 5.006_001; -use strict; -use warnings; - -use Cwd 'abs_path'; - -# Now this uses the new java codegen library. But the script still exist to -# map be able to call java the right way, setting the necessary properties - -my ($root, $def) = @ARGV; - -if (!defined $root || !defined $def) { - print "Usage make-config.pl <source root dir> <def file>\n"; - exit(1); -} - -#print "Root: $root\n" -# . "Def: $def\n"; - -my $subdir = &getRelativePath($root, &getPath($def)); - -my $cmd = "java" - . " -Dconfig.spec=$def" - . " -Dconfig.dest=$root" - . " -Dconfig.lang=cpp" - . " -Dconfig.subdir=$subdir" - . " -Dconfig.dumpTree=false" - . " -Xms64m -Xmx64m" - . " -jar $VESPA_HOME/lib/jars/configgen.jar"; - -print "Generating config: $cmd\n"; -exec($cmd); - -exit(0); # Will never be called due to exec above, but just to indicate end - -sub getRelativePath { - my ($from, $to) = @_; - - $from = abs_path($from); - $to = abs_path($to); - - # Escape $from so it can contain regex special characters in path - $from =~ s/([\+\*\(\)\{\}\.\?\[\]\$\&])/\\$1/g; - - $to =~ /^$from\/(.*)$/ - or die "The def file must be contained within the root"; - return $1; -} - -sub getPath { - my $file = $_[0]; - if ($file =~ /^(.*)\/[^\/]*$/) { - return $1; - } else { - return "."; - } -} diff --git a/configgen/src/main/resources/make-config-preproc.pl b/configgen/src/main/resources/make-config-preproc.pl deleted file mode 100755 index a3432957e04..00000000000 --- a/configgen/src/main/resources/make-config-preproc.pl +++ /dev/null @@ -1,952 +0,0 @@ -#!/usr/bin/perl -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -# This is the config pre-processor. -# It handles import statements, and does syntax checking etc. -# The idea is that it will be called directly from the script -# that does the code generation. -# -# Errors and warnings are printed in "next-error" compatible ways -# for emacs etc. -# -# Indented like this: -# (cperl-set-style "Whitesmith") -# (setq cperl-continued-brace-offset -4) - -require 5.006_001; -use strict; -use warnings; -use Digest::MD5; - -use Math::BigInt; -use Math::BigFloat; - -die "Usage: $0 <def-file>" unless $#ARGV == 0; - -my $defname = $ARGV[0]; - -my $md5 = Digest::MD5->new; - -my @c_keywords = - ("asm", "auto", "bool", "break", "case", "catch", - "char", "class", "const", "const_cast", "continue", "default", - "delete", "do", "double", "dynamic_cast", "else", "enum", "explicit", - "export", "extern", "false", "float", "for", "friend", "goto", "if", - "inline", "int", "long", "mutable", "namespace", "new", "operator", - "private", "protected", "public", "register", "reinterpret_cast", - "return", "short", "signed", "sizeof", "static", "static_cast", - "struct", "switch", "template", "this", "throw", "true", "try", - "typedef", "typeid", "typename", "union", "unsigned", - "using", "virtual", "void", "volatile", "wchar_t", "while", "and", "bitor", - "not", "or", "xor", "and_eq", "compl", "not_eq", "or_eq", "xor_eq", - "bitand"); - - -my @java_keywords = - ("abstract", "boolean", "break", "byte", "case", - "catch", "char", "class","continue", "default", "do", "double", - "else", "extends","false", "final", "finally", "float", "for", - "if","implements", "import", "instanceof", "int", "interface", - "long","native", "new", "null", "package", "private", - "protected","public", "return", "short", "static", - "strictfp","super","switch", "synchronized", "this", - "throw","throws","transient", "true", "try", "void", - "volatile","while", "byvalue", "cast", "const", "future", - "generic","goto", "inner", "operator", "outer", "rest", "var"); - -my %reserved_words; - -foreach my $word (@c_keywords) { - $reserved_words{$word} = "C"; -} - -foreach my $word (@java_keywords) { - my $x = $reserved_words{$word}; - if (defined($x)) { - $x = "$x, Java"; - } else { - $x = "Java"; - } - $reserved_words{$word} = $x; -} - -my $MIN_INT = -0x80000000; -my $MAX_INT = 0x7fffffff; -my $MIN_DOUBLE = -1e308; -my $MAX_DOUBLE = 1e308; - - -sub do_file { - my ($file, $prefix, $strip) = @_; - - local *FH; - open FH, "< $file" or die "Cannot open $file: $!\n"; - - local *COPY; - my $dir = $ENV{"VESPA_CONFIG_DEF_DIR"}; - my $copy; - my $file_version; - if (defined($dir)) { - $copy = $file; - $copy =~ s=.*/==; - $copy = "$dir/$copy"; - open COPY, ">$copy.new" or die "Cannot open file $copy.new: $!\n"; - } - - # Read line by line. - # 1. Strip away comments and trailing blanks - # 2. Report any errors - # 3. Handle import statements, disallow multi-level imports - # 4. Print everyting to stdout - - my $linenr = 0; - my $written_lines = 0; - my $quoted_strip = quotemeta($strip); - my $seen_version = 0; - - while (<FH>) { - print COPY $_ if $copy; - ++$linenr; - my $line = $_; - chomp $line; - - # Don't process comments or add them to md5 checksum, but print them - # such that codegen can include comments - if ($line =~ /^\s*#/) { - print "$line\n"; - next; - } - - # Strip away comments that are not at start of line - $line = &strip_trailing_comment($line, $linenr) - if ($line =~ m=[\\\#]=); - - if ($line eq "::error::") { - return -1; - } - - # Skip lines that are only whitespace - next if $line =~ m=^\s*$=; - - # Get rid of trailing whitespace - $line =~ s=\s+$==; - - if (!$seen_version) { - if ($line =~ m!^version=([a-zA-Z0-9][-a-zA-Z0-9_/?]*)!) { - $file_version = $1; - $seen_version = 1; - if ($prefix) { - print "$prefix imported $file"; - print ":$strip" if $strip; - print " "; - } - print "$line\n"; - next; - } else { - print STDERR "$file:$linenr: error: Definition file does not " - . "start with a valid version= identifier!\n"; - return -1; - } - } - - if ($strip) { - next unless $line =~ m=^${quoted_strip}[. \t]=; - } - - if (&check_syntax($line, $linenr, $file) == -1) { - return -1; - } - - # Handle import statements - my ($name, $type, $remains, $junk) = split(/\s+/, $line, 4); - if ($type eq "import") { - if ($strip || $prefix) { - my $col = index($line, $type, length("$name ")); - print STDERR "$file:$linenr:$col: error: Multi-level " - . "imports are disallowed.\n"; - return -1; - } - if ($junk) { - my $col = index($line, $junk, length("$name $type $remains")) - + 1; - print STDERR "$file:$linenr:$col: error: Junk after import " - . "target \"$remains\": \"$junk\"\n"; - return -1; - } - my ($impfile, $var) = split(/:/, $remains, 2); - $var = "" unless $var; # Make it defined. - - # Make sure only arrays can include arrays: - if ($name =~ m=\[\]$= && (!$var || $var !~ m=\[\]$=)) { - print STDERR "$file:$linenr: error: Array cannot import " - . "non-array in: $line\n"; - return -1; - } elsif ($name !~ m=\[\]$= && ($var && $var =~ m=\[\]$=)) { - print STDERR "$file:$linenr: error: Non-array cannot import " - . "array in: $line\n"; - return -1; - } - - local *X; - unless (open(X, "< $impfile")) { - my $col = index($line, $remains, length("$name $type")) + 1; - print STDERR "$file:$linenr:$col: error: Cannot open " - . "\"$impfile\": $!\n"; - return -1; - } - close X; - my $imported_lines = &do_file("$impfile", "$name", "$var"); - if ($imported_lines == -1) { - my $col = index($line, $remains, length("$name $type")) + 1; - print STDERR "$file:$linenr:$col: error: Imported from here " - . "as: $line\n"; - return -1; - } elsif ($imported_lines == 0) { - my $col = index($line, $remains, length("$name $type")) + 1; - print STDERR "$file:$linenr:$col: error: Import target " - . "\"$var\" not found in \"$impfile\"\n"; - return -1; - } - $written_lines += $imported_lines; - } else { - ++$written_lines; - if ($strip) { - $line =~ s=^${quoted_strip}=${prefix}= - } elsif ($prefix) { - $line = $prefix . "." . $line; - } - - if (&check_name_sanity($line, $linenr, $file) == -1 - || &check_enum_sanity($line, $linenr, $file) == -1) { - return -1; - } - - $line = &normalize_line($line, $linenr); - if ($line eq "::error::") { - return -1; - } - print $line . "\n"; - } - # Add this line to the md5 checksum - $md5->add("$line\n") unless $prefix; - } - - print "md5=" . $md5->hexdigest . "\n" unless $prefix; - close FH; - if ($copy) { - close COPY; - # We have made a copy. It needs a new name.. - my $new_name = $copy; - $new_name =~ s=\.def==; - $new_name .= ".${file_version}.def"; - if (-f $new_name) { - system "cmp $copy.new $new_name 2>/dev/null" and die "$file:1: error: Definition file $file differs from ${new_name}!\n"; - unlink("$copy.new"); - } else { - rename("$copy.new", "$new_name") or die "Rename $copy.new -> $new_name failed: $!\n"; - } - } - return $written_lines; -} - -sub normalize_enum { - my($x, $linenr, $colnr) = @_; - my $len = length($x); - my $char = ''; - my $output = '{ '; - my $index; - my %enum = (); - my $current_variable = ''; - for ($index = $colnr + 1; $index < $len; ++$index) { - $char = substr($x, $index, 1); - if ($char eq '}') { - if (length($current_variable) < 2) { - print STDERR "$defname:$linenr:$index: error: ". - " variable must be at least two characters: $x\n" ; - return ('', 0); - } elsif ($enum{$current_variable}) { - print STDERR "$defname:$linenr:$index: error: ". - " enum variable declared twice: $x\n" ; - return ('', 0); - } elsif (!%enum && !$current_variable) { - print STDERR "$defname:$linenr:$index: error: ". - " enum cannot be empty: $x\n" ; - return ('', 0); - } - return ($output.$current_variable." } ", $index); - } elsif ($char eq ',') { - if (length($current_variable) < 2) { - print STDERR "$defname:$linenr:$index: error: ". - " variable must be at least two characters: $x\n" ; - return ('', 0); - } elsif ($enum{$current_variable}) { - print STDERR "$defname:$linenr:$index: error: ". - " enum variable declared twice: $x\n" ; - return ('', 0); - } - $enum{$current_variable} = 1; - $output .= "$current_variable, "; - $current_variable = ''; - } elsif ($char =~ m=[A-Z]=) { - $current_variable .= $char; - } elsif ($char =~ m=[0-9_]= && $current_variable) { - $current_variable .= $char; - } elsif ($char =~ m=\s=) { - if ($current_variable && !($x =~ /^.{$index}\s*[,\}]/)) { - print STDERR "$defname:$linenr:$index: error: ". - "expected ',' or '}': $x\n" ; - return ("", 0); - } else { - # skip whitespace - } - } else { - print STDERR "<$char> <$current_variable>\n"; - - print STDERR "$defname:$linenr:$index: error: ". - "Enum must match [A-Z][A-Z0-9_]+: $x\n"; - } - } - return ($output, $index); -} - -{ package Range; - - $Range::DOUBLE_RANGE = - new Range("a double range=[$MIN_DOUBLE,$MAX_DOUBLE] ",0,14); - $Range::INT_RANGE = new Range("a int range=[$MIN_INT,$MAX_INT] ",0,11); - - - sub in_range { - my($self, $value) = @_; - - if ($value =~ s/KB$//) { - $value *= 1024; - } elsif ($value =~ s/MB$//) { - $value *= (1024 * 1024); - } elsif ($value =~ s/GB$//) { - $value *= (1024*1024*1024); - } elsif ($value =~ s/k$//) { - $value *= 1000; - } elsif ($value =~ s/M$//) { - $value *= 1_000_000; - } elsif ($value =~ s/G$//) { - $value *= 1_000_000_000; - } elsif ($value =~ m=^0[xX]=) { - $value = hex($value); - } - - if ($self->{start_bracket} eq '(' ) { - return 0 if $value <= $self->{min}; - } elsif ($self->{start_bracket} eq '[' ) { - return 0 if $value < $self->{min}; - } else { - print STDERR "Illegal start_bracket '$self->{start_bracket}'\n"; - return undef; - } - if ($self->{end_bracket} eq ')' ) { - return 0 if $value >= $self->{max}; - } elsif ($self->{end_bracket} eq ']' ) { - return 0 if $value > $self->{max}; - } else { - print STDERR "Illegal end_bracket '$self->{start_bracket}'\n"; - return undef; - } - return 1; - } - - - sub new { - my($class, $x, $linenr, $colnr) = @_; - my $len = length($x); - my $self = {}; - bless($self, $class); - $self->{min_value} = ''; - my $index; - for ($index = $colnr + 1; $index < $len; ++$index) { - my $char = substr($x, $index, 1); - if (($char eq '(' || $char eq '[') && !$self->{start_bracket}) { - $self->{start_bracket} = $char; - } elsif (($char eq ')' || $char eq ']') && !$self->{end_bracket}) { - $self->{end_bracket} = $char; - last; - } elsif ($char =~ m=\s=) { - #ignore whitespace - } elsif ($char eq ',' && !defined($self->{max_value})) { - $self->{max_value} = ''; - } elsif ($char =~ m=[\d\.\+eE-]= ) { - (defined($self->{max_value}) - ? $self->{max_value} : $self->{min_value}) .= $char; - } else { - print STDERR "$defname:$linenr:$index: error: ". - " syntax error: $x\n" ; - return undef; - } - } - if ($self->{min_value} eq '' && $self->{max_value} eq '') { - print STDERR "$defname:$linenr:$colnr: error: ". - " range cannot be unbounded in both ends: $x\n" ; - return undef; - } - unless ($self->{start_bracket} && $self->{end_bracket}) { - print STDERR "$defname:$linenr:$colnr: error: ". - " missing bracket: $x\n" ; - return undef; - } - - - my @arr = split(/\s+/, $x, 3); - if ($arr[1] eq 'int') { - $self->{min} = Math::BigInt->new - ($self->{min_value} eq '' ? $MIN_INT : $self->{min_value}); - unless (defined($self->{min}) && $self->{min} ne 'NaN') { - print STDERR "$defname:$linenr:$colnr: error: ". - " parse error $self->{min_value}: $x\n" ; - return undef; - } - my $min_val = - $self->{min} + ($self->{start_bracket} eq '('? 1 : 0); - - $self->{max} = Math::BigInt->new - ($self->{max_value} eq '' ? $MAX_INT : $self->{max_value}); - unless (defined($self->{max}) && $self->{max} ne 'NaN') { - print STDERR "$defname:$linenr:$colnr: error: ". - " parse error $self->{max_value}: $x\n" ; - return undef; - } - my $max_val = - $self->{max} - ($self->{end_bracket} eq ')'? 1 : 0); - - if ($min_val < $MIN_INT ) { - print STDERR "$defname:$linenr:$colnr: error: ". - " start of interval less than MIN_INT: $x\n" ; - return undef; - } - if ($max_val > $MAX_INT) { - print STDERR "$self->{max} - 1 > $MAX_INT\n"; - print STDERR "$defname:$linenr:$colnr: error: ". - " end of interval greater than MAX_INT: $x\n" ; - return undef; - } - if ($max_val < $min_val) { - print STDERR "$defname:$linenr:$colnr: error: ". - " illegal range: $x\n" ; - return undef; - } - $self->{string} = - "$self->{start_bracket}$self->{min},$self->{max}$self->{end_bracket}"; - $self->{string} =~ s/\+//g; - $self->{index} = $index; - return $self; - } elsif ($arr[1] eq 'double') { - $self->{min} = Math::BigFloat->new - ($self->{min_value} eq '' ? $MIN_DOUBLE : $self->{min_value}); - unless (defined($self->{min}) && $self->{min} ne 'NaN') { - print STDERR "$defname:$linenr:$colnr: error: ". - " parse error $self->{min_value}: $x\n" ; - return undef; - } - $self->{max} = Math::BigFloat->new - ($self->{max_value} eq '' ? $MAX_DOUBLE : $self->{max_value}); - unless (defined($self->{max}) && $self->{max} ne 'NaN') { - print STDERR "$defname:$linenr:$colnr: error: ". - " parse error $self->{max_value}: $x\n" ; - return undef; - } - if ($self->{min} < $MIN_DOUBLE) { - print STDERR "$defname:$linenr:$colnr: error: ". - " start of interval less than MIN_DOUBLE: $x\n" ; - return undef; - } - if ($self->{max} > $MAX_DOUBLE) { - print STDERR "$defname:$linenr:$colnr: error: ". - " start of interval greater than MAX_DOUBLE: $x\n" ; - return undef; - } - if ($self->{max} < $self->{min}) { - print STDERR "$defname:$linenr:$colnr: error: ". - " illegal range: $x\n" ; - return undef; - } - if (($self->{start_bracket} eq '(' || $self->{end_bracket} eq ')') - && ($self->{min_value} + $self->{min_value} - >= $self->{min_value} + $self->{max_value}) - && ($self->{max_value} + $self->{max_value} - <= $self->{min_value} + $self->{max_value})) { - print STDERR "$defname:$linenr:$colnr: error: ". - " illegal range: $x\n" ; - return undef; - } - $self->{string} = $self->{start_bracket}.$self->{min}->fnorm. - ','.$self->{max}->fnorm.$self->{end_bracket}; - $self->{string} =~ s/\+//g; - $self->{index} = $index; - return $self; - } else { - print STDERR "$defname:$linenr:$colnr: error: ". - " range-option works only for type 'int' and 'double': $x\n" ; - return undef; - } - print STDERR "$defname:$linenr:$colnr: error: ". - " script error: $x\n" ; - return undef; - - } - -} - - - - -sub strip_trailing_comment { - my ($x, $linenr) = @_; - - my $index = 0; - my $len = length($x); - my $in_quotes = 0; - - # ### Support both " and ' quotes maybe? - - for ($index = 0; $index < $len; ++$index) { - if (substr($x, $index, 1) eq "\\") { - ++$index; - next; - } - if (substr($x, $index, 1) eq "\"") { - $in_quotes ^= 1; - } - if ($in_quotes == 0 && substr($x, $index, 1) eq "#") { - if (!(substr($x, $index - 1, 1) =~ m=\s=)) { - my $col = $index + 1; - print STDERR "$defname:$linenr:$col: warning: No whitespace " - . "before comment in line: $x\n"; - } - print substr($x, $index). "\n"; - $x = substr($x, 0, $index); - last; - } - } - if ($index > $len) { - print STDERR "$defname:$linenr:$len: error: syntax error, line " - . "ends with \\: \"$x\"\n"; - return "::error::"; - } - - return $x; -} - -sub normalize_line { - my ($x, $linenr) = @_; - - my $index = 0; - my $len = length($x); - my $in_quotes = 0; - my $char = ''; - my $output = ''; - my %hash = (); - - my @arr = split(/\s+/, $x, 3); - $hash{type} = $arr[1]; - - for ($index = 0; $index < length($x); ++$index) { - $char = substr($x, $index, 1); - if ($char eq "\\") { - $output .= substr($x, $index, 2); - ++$index; - next; - } - if ($char eq "\"") { - $in_quotes ^= 1; - $output .= $char; - next; - } - my $ends_with_whitespace = ($output =~ m= $=); - - if ($in_quotes == 0) { - if ($char =~ m=\s=) { - #delete multiple spaces - if (!$ends_with_whitespace) { # && ($output =~ !m=\=$=)) { - $output .= ' '; - } - } elsif ($char eq '{') { - my($enum, $i) = &normalize_enum($x, $linenr, $index); - return "::error::" unless $i; - $index = $i; - $output .= ($ends_with_whitespace) ? $enum : " $enum "; - } elsif ($char eq ',') { - chop $output if ($ends_with_whitespace); - $output .= ','; - } elsif ($char eq '=') { - chop $output if ($ends_with_whitespace); - $output .= '='; - if ($output =~ /range=$/) { - $hash{range} = - new Range($x, $linenr, $index); - return "::error::" unless $hash{range}; - $index = $hash{range}->{index}; - $output .= $hash{range}->{string}." "; - } - if ($output =~ /default=$/ - && ($hash{type} eq 'int' || $hash{type} eq 'double')) { - $x =~ /^.{$index}=\s*(\S+)/; - $hash{default} = $1; - if ($hash{type} eq 'int' && - !$Range::INT_RANGE->in_range($hash{default})) { - print STDERR "$defname:$linenr:$index: error: ". - "Default not in range: $x\n"; - return "::error::"; - } - if ($hash{type} eq 'double' && - !$Range::DOUBLE_RANGE->in_range($hash{default})) { - print STDERR "$defname:$linenr:$index: error: ". - "Default not in range: $x\n"; - return "::error::"; - } - } - if (defined($hash{default}) && $hash{range}) { - unless ($hash{range}->in_range($hash{default})) { - print STDERR "$defname:$linenr:$index: error: ". - "Default not in range: $x\n"; - return "::error::"; - } - } - } else { - $output .= $char; - } - } else { - $output .= $char; - } - } - if ($index > $len) { - print STDERR "$defname:$linenr:$len: error: syntax error, line " - . "ends with \\: \"$x\"\n"; - return "::error::"; - } - chop $output if $output =~ m/ $/; - return $output; -} - -my %used_enum; -sub check_enum_sanity { - my ($line, $linenr, $file) = @_; - - my ($name, $type, $rest) = split(/\s+/, $line, 3); - return 0 unless ($type eq "enum"); - - $name =~ /(.*)\./; - my $prefix = $1; - $prefix = "" unless defined $prefix; # Make top level prefix - $used_enum{"$prefix"} = $used_enum{"$prefix"} || {}; - $rest = "" unless defined $rest; - $rest =~ /\{\s*(.*?)\}/; - my @values = split(/[,\s]+/, $1); - foreach my $value (@values) { - if ($used_enum{"$prefix"}->{$value}) { - print STDERR - "$file:$linenr: error: Name \"$value\" is already defined\n"; - my $prevdef = $used_enum{"$prefix"}->{$value}; - print STDERR "$prevdef: error: At this point\n"; - return -1; - } else { - $used_enum{"$prefix"}->{$value} = "$file:$linenr"; - } - } - return 0; -} - - -my %used_name; -my %used_component; -my %banned_prefixes; -my $cns_prev_name; -sub check_name_sanity { - my ($line, $linenr, $file) = @_; - my ($name, $junk) = split(/\s+/, $line, 2); - - my $plain_name = $name; - $plain_name =~ s=\[\]$==; - - # See if the name is already used. - if ($used_name{"$plain_name"}) { - print STDERR - "$file:$linenr: error: Name \"$name\" is already defined\n"; - my $prevdef = $used_name{$name}; - print STDERR "$prevdef: error: At this point\n"; - return -1; - } else { - $used_name{$name} = "$file:$linenr"; - } - - # Test for bans - my $banned = "${name}."; - do { - my $err = $banned_prefixes{$banned}; - if (defined($err)) { - print STDERR "$file:$linenr: error: The prefix \"$banned\" is illegal here\n"; - print STDERR "$err\n"; - return -1; - } - } while (($banned =~ s=[.][^.]+[.]$=.=)); - - # Add any new bans generated by this line - $banned_prefixes{"${name}."} = "$file:$linenr: error: \"${name}\" cannot " - . "be both a struct and a non-struct!"; - if ($cns_prev_name) { - my $prev = $cns_prev_name; - my $oldprev = $prev; - while (($prev =~ s=[.][^.]+[.]?$=.=)) { - if (substr($name, 0, length($prev)) eq $prev) { - $banned_prefixes{"$oldprev"} = "$file:" . ($linenr - 1) - . ": error: Last possible line is after this"; - last; - } - $oldprev = $prev; - } - } - $cns_prev_name = $name; - - # See if any of the components previously have a different "arrayness" - my $part_name = $name; - while (($part_name =~ s=[.][^.]+$==)) { - my $clashing_name = $part_name; - if ($part_name =~ m=\[\]$=) { - $clashing_name =~ s=\[\]$==; - } else { - $clashing_name .= "[]"; - } - my $clashline = $used_component{"$clashing_name"}; - if (defined $clashline) { - print STDERR "$file:$linenr: error: \"$clashing_name\" cannot be both array and non-array\n"; - print STDERR "$clashline: error: Previously defined here\n"; - return -1; - } elsif (!$used_component{"$part_name"}) { - $used_component{"$part_name"} = "$file:$linenr"; - } - } - return 0; -} - -# These are all the allowed types/commands -my %types = ( "int" => \&check_int, - "double" => \&check_double, - "string" => \&check_string, - "reference" => \&check_reference, - "enum" => \&check_enum, - "bool" => \&check_bool, - "properties" => \&check_properties, - "import" => \&check_import ); - -sub check_syntax { - my ($line, $linenr, $file) = @_; - - my $col = 0; - my $llen = length($line); - - # Step 1. Sanity check the name. - my $atstart = 1; - my $array_ok = 1; - - for ($col = 0; $col < $llen; ++$col) { - my $c = substr($line, $col, 1); - if ($atstart) { - if ($c !~ m=[a-zA-Z]=) { - print STDERR "$file:$linenr:$col: error: Non-alphabetic start " - . "of variable name in $line\n"; - return -1; - } - $atstart = 0; - } else { - if ($c =~ m=[a-zA-Z0-9_]=) { - 0; # Do nothing - } elsif ($c eq ".") { - $atstart = 1; - $array_ok = 1; - } elsif ($c eq "[") { - if (!$array_ok) { - ++$col; - print STDERR "$file:$linenr:$col: error: Arrays cannot be " - . "multidimensional in $line\n"; - return -1; - } - ++$col; - $array_ok = 0; - $c = substr($line, $col, 1); - if ($c ne "]") { - ++$col; - print STDERR "$file:$linenr:$col: error: Expected ] to " - . "terminate array definition in $line\n"; - return -1; - } - } elsif ($c =~ m=\s=) { - last; - } else { - ++$col; - print STDERR "$file:$linenr:$col: error: Syntax error, " - . "unexpected character in $line\n"; - return -1; - } - } - } - - my $name = substr($line, 0, $col); - $name =~ s=.*[.]==; - $name =~ s=[[]]$==; - - my $clash = $reserved_words{$name}; - if ($clash) { - $col -= (3 + length($name)); - $col = index($line, $name, $col) + 1; - print STDERR "$file:$linenr:$col: error: $name is a reserved word in: " - . "${clash}\n"; - return -1; - } - - while (substr($line, $col, 1) =~ m=\s=) { - ++$col; - } - - # At this point the name is sane. Next, check the type. - my ($type) = split(/\s/, substr($line, $col)); - - unless (defined $types{$type}) { - ++$col; - print STDERR "$file:$linenr:$col: error: Unknown type/command " - . "\"$type\"\n"; - return -1; - } - $col += length($type); - while (substr($line, $col, 1) =~ m=\s=) { - ++$col; - } - return $types{$type}($col, $line, $linenr, $file); -} - -sub reg_words_check { - my ($col, $line, $linenr, $file, $reg) = @_; - my $remainder = substr($line, $col); - my @options = split(/\s+/, $remainder); - - foreach my $option (@options) { - # Keep track of where we are for error reporting - $col = index($line, $option, $col) + 1; - unless ($option =~ m!${reg}!) { - print STDERR "$file:$linenr:$col: error: Bad option \"$option\" no match for m!${reg}!\n"; - return -1; - } - } - return 0; -} - -sub check_int { - my ($col, $line, $linenr, $file) = @_; - my $num = "(-?\\d+(KB|MB|GB|k|M|G)?|0x[0-9a-fA-F]+)"; # All legal numbers - my $optnum = "(${num})?"; # All legal optional numbers - return ®_words_check($col, $line, $linenr, $file, - "^(" - . "default=${num}" - . "|range=[[(]${optnum},${optnum}"."[])]" - . "|restart" - . ")\$"); -} - -sub check_double { - my ($col, $line, $linenr, $file) = @_; - my $num = "-?(\\d+(\\.\\d*)?|\\.\\d+)([eE][+-]?\\d+)?"; # All legal doubles - my $optnum = "(${num})?"; # Optional doubles - return ®_words_check($col, $line, $linenr, $file, - "^(" - . "default=${num}" - . "|range=[[(]${optnum},${optnum}"."[])]" - . "|restart" - . ")\$"); -} - -sub check_string { - my ($col, $line, $linenr, $file) = @_; - my $opts = substr($line, $col); - - # not entirely correct either for something like \\" - my $def = "default=((\"(\\\"|[^\"])*\")|null)"; - - my $res = "restart"; - my $reg = "^(${def}\\s+${res}|(${def})?|${res}|${res}\\s+${def})\$"; - - unless ($opts =~ m!${reg}!) { - print STDERR "$file:$linenr:$col: error: Bad options \"$opts\", no match for m!${reg}!\n"; - return -1; - } - return 0; -} - -sub check_reference { - my ($col, $line, $linenr, $file) = @_; - my $opts = substr($line, $col); - my $def = "default=((\"(\\\"|[^\"])*\")|null)"; - - unless ($opts eq "" || $opts =~m!${def}!) { - print STDERR "$file:$linenr:$col: error: reference can only " - . "take the 'default' option\n"; - return -1; - } - return 0; -} - - -sub check_enum { - my ($col, $line, $linenr, $file) = @_; - my $ret = ®_words_check($col, $line, $linenr, $file, - "^(" - . "[{},]" - . "|[A-Z][A-Z0-9_]+,?" - . "|default=[A-Z][A-Z0-9_]+" - . "|restart" - . ")\$"); - return -1 if $ret; - $col = index($line, '}', $col) + 1; #move $col to end of enum --> } - while (substr($line, $col, 1) =~ m=[\s\{]=) { - ++$col; - } - return 0 if $col >= length($line); - - - return ®_words_check($col, $line, $linenr, $file, - "^(" - . "default=[A-Z][A-Z0-9_]+" - . "|restart" - . ")\$"); -} - -sub check_bool { - my ($col, $line, $linenr, $file) = @_; - return ®_words_check($col, $line, $linenr, $file, - "^(" - . "default=(true|false)" - . "|restart" - . ")\$"); -} - -sub check_properties { - my ($col, $line, $linenr, $file) = @_; - return ®_words_check($col, $line, $linenr, $file, "^restart\$"); -} - -sub check_import { - my ($col, $line, $linenr, $file) = @_; - my $word = "[a-zA-Z][_a-zA-Z0-9]*"; - my $fnam = "${word}(\\.${word})*"; - my $var = "${word}((\\[\\])?\.${word})*(\\[\\])?"; - return ®_words_check($col, $line, $linenr, $file, - "^${fnam}\\.def:(${var})?\$"); - return 0; -} - - -my $lines = &do_file($defname, "", ""); - -if ($lines == -1) { - die "There were irrecoverable errors in \"$defname\"!\n"; -} -if ($lines == 0) { - die "$defname:1: error: Resulting definition is empty!\n"; -} - -exit 0; diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/resource/ResourceSnapshot.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/resource/ResourceSnapshot.java index 7f7a6b758d5..e3a2781142a 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/resource/ResourceSnapshot.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/resource/ResourceSnapshot.java @@ -1,23 +1,63 @@ // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.api.integration.resource; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeRepositoryNode; + import java.time.Instant; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; /** * @author olaa */ public class ResourceSnapshot { - private final ResourceAllocation resourceAllocation; + private final ApplicationId applicationId; + private final double cpuCores; + private final double memoryGb; + private final double diskGb; private final Instant timestamp; - public ResourceSnapshot(ResourceAllocation resourceAllocation, Instant timestamp) { - this.resourceAllocation = resourceAllocation; + public ResourceSnapshot(ApplicationId applicationId, double cpuCores, double memoryGb, double diskGb, Instant timestamp) { + this.applicationId = applicationId; + this.cpuCores = cpuCores; + this.memoryGb = memoryGb; + this.diskGb = diskGb; this.timestamp = timestamp; } - public ResourceAllocation getResourceAllocation() { - return resourceAllocation; + public static ResourceSnapshot from(List<NodeRepositoryNode> nodes, Instant timestamp) { + Set<ApplicationId> applicationIds = nodes.stream() + .map(n -> ApplicationId.from(n.getOwner().tenant, n.getOwner().application, n.getOwner().instance)) + .collect(Collectors.toSet()); + + if (applicationIds.size() != 1) throw new IllegalArgumentException("List of nodes can only represent one application"); + + return new ResourceSnapshot( + applicationIds.iterator().next(), + nodes.stream().mapToDouble(NodeRepositoryNode::getMinCpuCores).sum(), + nodes.stream().mapToDouble(NodeRepositoryNode::getMinMainMemoryAvailableGb).sum(), + nodes.stream().mapToDouble(NodeRepositoryNode::getMinDiskAvailableGb).sum(), + timestamp + ); + } + + public ApplicationId getApplicationId() { + return applicationId; + } + + public double getCpuCores() { + return cpuCores; + } + + public double getMemoryGb() { + return memoryGb; + } + + public double getDiskGb() { + return diskGb; } public Instant getTimestamp() { diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/resource/ResourceSnapshotConsumer.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/resource/ResourceSnapshotConsumer.java index f7f3eddb482..bb6830770e2 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/resource/ResourceSnapshotConsumer.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/resource/ResourceSnapshotConsumer.java @@ -1,9 +1,7 @@ // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.api.integration.resource; -import com.yahoo.config.provision.ApplicationId; - -import java.util.Map; +import java.util.List; /** * Consumes a snapshot of resourses allocated/used per application. @@ -12,5 +10,7 @@ import java.util.Map; */ public interface ResourceSnapshotConsumer { - public void consume(Map<ApplicationId, ResourceSnapshot> resources); + public void consume(List<ResourceSnapshot> resources); + + public List<ResourceSnapshot> getResourceSnapshots(String tenantName, String applicationName); } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/MockResourceSnapshotConsumer.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/MockResourceSnapshotConsumer.java index d5d7b63e933..2cd3aef1903 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/MockResourceSnapshotConsumer.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/MockResourceSnapshotConsumer.java @@ -1,25 +1,29 @@ // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.api.integration.stubs; -import com.yahoo.config.provision.ApplicationId; import com.yahoo.vespa.hosted.controller.api.integration.resource.ResourceSnapshot; import com.yahoo.vespa.hosted.controller.api.integration.resource.ResourceSnapshotConsumer; -import java.util.Map; +import java.util.List; /** * @author olaa */ public class MockResourceSnapshotConsumer implements ResourceSnapshotConsumer { - private Map<ApplicationId, ResourceSnapshot> resources; + private List<ResourceSnapshot> resources; @Override - public void consume(Map<ApplicationId, ResourceSnapshot> resources){ + public void consume(List<ResourceSnapshot> resources){ this.resources = resources; } - public Map<ApplicationId, ResourceSnapshot> consumedResources() { - return resources; + @Override + public List<ResourceSnapshot> getResourceSnapshots(String tenantName, String applicationName) { + throw new UnsupportedOperationException(); + } + + public List<ResourceSnapshot> consumedResources() { + return this.resources; } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainer.java index c1fa02a554c..41460cd9fdc 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainer.java @@ -10,22 +10,17 @@ import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeReposi import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeOwner; import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeRepositoryNode; import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeState; -import com.yahoo.vespa.hosted.controller.api.integration.resource.ResourceAllocation; import com.yahoo.vespa.hosted.controller.api.integration.resource.ResourceSnapshot; import com.yahoo.vespa.hosted.controller.api.integration.resource.ResourceSnapshotConsumer; import java.time.Clock; import java.time.Duration; -import java.time.Instant; import java.util.Collections; import java.util.List; -import java.util.Map; -import java.util.function.Predicate; import java.util.stream.Collectors; /** * Creates a ResourceSnapshot per application, which is then passed on to a ResourceSnapshotConsumer - * TODO: Write JSON blob of node repo somewhere * * @author olaa */ @@ -36,8 +31,8 @@ public class ResourceMeterMaintainer extends Maintainer { private final NodeRepository nodeRepository; private final ResourceSnapshotConsumer resourceSnapshotConsumer; - private static final String metering_last_reported = "metering_last_reported"; - private static final String metering_total_reported = "metering_total_reported"; + private static final String METERING_LAST_REPORTED = "metering_last_reported"; + private static final String METERING_TOTAL_REPORTED = "metering_total_reported"; @SuppressWarnings("WeakerAccess") public ResourceMeterMaintainer(Controller controller, @@ -57,22 +52,12 @@ public class ResourceMeterMaintainer extends Maintainer { @Override protected void maintain() { List<NodeRepositoryNode> nodes = getNodes(); - Map<ApplicationId, ResourceAllocation> resourceAllocationByApplication = getResourceAllocationByApplication(nodes); - - // For now, we're only interested in resource allocation - Instant timeStamp = clock.instant(); - Map<ApplicationId, ResourceSnapshot> resourceSnapshots = resourceAllocationByApplication.entrySet().stream() - .collect(Collectors.toMap( - e -> e.getKey(), - e -> new ResourceSnapshot(e.getValue(), timeStamp)) - ); - + List<ResourceSnapshot> resourceSnapshots = getResourceSnapshots(nodes); resourceSnapshotConsumer.consume(resourceSnapshots); - metric.set(metering_last_reported, clock.millis() / 1000, metric.createContext(Collections.emptyMap())); - metric.set(metering_total_reported, resourceSnapshots.values().stream() - .map(ResourceSnapshot::getResourceAllocation) + metric.set(METERING_LAST_REPORTED, clock.millis() / 1000, metric.createContext(Collections.emptyMap())); + metric.set(METERING_TOTAL_REPORTED, resourceSnapshots.stream() .mapToDouble(r -> r.getCpuCores() + r.getMemoryGb() + r.getDiskGb()) // total metered resource usage, for alerting on drastic changes .sum() , metric.createContext(Collections.emptyMap())); @@ -88,11 +73,12 @@ public class ResourceMeterMaintainer extends Maintainer { .collect(Collectors.toList()); } - private Map<ApplicationId, ResourceAllocation> getResourceAllocationByApplication(List<NodeRepositoryNode> nodes) { + private List<ResourceSnapshot> getResourceSnapshots(List<NodeRepositoryNode> nodes) { return nodes.stream() .collect(Collectors.groupingBy( node -> applicationIdFromNodeOwner(node.getOwner()), - Collectors.collectingAndThen(Collectors.toList(), ResourceAllocation::from))); + Collectors.collectingAndThen(Collectors.toList(), nodeList -> ResourceSnapshot.from(nodeList, clock.instant())) + )).values().stream().collect(Collectors.toList()); } private ApplicationId applicationIdFromNodeOwner(NodeOwner owner) { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainerTest.java index 540efcba3b8..ad4511e7f11 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainerTest.java @@ -1,10 +1,8 @@ // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; -import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.ControllerTester; import com.yahoo.config.provision.ApplicationId; -import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeRepositoryNode; import com.yahoo.vespa.hosted.controller.api.integration.resource.ResourceSnapshot; import com.yahoo.vespa.hosted.controller.api.integration.stubs.MockResourceSnapshotConsumer; import com.yahoo.vespa.hosted.controller.integration.NodeRepositoryClientMock; @@ -13,10 +11,7 @@ import com.yahoo.vespa.hosted.controller.integration.ZoneApiMock; import org.junit.Test; import java.time.Duration; -import java.util.ArrayList; -import java.util.Collection; import java.util.List; -import java.util.Map; import static org.junit.Assert.*; @@ -41,20 +36,20 @@ public class ResourceMeterMaintainerTest { ResourceMeterMaintainer resourceMeterMaintainer = new ResourceMeterMaintainer(tester.controller(), Duration.ofMinutes(5), new JobControl(tester.curator()), nodeRepository, tester.clock(), metrics, snapshotConsumer); resourceMeterMaintainer.maintain(); - Map<ApplicationId, ResourceSnapshot> consumedResources = snapshotConsumer.consumedResources(); + List<ResourceSnapshot> consumedResources = snapshotConsumer.consumedResources(); + // The mocked repository contains two applications, so we should also consume two ResourceSnapshots assertEquals(2, consumedResources.size()); + ResourceSnapshot app1 = consumedResources.stream().filter(snapshot -> snapshot.getApplicationId().equals(ApplicationId.from("tenant1", "app1", "default"))).findFirst().orElseThrow(); + ResourceSnapshot app2 = consumedResources.stream().filter(snapshot -> snapshot.getApplicationId().equals(ApplicationId.from("tenant2", "app2", "default"))).findFirst().orElseThrow(); - ResourceSnapshot app1 = consumedResources.get(ApplicationId.from("tenant1", "app1", "default")); - ResourceSnapshot app2 = consumedResources.get(ApplicationId.from("tenant2", "app2", "default")); + assertEquals(24, app1.getCpuCores(), DELTA); + assertEquals(24, app1.getMemoryGb(), DELTA); + assertEquals(500, app1.getDiskGb(), DELTA); - assertEquals(24, app1.getResourceAllocation().getCpuCores(), DELTA); - assertEquals(24, app1.getResourceAllocation().getMemoryGb(), DELTA); - assertEquals(500, app1.getResourceAllocation().getDiskGb(), DELTA); - - assertEquals(40, app2.getResourceAllocation().getCpuCores(), DELTA); - assertEquals(24, app2.getResourceAllocation().getMemoryGb(), DELTA); - assertEquals(500, app2.getResourceAllocation().getDiskGb(), DELTA); + assertEquals(40, app2.getCpuCores(), DELTA); + assertEquals(24, app2.getMemoryGb(), DELTA); + assertEquals(500, app2.getDiskGb(), DELTA); assertEquals(tester.clock().millis()/1000, metrics.getMetric("metering_last_reported")); assertEquals(1112.0d, (Double) metrics.getMetric("metering_total_reported"), DELTA); |