diff options
95 files changed, 476 insertions, 5958 deletions
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/VespaDomBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/VespaDomBuilder.java index 62b76374af9..a4f2ecd6675 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/VespaDomBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/VespaDomBuilder.java @@ -45,7 +45,9 @@ public class VespaDomBuilder extends VespaModelBuilder { public static final String JVMARGS_ATTRIB_NAME = "jvmargs"; public static final String JVM_OPTIONS = "jvm-options"; + public static final String OPTIONS = "options"; public static final String JVM_GC_OPTIONS = "jvm-gc-options"; + public static final String GC_OPTIONS = "gc-options"; public static final String PRELOAD_ATTRIB_NAME = "preload"; // Intended for vespa engineers public static final String MMAP_NOCORE_LIMIT = "mmap-core-limit"; // Intended for vespa engineers public static final String CORE_ON_OOM = "core-on-oom"; // Intended for vespa engineers diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java index 7e80c6be221..46589d655ac 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java @@ -456,6 +456,31 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { } return jvmOptions; } + + void extractJvmFromLegacyNodesTag(List<ApplicationContainer> nodes, ApplicationContainerCluster cluster, + Element nodesElement, ConfigModelContext context) + { + applyNodesTagJvmArgs(nodes, getJvmOptions(cluster, nodesElement, context.getDeployLogger())); + + if (!cluster.getJvmGCOptions().isPresent()) { + String jvmGCOptions = nodesElement.hasAttribute(VespaDomBuilder.JVM_GC_OPTIONS) + ? nodesElement.getAttribute(VespaDomBuilder.JVM_GC_OPTIONS) + : null; + cluster.setJvmGCOptions(buildJvmGCOptions(context.getDeployState().zone(), jvmGCOptions, context.getDeployState().isHosted())); + } + + applyMemoryPercentage(cluster, nodesElement.getAttribute(VespaDomBuilder.Allocated_MEMORY_ATTRIB_NAME)); + } + void extractJvmTag(List<ApplicationContainer> nodes, ApplicationContainerCluster cluster, + Element jvmElement, ConfigModelContext context) + { + applyNodesTagJvmArgs(nodes, jvmElement.getAttribute(VespaDomBuilder.OPTIONS)); + applyMemoryPercentage(cluster, jvmElement.getAttribute(VespaDomBuilder.Allocated_MEMORY_ATTRIB_NAME)); + String jvmGCOptions = jvmElement.hasAttribute(VespaDomBuilder.GC_OPTIONS) + ? jvmElement.getAttribute(VespaDomBuilder.GC_OPTIONS) + : null; + cluster.setJvmGCOptions(buildJvmGCOptions(context.getDeployState().zone(), jvmGCOptions, context.getDeployState().isHosted())); + } private void addNodesFromXml(ApplicationContainerCluster cluster, Element containerElement, ConfigModelContext context) { Element nodesElement = XML.getChild(containerElement, "nodes"); Element rotationsElement = XML.getChild(containerElement, "rotations"); @@ -467,22 +492,19 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { cluster.addContainers(Collections.singleton(node)); } else { List<ApplicationContainer> nodes = createNodes(cluster, nodesElement, rotationsElement, context); - applyNodesTagJvmArgs(nodes, getJvmOptions(cluster, nodesElement, context.getDeployLogger())); - if ( !cluster.getJvmGCOptions().isPresent()) { - String jvmGCOptions = nodesElement.hasAttribute(VespaDomBuilder.JVM_GC_OPTIONS) - ? nodesElement.getAttribute(VespaDomBuilder.JVM_GC_OPTIONS) - : null; - cluster.setJvmGCOptions(buildJvmGCOptions(context.getDeployState().zone(), jvmGCOptions, context.getDeployState().isHosted())); + Element jvmElement = XML.getChild(nodesElement, "jvm"); + if (jvmElement == null) { + extractJvmFromLegacyNodesTag(nodes, cluster, nodesElement, context); + } else { + extractJvmTag(nodes, cluster, jvmElement, context); } - applyRoutingAliasProperties(nodes, cluster); applyDefaultPreload(nodes, nodesElement); String environmentVars = getEnvironmentVariables(XML.getChild(nodesElement, ENVIRONMENT_VARIABLES_ELEMENT)); if (environmentVars != null && !environmentVars.isEmpty()) { cluster.setEnvironmentVars(environmentVars); } - applyMemoryPercentage(cluster, nodesElement.getAttribute(VespaDomBuilder.Allocated_MEMORY_ATTRIB_NAME)); if (useCpuSocketAffinity(nodesElement)) AbstractService.distributeCpuSocketAffinity(nodes); @@ -511,7 +533,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { return createNodesFromNodeList(context.getDeployState(), cluster, nodesElement); } - private void applyRoutingAliasProperties(List<ApplicationContainer> result, ApplicationContainerCluster cluster) { + private static void applyRoutingAliasProperties(List<ApplicationContainer> result, ApplicationContainerCluster cluster) { if (!cluster.serviceAliases().isEmpty()) { result.forEach(container -> { container.setProp("servicealiases", cluster.serviceAliases().stream().collect(Collectors.joining(","))); @@ -524,7 +546,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { } } - private void applyMemoryPercentage(ApplicationContainerCluster cluster, String memoryPercentage) { + private static void applyMemoryPercentage(ApplicationContainerCluster cluster, String memoryPercentage) { if (memoryPercentage == null || memoryPercentage.isEmpty()) return; memoryPercentage = memoryPercentage.trim(); @@ -688,21 +710,21 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { return Optional.empty(); } - private boolean useCpuSocketAffinity(Element nodesElement) { + private static boolean useCpuSocketAffinity(Element nodesElement) { if (nodesElement.hasAttribute(VespaDomBuilder.CPU_SOCKET_AFFINITY_ATTRIB_NAME)) return Boolean.parseBoolean(nodesElement.getAttribute(VespaDomBuilder.CPU_SOCKET_AFFINITY_ATTRIB_NAME)); else return false; } - private void applyNodesTagJvmArgs(List<ApplicationContainer> containers, String jvmArgs) { + private static void applyNodesTagJvmArgs(List<ApplicationContainer> containers, String jvmArgs) { for (Container container: containers) { if (container.getAssignedJvmOptions().isEmpty()) container.prependJvmOptions(jvmArgs); } } - private void applyDefaultPreload(List<ApplicationContainer> containers, Element nodesElement) { + private static void applyDefaultPreload(List<ApplicationContainer> containers, Element nodesElement) { if (! nodesElement.hasAttribute(VespaDomBuilder.PRELOAD_ATTRIB_NAME)) return; for (Container container: containers) container.setPreLoad(nodesElement.getAttribute(VespaDomBuilder.PRELOAD_ATTRIB_NAME)); diff --git a/config-model/src/main/resources/schema/containercluster.rnc b/config-model/src/main/resources/schema/containercluster.rnc index d3a3b26c635..73492f6d650 100644 --- a/config-model/src/main/resources/schema/containercluster.rnc +++ b/config-model/src/main/resources/schema/containercluster.rnc @@ -215,6 +215,11 @@ NodesOfContainerCluster = element nodes { attribute preload { text }? & attribute allocated-memory { text }? & attribute cpu-socket-affinity { xsd:boolean }? & + element jvm { + attribute options { text }? & + attribute gc-options { text }? & + attribute allocated-memory { text }? + } ? & Resources? & element environment-variables { anyElement + diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java index 31077df7f7c..c7816c23119 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java @@ -68,6 +68,31 @@ import static org.junit.Assert.fail; public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase { @Test + public void verify_jvm_tag_with_attributes() throws IOException, SAXException { + String servicesXml = + "<container version='1.0'>" + + " <search/>" + + " <nodes>" + + " <jvm options='-XX:SoftRefLRUPolicyMSPerMB=2500' gc-options='-XX:+UseParNewGC' allocated-memory='45%'/>" + + " <node hostalias='mockhost'/>" + + " </nodes>" + + "</container>"; + ApplicationPackage applicationPackage = new MockApplicationPackage.Builder().withServices(servicesXml).build(); + // Need to create VespaModel to make deploy properties have effect + final MyLogger logger = new MyLogger(); + VespaModel model = new VespaModel(new NullConfigModelRegistry(), new DeployState.Builder() + .applicationPackage(applicationPackage) + .deployLogger(logger) + .properties(new TestProperties().setHostedVespa(true)) + .build()); + QrStartConfig.Builder qrStartBuilder = new QrStartConfig.Builder(); + model.getConfig(qrStartBuilder, "container/container.0"); + QrStartConfig qrStartConfig = new QrStartConfig(qrStartBuilder); + assertEquals("-XX:+UseParNewGC", qrStartConfig.jvm().gcopts()); + assertEquals(45, qrStartConfig.jvm().heapSizeAsPercentageOfPhysicalMemory()); + assertEquals("-XX:SoftRefLRUPolicyMSPerMB=2500", model.getContainerClusters().values().iterator().next().getContainers().get(0).getJvmOptions()); + } + @Test public void detect_conflicting_jvmgcoptions_in_jvmargs() { assertFalse(ContainerModelBuilder.incompatibleGCOptions("")); assertFalse(ContainerModelBuilder.incompatibleGCOptions("UseG1GC")); @@ -79,15 +104,15 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase { @Test public void honours_jvm_gc_options() { Element clusterElem = DomBuilderTest.parse( - "<jdisc version='1.0'>", + "<container version='1.0'>", " <search/>", " <nodes jvm-gc-options='-XX:+UseG1GC'>", " <node hostalias='mockhost'/>", " </nodes>", - "</jdisc>" ); + "</container>" ); createModel(root, clusterElem); QrStartConfig.Builder qrStartBuilder = new QrStartConfig.Builder(); - root.getConfig(qrStartBuilder, "jdisc/container.0"); + root.getConfig(qrStartBuilder, "container/container.0"); QrStartConfig qrStartConfig = new QrStartConfig(qrStartBuilder); assertEquals("-XX:+UseG1GC", qrStartConfig.jvm().gcopts()); } @@ -99,11 +124,11 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase { } private static void verifyIgnoreJvmGCOptionsIfJvmArgs(boolean isHosted, String jvmOptionsName, String expectedGC) throws IOException, SAXException { String servicesXml = - "<jdisc version='1.0'>" + + "<container version='1.0'>" + " <nodes jvm-gc-options='-XX:+UseG1GC' " + jvmOptionsName + "='-XX:+UseParNewGC'>" + " <node hostalias='mockhost'/>" + " </nodes>" + - "</jdisc>"; + "</container>"; ApplicationPackage applicationPackage = new MockApplicationPackage.Builder().withServices(servicesXml).build(); // Need to create VespaModel to make deploy properties have effect final MyLogger logger = new MyLogger(); @@ -114,7 +139,7 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase { .properties(new TestProperties().setHostedVespa(isHosted)) .build()); QrStartConfig.Builder qrStartBuilder = new QrStartConfig.Builder(); - model.getConfig(qrStartBuilder, "jdisc/container.0"); + model.getConfig(qrStartBuilder, "container/container.0"); QrStartConfig qrStartConfig = new QrStartConfig(qrStartBuilder); assertEquals(expectedGC, qrStartConfig.jvm().gcopts()); } @@ -127,11 +152,11 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase { private void verifyJvmGCOptions(boolean isHosted, String override, Zone zone, String expected) throws IOException, SAXException { String servicesXml = - "<jdisc version='1.0'>" + + "<container version='1.0'>" + " <nodes " + ((override == null) ? ">" : ("jvm-gc-options='" + override + "'>")) + " <node hostalias='mockhost'/>" + " </nodes>" + - "</jdisc>"; + "</container>"; ApplicationPackage applicationPackage = new MockApplicationPackage.Builder().withServices(servicesXml).build(); // Need to create VespaModel to make deploy properties have effect final MyLogger logger = new MyLogger(); @@ -142,7 +167,7 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase { .properties(new TestProperties().setHostedVespa(isHosted)) .build()); QrStartConfig.Builder qrStartBuilder = new QrStartConfig.Builder(); - model.getConfig(qrStartBuilder, "jdisc/container.0"); + model.getConfig(qrStartBuilder, "container/container.0"); QrStartConfig qrStartConfig = new QrStartConfig(qrStartBuilder); assertEquals(expected, qrStartConfig.jvm().gcopts()); } @@ -164,25 +189,25 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase { @Test public void default_port_is_4080() { Element clusterElem = DomBuilderTest.parse( - "<jdisc version='1.0'>", + "<container version='1.0'>", nodesXml, - "</jdisc>" ); + "</container>" ); createModel(root, clusterElem); - AbstractService container = (AbstractService)root.getProducer("jdisc/container.0"); + AbstractService container = (AbstractService)root.getProducer("container/container.0"); assertThat(container.getRelativePort(0), is(getDefaults().vespaWebServicePort())); } @Test public void http_server_port_is_configurable_and_does_not_affect_other_ports() { Element clusterElem = DomBuilderTest.parse( - "<jdisc version='1.0'>", + "<container version='1.0'>", " <http>", " <server port='9000' id='foo' />", " </http>", nodesXml, - "</jdisc>" ); + "</container>" ); createModel(root, clusterElem); - AbstractService container = (AbstractService)root.getProducer("jdisc/container.0"); + AbstractService container = (AbstractService)root.getProducer("container/container.0"); assertThat(container.getRelativePort(0), is(9000)); assertThat(container.getRelativePort(1), is(not(9001))); } @@ -194,12 +219,12 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase { "<admin version='3.0'>" + " <nodes count='1'/>" + "</admin>" + - "<jdisc version='1.0'>" + + "<container version='1.0'>" + " <http>" + " <server port='9000' id='foo' />" + " </http>" + nodesXml + - "</jdisc>" + + "</container>" + "</services>"; ApplicationPackage applicationPackage = new MockApplicationPackage.Builder().withServices(servicesXml).build(); // Need to create VespaModel to make deploy properties have effect @@ -224,26 +249,26 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase { @Test public void one_cluster_with_explicit_port_and_one_without_is_ok() { Element cluster1Elem = DomBuilderTest.parse( - "<jdisc id='cluster1' version='1.0' />"); + "<container id='cluster1' version='1.0' />"); Element cluster2Elem = DomBuilderTest.parse( - "<jdisc id='cluster2' version='1.0'>", + "<container id='cluster2' version='1.0'>", " <http>", " <server port='8000' id='foo' />", " </http>", - "</jdisc>"); + "</container>"); createModel(root, cluster1Elem, cluster2Elem); } @Test public void two_clusters_without_explicit_port_throws_exception() { Element cluster1Elem = DomBuilderTest.parse( - "<jdisc id='cluster1' version='1.0'>", + "<container id='cluster1' version='1.0'>", nodesXml, - "</jdisc>" ); + "</container>" ); Element cluster2Elem = DomBuilderTest.parse( - "<jdisc id='cluster2' version='1.0'>", + "<container id='cluster2' version='1.0'>", nodesXml, - "</jdisc>" ); + "</container>" ); try { createModel(root, cluster1Elem, cluster2Elem); fail("Expected exception"); @@ -255,7 +280,7 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase { @Test public void verify_bindings_for_builtin_handlers() { Element clusterElem = DomBuilderTest.parse( - "<jdisc id='default' version='1.0' />" + "<container id='default' version='1.0' />" ); createModel(root, clusterElem); JdiscBindingsConfig config = root.getConfig(JdiscBindingsConfig.class, "default/container.0"); @@ -275,11 +300,11 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase { @Test public void default_root_handler_is_disabled_when_user_adds_a_handler_with_same_binding() { Element clusterElem = DomBuilderTest.parse( - "<jdisc id='default' version='1.0'>" + + "<container id='default' version='1.0'>" + " <handler id='userRootHandler'>" + " <binding>" + ContainerCluster.ROOT_HANDLER_BINDING + "</binding>" + " </handler>" + - "</jdisc>"); + "</container>"); createModel(root, clusterElem); ComponentsConfig.Components userRootHandler = getComponent(componentsConfig(), BindingsOverviewHandler.class.getName()); @@ -304,13 +329,13 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase { private void createClusterWithJDiscHandler() { Element clusterElem = DomBuilderTest.parse( - "<jdisc id='default' version='1.0'>", + "<container id='default' version='1.0'>", " <handler id='discHandler'>", " <binding>binding0</binding>", " <binding>binding1</binding>", " <clientBinding>clientBinding</clientBinding>", " </handler>", - "</jdisc>"); + "</container>"); createModel(root, clusterElem); } @@ -336,14 +361,14 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase { private void createClusterWithServlet() { Element clusterElem = DomBuilderTest.parse( - "<jdisc id='default' version='1.0'>", + "<container id='default' version='1.0'>", " <servlet id='myServlet' class='myClass' bundle='myBundle'>", " <path>p/a/t/h</path>", " <servlet-config>", " <myKey>myValue</myKey>", " </servlet-config>", " </servlet>", - "</jdisc>"); + "</container>"); createModel(root, clusterElem); } @@ -352,12 +377,12 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase { @Test public void processing_handler_bindings_can_be_overridden() { Element clusterElem = DomBuilderTest.parse( - "<jdisc id='default' version='1.0'>", + "<container id='default' version='1.0'>", " <processing>", " <binding>binding0</binding>", " <binding>binding1</binding>", " </processing>", - "</jdisc>"); + "</container>"); createModel(root, clusterElem); @@ -385,13 +410,13 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase { private void createModelWithClientProvider() { Element clusterElem = DomBuilderTest.parse( - "<jdisc id='default' version='1.0'>" + + "<container id='default' version='1.0'>" + " <client id='discClient'>" + " <binding>binding0</binding>" + " <binding>binding1</binding>" + " <serverBinding>serverBinding</serverBinding>" + " </client>" + - "</jdisc>" ); + "</container>" ); createModel(root, clusterElem); } @@ -399,9 +424,9 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase { @Test public void serverProviders_are_included_in_components_config() { Element clusterElem = DomBuilderTest.parse( - "<jdisc id='default' version='1.0'>" + + "<container id='default' version='1.0'>" + " <server id='discServer' />" + - "</jdisc>" ); + "</container>" ); createModel(root, clusterElem); @@ -433,7 +458,7 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase { private void createClusterWithProcessingAndSearchChains() { Element clusterElem = DomBuilderTest.parse( - "<jdisc id='default' version='1.0'>" + + "<container id='default' version='1.0'>" + " <search>" + " <chain id='default'>" + " <searcher id='testSearcher' />" + @@ -445,7 +470,7 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase { " </chain>" + " </processing>" + nodesXml + - " </jdisc>"); + " </container>"); createModel(root, clusterElem); } @@ -453,7 +478,7 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase { @Test public void user_config_can_be_overridden_on_node() { Element containerElem = DomBuilderTest.parse( - "<jdisc id='default' version='1.0'>", + "<container id='default' version='1.0'>", " <config name=\"prelude.cluster.qr-monitor\">" + " <requesttimeout>111</requesttimeout>", " </config> " + @@ -465,7 +490,7 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase { " </config> ", " </node>", " </nodes>", - "</jdisc>"); + "</container>"); root = ContentClusterUtils.createMockRoot(new String[]{"host1", "host2"}); createModel(root, containerElem); @@ -478,14 +503,14 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase { @Test public void nested_components_are_injected_to_handlers() throws Exception { Element clusterElem = DomBuilderTest.parse( - "<jdisc id='default' version='1.0'>", + "<container id='default' version='1.0'>", " <handler id='myHandler'>", " <component id='injected' />", " </handler>", " <client id='myClient'>", // remember, a client is also a request handler " <component id='injected' />", " </client>", - "</jdisc>"); + "</container>"); createModel(root, clusterElem); Component<?,?> handler = getContainerComponent("default", "myHandler"); @@ -509,14 +534,14 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase { @Test public void affinity_is_set() { Element clusterElem = DomBuilderTest.parse( - "<jdisc id='default' version='1.0'>", + "<container id='default' version='1.0'>", " <http>", " <server port='" + getDefaults().vespaWebServicePort() + "' id='main' />", " </http>", " <nodes cpu-socket-affinity='true'>", " <node hostalias='node1' />", " </nodes>" + - "</jdisc>"); + "</container>"); createModel(root, clusterElem); assertTrue(getContainerCluster("default").getContainers().get(0).getAffinity().isPresent()); assertThat(getContainerCluster("default").getContainers().get(0).getAffinity().get().cpuSocket(), is(0)); @@ -524,7 +549,7 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase { @Test public void singlenode_servicespec_is_used_with_hosts_xml() throws IOException, SAXException { - String servicesXml = "<jdisc id='default' version='1.0' />"; + String servicesXml = "<container id='default' version='1.0' />"; String hostsXml = "<hosts>\n" + " <host name=\"test1.yahoo.com\">\n" + " <alias>node1</alias>\n" + @@ -541,7 +566,7 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase { @Test public void http_aliases_are_stored_on_cluster_and_on_service_properties() { Element clusterElem = DomBuilderTest.parse( - "<jdisc id='default' version='1.0'>", + "<container id='default' version='1.0'>", " <aliases>", " <service-alias>service1</service-alias>", " <service-alias>service2</service-alias>", @@ -551,7 +576,7 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase { " <nodes>", " <node hostalias='host1' />", " </nodes>", - "</jdisc>"); + "</container>"); createModel(root, clusterElem); assertEquals(getContainerCluster("default").serviceAliases().get(0), "service1"); @@ -566,7 +591,7 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase { @Test public void http_aliases_are_only_honored_in_prod_environment() { Element clusterElem = DomBuilderTest.parse( - "<jdisc id='default' version='1.0'>", + "<container id='default' version='1.0'>", " <aliases>", " <service-alias>service1</service-alias>", " <endpoint-alias>foo1.bar1.com</endpoint-alias>", @@ -574,7 +599,7 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase { " <nodes>", " <node hostalias='host1' />", " </nodes>", - "</jdisc>"); + "</container>"); DeployState deployState = new DeployState.Builder().zone(new Zone(Environment.dev, RegionName.from("us-east-1"))).build(); createModel(root, deployState, null, clusterElem); @@ -587,7 +612,7 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase { @Test public void singlenode_servicespec_is_used_with_hosted_vespa() throws IOException, SAXException { - String servicesXml = "<jdisc id='default' version='1.0' />"; + String servicesXml = "<container id='default' version='1.0' />"; ApplicationPackage applicationPackage = new MockApplicationPackage.Builder().withServices(servicesXml).build(); VespaModel model = new VespaModel(new NullConfigModelRegistry(), new DeployState.Builder() .modelHostProvisioner(new InMemoryProvisioner(true, "host1.yahoo.com", "host2.yahoo.com")) @@ -617,9 +642,9 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase { @Test public void vip_status_handler_uses_file_for_hosted_vespa() throws Exception { String servicesXml = "<services>" + - "<jdisc version='1.0'>" + + "<container version='1.0'>" + nodesXml + - "</jdisc>" + + "</container>" + "</services>"; ApplicationPackage applicationPackage = new MockApplicationPackage.Builder().withServices(servicesXml).build(); @@ -629,7 +654,7 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase { .build()); AbstractConfigProducerRoot modelRoot = model.getRoot(); - VipStatusConfig vipStatusConfig = modelRoot.getConfig(VipStatusConfig.class, "jdisc/component/status.html-status-handler"); + VipStatusConfig vipStatusConfig = modelRoot.getConfig(VipStatusConfig.class, "container/component/status.html-status-handler"); assertTrue(vipStatusConfig.accessdisk()); assertEquals(ContainerModelBuilder.HOSTED_VESPA_STATUS_FILE, vipStatusConfig.statusfile()); } @@ -641,11 +666,11 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase { "<admin version='3.0'>" + " <nodes count='1'/>" + "</admin>" + - "<jdisc id ='default' version='1.0'>" + + "<container id ='default' version='1.0'>" + " <nodes>" + " <node hostalias='node1' />" + " </nodes>" + - "</jdisc>" + + "</container>" + "</services>"; ApplicationPackage applicationPackage = new MockApplicationPackage.Builder() @@ -671,13 +696,13 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase { @Test public void secret_store_can_be_set_up() { Element clusterElem = DomBuilderTest.parse( - "<jdisc version='1.0'>", + "<container version='1.0'>", " <secret-store>", " <group name='group1' environment='env1'/>", " </secret-store>", - "</jdisc>"); + "</container>"); createModel(root, clusterElem); - SecretStore secretStore = getContainerCluster("jdisc").getSecretStore().get(); + SecretStore secretStore = getContainerCluster("container").getSecretStore().get(); assertEquals("group1", secretStore.getGroups().get(0).name); assertEquals("env1", secretStore.getGroups().get(0).environment); } @@ -703,10 +728,10 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase { private Element generateContainerElementWithRenderer(String rendererId) { return DomBuilderTest.parse( - "<jdisc id='default' version='1.0'>", + "<container id='default' version='1.0'>", " <search>", String.format(" <renderer id='%s'/>", rendererId), " </search>", - "</jdisc>"); + "</container>"); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java index 472a0c5fb7e..fdadae065d2 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java @@ -451,20 +451,33 @@ public class InternalStepRunner implements StepRunner { } catch (Exception e) { logger.log(INFO, "Failure getting vespa logs for " + id, e); + return Optional.of(error); } - return Optional.of(running); // Don't let failure here stop cleanup. + return Optional.of(running); } private Optional<RunStatus> deactivateReal(RunId id, DualLogger logger) { - logger.log("Deactivating deployment of " + id.application() + " in " + id.type().zone(controller.system()) + " ..."); - controller.applications().deactivate(id.application(), id.type().zone(controller.system())); - return Optional.of(running); + try { + logger.log("Deactivating deployment of " + id.application() + " in " + id.type().zone(controller.system()) + " ..."); + controller.applications().deactivate(id.application(), id.type().zone(controller.system())); + return Optional.of(running); + } + catch (RuntimeException e) { + logger.log(WARNING, "Failed deleting application " + id.application(), e); + return Optional.of(error); + } } private Optional<RunStatus> deactivateTester(RunId id, DualLogger logger) { - logger.log("Deactivating tester of " + id.application() + " in " + id.type().zone(controller.system()) + " ..."); - controller.jobController().deactivateTester(id.tester(), id.type()); - return Optional.of(running); + try { + logger.log("Deactivating tester of " + id.application() + " in " + id.type().zone(controller.system()) + " ..."); + controller.jobController().deactivateTester(id.tester(), id.type()); + return Optional.of(running); + } + catch (RuntimeException e) { + logger.log(WARNING, "Failed deleting tester of " + id.application(), e); + return Optional.of(error); + } } private Optional<RunStatus> report(RunId id, DualLogger logger) { @@ -482,6 +495,7 @@ public class InternalStepRunner implements StepRunner { } catch (IllegalStateException e) { logger.log(INFO, "Job '" + id.type() + "'no longer supposed to run?:", e); + return Optional.of(error); } return Optional.of(running); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Run.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Run.java index 0cb400004aa..f052c7d91ab 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Run.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Run.java @@ -14,6 +14,7 @@ import java.util.Optional; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.aborted; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.running; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.success; +import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.failed; import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.succeeded; import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.unfinished; import static java.util.Objects.requireNonNull; @@ -173,15 +174,14 @@ public class Run { .iterator()); } - /** Returns the list of not-yet-succeeded run-always steps whose run-always prerequisites have all succeeded. */ + /** Returns the list of not-yet-run run-always steps whose run-always prerequisites have all run. */ private List<Step> forcedSteps() { return ImmutableList.copyOf(steps.entrySet().stream() - .filter(entry -> entry.getValue() != succeeded + .filter(entry -> entry.getValue() == unfinished && JobProfile.of(id.type()).alwaysRun().contains(entry.getKey()) && entry.getKey().prerequisites().stream() .filter(JobProfile.of(id.type()).alwaysRun()::contains) - .allMatch(step -> steps.get(step) == null - || steps.get(step) == succeeded)) + .allMatch(step -> steps.get(step) != unfinished)) .map(Map.Entry::getKey) .iterator()); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Step.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Step.java index 051a99074d1..a5f9ef86da4 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Step.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Step.java @@ -15,7 +15,7 @@ import java.util.List; * only the prerequisites of a step which are included in a run's profile will be considered. * Under normal circumstances, a step will run only after each of its prerequisites have succeeded. * When a run has failed, however, each of the always-run steps of the run's profile will be run, - * again in a topological order, and again requiring success of all their always-run prerequisites. + * again in a topological order, and requiring all their always-run prerequisites to have run. * * 2. A step will never run concurrently with its prerequisites. This is to ensure, e.g., that relevant * information from a failed run is stored, and that deployment does not occur after deactivation. diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java index 48cf2a7824d..f3057baaf9a 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java @@ -351,6 +351,9 @@ class JobControllerApiHandlerHelper { private static void applicationVersionToSlime(Cursor versionObject, ApplicationVersion version) { versionObject.setString("hash", version.id()); + if (version.isUnknown()) + return; + versionObject.setLong("build", version.buildNumber().getAsLong()); Cursor sourceObject = versionObject.setObject("source"); sourceObject.setString("gitRepository", version.source().get().repository()); diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index c878487cf4e..9b74dc61bbb 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -3,8 +3,6 @@ package com.yahoo.vespa.flags; import com.yahoo.vespa.defaults.Defaults; -import java.util.ArrayList; -import java.util.Collections; import java.util.List; import java.util.Optional; import java.util.TreeMap; @@ -66,7 +64,7 @@ public class Flags { HOSTNAME); public static final UnboundListFlag<String> DISABLED_HOST_ADMIN_TASKS = defineListFlag( - "disabled-host-admin-tasks", Collections.emptyList(), + "disabled-host-admin-tasks", List.of(), "List of host-admin task names (as they appear in the log, e.g. root>main>UpgradeTask) that should be skipped", "Takes effect on next host admin tick", HOSTNAME, NODE_TYPE); @@ -119,6 +117,11 @@ public class Flags { "Takes effect on next deployment", APPLICATION_ID); + public static final UnboundListFlag<String> DYNAMIC_PROVISIONING_FLAVORS = defineListFlag( + "dynamic-provisioning-flavors", List.of(), + "List of additional Vespa flavor names that can be used for dynamic provisioning", + "Takes effect on next provisioning"); + public static final UnboundBooleanFlag ENABLE_DISK_WRITE_TEST = defineFeatureFlag( "enable-disk-write-test", false, "Regularly issue a small write to disk and fail the host if it is not successful", @@ -237,7 +240,7 @@ public class Flags { } public static List<FlagDefinition> getAllFlags() { - return new ArrayList<>(flags.values()); + return List.copyOf(flags.values()); } public static Optional<FlagDefinition> getFlag(FlagId flagId) { diff --git a/hosted-api/src/main/java/ai/vespa/hosted/api/ControllerHttpClient.java b/hosted-api/src/main/java/ai/vespa/hosted/api/ControllerHttpClient.java index 17cf38a9118..278c80c48ba 100644 --- a/hosted-api/src/main/java/ai/vespa/hosted/api/ControllerHttpClient.java +++ b/hosted-api/src/main/java/ai/vespa/hosted/api/ControllerHttpClient.java @@ -146,7 +146,7 @@ public abstract class ControllerHttpClient { private URI deploymentJobPath(ApplicationId id, ZoneId zone) { return concatenated(instancePath(id), - "job", zone.environment().value() + "-" + zone.region().value()); + "deploy", zone.environment().value() + "-" + zone.region().value()); } private URI defaultRegionPath() { diff --git a/metrics/src/tests/CMakeLists.txt b/metrics/src/tests/CMakeLists.txt index cb6b5212e28..c7ca296e7b5 100644 --- a/metrics/src/tests/CMakeLists.txt +++ b/metrics/src/tests/CMakeLists.txt @@ -1,15 +1,16 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. # Runner for unit tests written in gtest. -# NOTE: All new test classes should be added here. vespa_add_executable(metrics_gtest_runner_app TEST SOURCES countmetrictest.cpp loadmetrictest.cpp metric_timer_test.cpp + metricmanagertest.cpp metricsettest.cpp metrictest.cpp snapshottest.cpp + stresstest.cpp summetrictest.cpp valuemetrictest.cpp gtest_runner.cpp @@ -24,19 +25,3 @@ vespa_add_test( COMMAND metrics_gtest_runner_app ) -# Runner for unit tests written in CppUnit (DEPRECATED). -vespa_add_executable(metrics_testrunner_app TEST - SOURCES - testrunner.cpp - metricmanagertest.cpp - stresstest.cpp - DEPENDS - metrics - vdstestlib -) - -# TODO: Test with a larger chunk size to parallelize test suite runs -vespa_add_test( - NAME metrics_testrunner_app - COMMAND metrics_testrunner_app -) diff --git a/metrics/src/tests/metricmanagertest.cpp b/metrics/src/tests/metricmanagertest.cpp index b89dbad738b..f260089ec84 100644 --- a/metrics/src/tests/metricmanagertest.cpp +++ b/metrics/src/tests/metricmanagertest.cpp @@ -1,14 +1,14 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/metrics/metrics.h> -#include <vespa/metrics/xmlwriter.h> #include <vespa/metrics/jsonwriter.h> -#include <vespa/metrics/textwriter.h> +#include <vespa/metrics/metrics.h> #include <vespa/metrics/printutils.h> #include <vespa/metrics/state_api_adapter.h> -#include <vespa/vdstestlib/cppunit/macros.h> -#include <vespa/vespalib/stllike/asciistream.h> +#include <vespa/metrics/textwriter.h> +#include <vespa/metrics/xmlwriter.h> #include <vespa/vespalib/data/slime/slime.h> +#include <vespa/vespalib/gtest/gtest.h> +#include <vespa/vespalib/stllike/asciistream.h> #include <vespa/vespalib/util/xmlstream.h> #include <vespa/log/log.h> @@ -16,34 +16,8 @@ LOG_SETUP(".test.metricmanager"); namespace metrics { -struct MetricManagerTest : public CppUnit::TestFixture { - void testConsumerVisitor(); - void testSnapshots(); - void testPrintUtils(); - void testXmlOutput(); - void testJsonOutput(); - void jsonOutputSupportsMultipleDimensions(); - void jsonOutputCanNestDimensionsFromMultipleMetricSets(); - void jsonOutputCanHaveMultipleSetsWithSameName(); - void testTextOutput(); - void textOutputSupportsDimensions(); - void testUpdateHooks(); - - CPPUNIT_TEST_SUITE(MetricManagerTest); - CPPUNIT_TEST(testConsumerVisitor); - CPPUNIT_TEST(testSnapshots); - CPPUNIT_TEST(testPrintUtils); - CPPUNIT_TEST(testXmlOutput); - CPPUNIT_TEST(testJsonOutput); - CPPUNIT_TEST(jsonOutputSupportsMultipleDimensions); - CPPUNIT_TEST(jsonOutputCanNestDimensionsFromMultipleMetricSets); - CPPUNIT_TEST(jsonOutputCanHaveMultipleSetsWithSameName); - CPPUNIT_TEST(testTextOutput); - CPPUNIT_TEST(textOutputSupportsDimensions); - CPPUNIT_TEST(testUpdateHooks); - CPPUNIT_TEST_SUITE_END(); +struct MetricManagerTest : public ::testing::Test { -public: // Ugly indirection hack caused by some tests using friended internals of // MetricManager that aren't accessible to "freestanding" fixtures. So we // get the test to do the necessary poking and prodding for us instead. @@ -52,8 +26,6 @@ public: } }; -CPPUNIT_TEST_SUITE_REGISTRATION(MetricManagerTest); - namespace { struct SubMetricSet : public MetricSet @@ -167,45 +139,46 @@ struct MetricNameVisitor : public MetricVisitor { } namespace { - std::pair<std::string, std::string> getMatchedMetrics( - const vespalib::string& config) - { - FastOS_ThreadPool pool(256 * 1024); - TestMetricSet mySet; - MetricManager mm; - mm.registerMetric(mm.getMetricLock(), mySet.set); - mm.init(config, pool); - MetricNameVisitor visitor; - - /** Take a copy to verify clone works. - std::list<Metric::SP> ownerList; - MetricSet::UP copy(dynamic_cast<MetricSet*>( - mm.getMetrics().clone(ownerList))); - mm.visit(*copy, visitor, "consumer"); - */ - - MetricLockGuard g(mm.getMetricLock()); - mm.visit(g, mm.getActiveMetrics(g), visitor, "consumer"); - MetricManager::ConsumerSpec::SP consumerSpec( - mm.getConsumerSpec(g, "consumer")); - return std::pair<std::string, std::string>( - visitor.toString(), - consumerSpec.get() ? consumerSpec->toString() - : "Non-existing consumer"); - } + +std::pair<std::string, std::string> +getMatchedMetrics(const vespalib::string& config) +{ + FastOS_ThreadPool pool(256 * 1024); + TestMetricSet mySet; + MetricManager mm; + mm.registerMetric(mm.getMetricLock(), mySet.set); + mm.init(config, pool); + MetricNameVisitor visitor; + + /** Take a copy to verify clone works. + std::list<Metric::SP> ownerList; + MetricSet::UP copy(dynamic_cast<MetricSet*>( + mm.getMetrics().clone(ownerList))); + mm.visit(*copy, visitor, "consumer"); + */ + + MetricLockGuard g(mm.getMetricLock()); + mm.visit(g, mm.getActiveMetrics(g), visitor, "consumer"); + MetricManager::ConsumerSpec::SP consumerSpec( + mm.getConsumerSpec(g, "consumer")); + return std::pair<std::string, std::string>( + visitor.toString(), + consumerSpec.get() ? consumerSpec->toString() + : "Non-existing consumer"); +} + } #define ASSERT_CONSUMER_MATCH(name, expected, config) \ { \ std::pair<std::string, std::string> consumerMatch( \ getMatchedMetrics(config)); \ - CPPUNIT_ASSERT_EQUAL_MSG(name + std::string(": ") + consumerMatch.second, \ - "\n" + expected, "\n" + consumerMatch.first); \ + EXPECT_EQ("\n" + expected, "\n" + consumerMatch.first) << (name + std::string(": ") + consumerMatch.second); \ } -void MetricManagerTest::testConsumerVisitor() +TEST_F(MetricManagerTest, test_consumer_visitor) { - // Add one tag and a name, check that we get all three. + // Add one tag and a name, check that we get all three. ASSERT_CONSUMER_MATCH("testAddTagAndName", std::string( "temp.val1\n" "temp.val2\n" @@ -221,7 +194,7 @@ void MetricManagerTest::testConsumerVisitor() "consumer[0].addedmetrics[0] temp.val4\n" "consumer[0].addedmetrics[1] temp.multisub.sum.val1\n" ); - // Add two tags, remove one + // Add two tags, remove one ASSERT_CONSUMER_MATCH("testAddAndRemoveTag", std::string( "temp.val1\n" "temp.val4\n" @@ -235,7 +208,7 @@ void MetricManagerTest::testConsumerVisitor() "consumer[0].removedtags[1]\n" "consumer[0].removedtags[0] tag2\n" ); - // Test simple wildcards + // Test simple wildcards ASSERT_CONSUMER_MATCH("testWildCards", std::string( "temp.val1\n" "temp.val2\n" @@ -255,7 +228,7 @@ void MetricManagerTest::testConsumerVisitor() "consumer[0].removedmetrics[0] temp.sub.*\n" "consumer[0].removedmetrics[1] temp.multisub.*\n" ); - // Test more wildcards + // Test more wildcards ASSERT_CONSUMER_MATCH("testWildCards2", std::string( "temp.sub.val1\n" ), @@ -265,7 +238,7 @@ void MetricManagerTest::testConsumerVisitor() "consumer[0].addedmetrics[1]\n" "consumer[0].addedmetrics[0] temp.*.val1\n" ); - // test adding all + // test adding all ASSERT_CONSUMER_MATCH("testAddAll", std::string( "metricmanager.periodichooklatency\n" "metricmanager.snapshothooklatency\n" @@ -300,7 +273,7 @@ void MetricManagerTest::testConsumerVisitor() "consumer[0].addedmetrics[1]\n" "consumer[0].addedmetrics[0] *\n" ); - // test adding all using tags + // test adding all using tags ASSERT_CONSUMER_MATCH("testAddAll2", std::string( "temp.val1\n" "temp.val2\n" @@ -330,7 +303,7 @@ void MetricManagerTest::testConsumerVisitor() "consumer[0].tags[1]\n" "consumer[0].tags[0] *\n" ); - // Test that all metrics are added when a metricsset is added by name + // Test that all metrics are added when a metricsset is added by name ASSERT_CONSUMER_MATCH("testSpecifiedSetName", std::string( "temp.sub.val1\n" "temp.sub.val2\n" @@ -342,7 +315,7 @@ void MetricManagerTest::testConsumerVisitor() "consumer[0].addedmetrics[1]\n" "consumer[0].addedmetrics[0] temp.sub\n" ); - // Test that all metrics are added when a metricsset is added by tag + // Test that all metrics are added when a metricsset is added by tag ASSERT_CONSUMER_MATCH("testSpecifiedSetTag", std::string( "temp.sub.val1\n" "temp.sub.val2\n" @@ -360,8 +333,8 @@ void MetricManagerTest::testConsumerVisitor() "consumer[0].tags[1]\n" "consumer[0].tags[0] sub\n" ); - // Test that all metrics are added from the set except those with a - // certain tag. + // Test that all metrics are added from the set except those with a + // certain tag. ASSERT_CONSUMER_MATCH("testSpecifiedSetTagWithExceptionTags", std::string( "temp.val1\n" "temp.val4\n" @@ -385,79 +358,80 @@ void MetricManagerTest::testConsumerVisitor() }; namespace { - struct FakeTimer : public MetricManager::Timer { - time_t _time; - FakeTimer(time_t startTime = 0) : _time(startTime) {} - time_t getTime() const override { return _time; } - }; - struct BriefValuePrinter : public MetricVisitor { - uint32_t count; - std::ostringstream ost; +struct FakeTimer : public MetricManager::Timer { + time_t _time; + FakeTimer(time_t startTime = 0) : _time(startTime) {} + time_t getTime() const override { return _time; } +}; - BriefValuePrinter() : count(0), ost() {} +struct BriefValuePrinter : public MetricVisitor { + uint32_t count; + std::ostringstream ost; - bool visitMetric(const Metric& metric, bool) override { - if (++count > 1) ost << ","; - //ost << metric.getPath() << ":"; - ost << metric.getDoubleValue("value"); - return true; - } - }; + BriefValuePrinter() : count(0), ost() {} - bool waitForTimeProcessed(const MetricManager& mm, - time_t processtime, uint32_t timeout = 120) - { - uint32_t lastchance = time(0) + timeout; - while (time(0) < lastchance) { - if (mm.getLastProcessedTime() >= processtime) return true; - mm.timeChangedNotification(); - FastOS_Thread::Sleep(10); - } - return false; + bool visitMetric(const Metric& metric, bool) override { + if (++count > 1) ost << ","; + //ost << metric.getPath() << ":"; + ost << metric.getDoubleValue("value"); + return true; + } +}; + +bool waitForTimeProcessed(const MetricManager& mm, + time_t processtime, uint32_t timeout = 120) +{ + uint32_t lastchance = time(0) + timeout; + while (time(0) < lastchance) { + if (mm.getLastProcessedTime() >= processtime) return true; + mm.timeChangedNotification(); + FastOS_Thread::Sleep(10); } + return false; +} - std::string dumpAllSnapshots(const MetricManager& mm, - const std::string& consumer) +std::string dumpAllSnapshots(const MetricManager& mm, + const std::string& consumer) +{ + std::ostringstream ost; + ost << "\n"; { - std::ostringstream ost; - ost << "\n"; - { - MetricLockGuard metricLock(mm.getMetricLock()); - BriefValuePrinter briefValuePrinter; - mm.visit(metricLock, mm.getActiveMetrics(metricLock), - briefValuePrinter, consumer); - ost << "Current: " << briefValuePrinter.ost.str() << "\n"; - } - { - MetricLockGuard metricLock(mm.getMetricLock()); + MetricLockGuard metricLock(mm.getMetricLock()); + BriefValuePrinter briefValuePrinter; + mm.visit(metricLock, mm.getActiveMetrics(metricLock), + briefValuePrinter, consumer); + ost << "Current: " << briefValuePrinter.ost.str() << "\n"; + } + { + MetricLockGuard metricLock(mm.getMetricLock()); + BriefValuePrinter briefValuePrinter; + mm.visit(metricLock, mm.getTotalMetricSnapshot(metricLock), + briefValuePrinter, consumer); + ost << "Total: " << briefValuePrinter.ost.str() << "\n"; + } + std::vector<uint32_t> periods; + { + MetricLockGuard metricLock(mm.getMetricLock()); + periods = mm.getSnapshotPeriods(metricLock); + } + for (uint32_t i=0; i<periods.size(); ++i) { + MetricLockGuard metricLock(mm.getMetricLock()); + const MetricSnapshotSet& set(mm.getMetricSnapshotSet( + metricLock, periods[i])); + ost << set.getName() << "\n"; + uint32_t count = 0; + for (uint32_t j=0; j<2; ++j) { + if (set.getCount() == 1 && j == 1) continue; + const MetricSnapshot& snap(set.getSnapshot(j == 1)); BriefValuePrinter briefValuePrinter; - mm.visit(metricLock, mm.getTotalMetricSnapshot(metricLock), - briefValuePrinter, consumer); - ost << "Total: " << briefValuePrinter.ost.str() << "\n"; - } - std::vector<uint32_t> periods; - { - MetricLockGuard metricLock(mm.getMetricLock()); - periods = mm.getSnapshotPeriods(metricLock); - } - for (uint32_t i=0; i<periods.size(); ++i) { - MetricLockGuard metricLock(mm.getMetricLock()); - const MetricSnapshotSet& set(mm.getMetricSnapshotSet( - metricLock, periods[i])); - ost << set.getName() << "\n"; - uint32_t count = 0; - for (uint32_t j=0; j<2; ++j) { - if (set.getCount() == 1 && j == 1) continue; - const MetricSnapshot& snap(set.getSnapshot(j == 1)); - BriefValuePrinter briefValuePrinter; - mm.visit(metricLock, snap, briefValuePrinter, consumer); - ost << " " << count++ << " " << &snap.getMetrics() << ": " + mm.visit(metricLock, snap, briefValuePrinter, consumer); + ost << " " << count++ << " " << &snap.getMetrics() << ": " << briefValuePrinter.ost.str() << "\n"; - } } - return ost.str(); } + return ost.str(); +} } @@ -475,8 +449,7 @@ namespace { mm.visit(lockGuard, mm.getMetricSnapshot(lockGuard, period), \ briefValuePrinter, "snapper"); \ } \ - CPPUNIT_ASSERT_EQUAL_MSG(dumpAllSnapshots(mm, "snapper"), \ - std::string(expected), briefValuePrinter.ost.str()); \ + EXPECT_EQ(std::string(expected), briefValuePrinter.ost.str()) << dumpAllSnapshots(mm, "snapper"); \ } #define ASSERT_PROCESS_TIME(mm, time) \ @@ -484,10 +457,10 @@ namespace { LOG(info, "Waiting for processed time %u.", time); \ bool gotToCorrectProgress = waitForTimeProcessed(mm, time); \ if (!gotToCorrectProgress) \ - CPPUNIT_FAIL("Failed to get to processed time within timeout"); \ + FAIL() << "Failed to get to processed time within timeout"; \ } -void MetricManagerTest::testSnapshots() +TEST_F(MetricManagerTest, test_snapshots) { FastOS_ThreadPool pool(256 * 1024); FakeTimer* timer = new FakeTimer(1000); @@ -512,29 +485,27 @@ void MetricManagerTest::testSnapshots() MetricLockGuard lockGuard(mm.getMetricLock()); mm.visit(lockGuard, mm.getActiveMetrics(lockGuard), visitor, "snapper"); MetricManager::ConsumerSpec::SP consumerSpec( - mm.getConsumerSpec(lockGuard, "snapper")); - CPPUNIT_ASSERT_EQUAL_MSG(consumerSpec.get() ? consumerSpec->toString() - : "Non-existing consumer", - std::string("\n" - "temp.val6\n" - "temp.sub.val1\n" - "*temp.sub.valsum\n" - "temp.multisub.count\n" - "temp.multisub.a.val1\n" - "*temp.multisub.a.valsum\n" - "temp.multisub.b.val1\n" - "*temp.multisub.b.valsum\n" - "*temp.multisub.sum.val1\n" - "*temp.multisub.sum.val2\n" - "*temp.multisub.sum.valsum\n" - ), - "\n" + visitor.toString()); + mm.getConsumerSpec(lockGuard, "snapper")); + EXPECT_EQ(std::string("\n" + "temp.val6\n" + "temp.sub.val1\n" + "*temp.sub.valsum\n" + "temp.multisub.count\n" + "temp.multisub.a.val1\n" + "*temp.multisub.a.valsum\n" + "temp.multisub.b.val1\n" + "*temp.multisub.b.valsum\n" + "*temp.multisub.sum.val1\n" + "*temp.multisub.sum.val2\n" + "*temp.multisub.sum.valsum\n"), + "\n" + visitor.toString()) << (consumerSpec.get() ? consumerSpec->toString() + : "Non-existing consumer"); } - // Initially, there should be no metrics logged + // Initially, there should be no metrics logged ASSERT_PROCESS_TIME(mm, 1000); ASSERT_VALUES(mm, 5 * 60, ""); - // Adding metrics done in first five minutes. + // Adding metrics done in first five minutes. mySet.val6.addValue(2); mySet.val9.val1.addValue(4); mySet.val10.count.inc(); @@ -547,8 +518,8 @@ void MetricManagerTest::testSnapshots() ASSERT_VALUES(mm, 60 * 60, ""); ASSERT_VALUES(mm, 0 * 60, "2,4,4,1,7,9,1,1,8,2,10"); - // Adding metrics done in second five minute period. Total should - // be updated to account for both + // Adding metrics done in second five minute period. Total should + // be updated to account for both mySet.val6.addValue(4); mySet.val9.val1.addValue(5); mySet.val10.count.inc(); @@ -563,16 +534,16 @@ void MetricManagerTest::testSnapshots() //std::cerr << dumpAllSnapshots(mm, "snapper") << "\n"; - // Adding another five minute period where nothing have happened. - // Metric for last 5 minutes should be 0. + // Adding another five minute period where nothing have happened. + // Metric for last 5 minutes should be 0. timer->_time += 5 * 60; ASSERT_PROCESS_TIME(mm, 1000 + 5 * 60 * 3); ASSERT_VALUES(mm, 5 * 60, "0,0,0,0,0,0,0,0,0,0,0"); ASSERT_VALUES(mm, 60 * 60, ""); ASSERT_VALUES(mm, 0 * 60, "4,5,5,2,8,11,2,2,10,3,13"); - // Advancing time to 60 minute period, we should create a proper - // 60 minute period timer. + // Advancing time to 60 minute period, we should create a proper + // 60 minute period timer. mySet.val6.addValue(6); for (uint32_t i=0; i<9; ++i) { // 9 x 5 minutes. Avoid snapshot bumping // due to taking snapshots in the past @@ -583,7 +554,7 @@ void MetricManagerTest::testSnapshots() ASSERT_VALUES(mm, 60 * 60, "6,5,5,2,8,11,2,2,10,3,13"); ASSERT_VALUES(mm, 0 * 60, "6,5,5,2,8,11,2,2,10,3,13"); - // Test that reset works + // Test that reset works mm.reset(1000); ASSERT_VALUES(mm, -1, "0,0,0,0,0,0,0,0,0,0,0"); ASSERT_VALUES(mm, 5 * 60, "0,0,0,0,0,0,0,0,0,0,0"); @@ -591,7 +562,7 @@ void MetricManagerTest::testSnapshots() ASSERT_VALUES(mm, 0 * 60, "0,0,0,0,0,0,0,0,0,0,0"); } -void MetricManagerTest::testPrintUtils() +TEST_F(MetricManagerTest, test_print_utils) { FastOS_ThreadPool pool(256 * 1024); FakeTimer* timer = new FakeTimer(1000); @@ -623,11 +594,11 @@ void MetricManagerTest::testPrintUtils() MetricLockGuard lockGuard(mm.getMetricLock()); MetricSource source(mm.getActiveMetrics(lockGuard), "temp.multisub"); - CPPUNIT_ASSERT(source.getMetric("count") != 0); - CPPUNIT_ASSERT(source.getMetric("a.val1") != 0); - CPPUNIT_ASSERT(source.getMetric("a.valsum") != 0); - CPPUNIT_ASSERT(source.getMetric("sum.val1") != 0); - CPPUNIT_ASSERT(source.getMetric("sum.valsum") != 0); + ASSERT_TRUE(source.getMetric("count") != nullptr); + ASSERT_TRUE(source.getMetric("a.val1") != nullptr); + ASSERT_TRUE(source.getMetric("a.valsum") != nullptr); + ASSERT_TRUE(source.getMetric("sum.val1") != nullptr); + ASSERT_TRUE(source.getMetric("sum.valsum") != nullptr); std::vector<Metric::String> metrics( source.getPathsMatchingPrefix("a.val")); @@ -635,7 +606,7 @@ void MetricManagerTest::testPrintUtils() expected.push_back("val1"); expected.push_back("val2"); expected.push_back("valsum"); - CPPUNIT_ASSERT_EQUAL(expected, metrics); + EXPECT_EQ(expected, metrics); HttpTable table("mytable", "stuff"); table.colNames.push_back("values"); @@ -645,16 +616,16 @@ void MetricManagerTest::testPrintUtils() + getLongMetric("a.val2.value", source)); std::ostringstream ost; table.print(ost); - CPPUNIT_ASSERT_EQUAL(std::string( - "<h3>mytable</h3>\n" - "<table border=\"1\">\n" - "<tr><th>stuff</th><th>values</th></tr>\n" - "<tr><td>valsum</td><td align=\"right\">9</td></tr>\n" - "</table>\n" - ), ost.str()); + EXPECT_EQ(std::string( + "<h3>mytable</h3>\n" + "<table border=\"1\">\n" + "<tr><th>stuff</th><th>values</th></tr>\n" + "<tr><td>valsum</td><td align=\"right\">9</td></tr>\n" + "</table>\n" + ), ost.str()); } -void MetricManagerTest::testXmlOutput() +TEST_F(MetricManagerTest, test_xml_output) { FastOS_ThreadPool pool(256 * 1024); FakeTimer* timer = new FakeTimer(1000); @@ -677,7 +648,7 @@ void MetricManagerTest::testXmlOutput() "consumer[1].tags[0] snaptest\n", pool); - mm.takeSnapshots(mm.getMetricLock(), 1000); + takeSnapshots(mm, 1000); // Adding metrics to have some values in them mySet.val6.addValue(2); @@ -688,7 +659,7 @@ void MetricManagerTest::testXmlOutput() mySet.val10.b.val1.addValue(1); timer->_time = 1300; - mm.takeSnapshots(mm.getMetricLock(), 1300); + takeSnapshots(mm, 1300); std::string expected( "'<snapshot name=\"5 minute\" from=\"1000\" to=\"1300\" period=\"300\">\n" @@ -729,10 +700,10 @@ void MetricManagerTest::testXmlOutput() // Not bothering to match all the nitty gritty details as it will test // more than it needs to. Just be here in order to check on XML output // easily if needed. - CPPUNIT_ASSERT_EQUAL(expected, "'" + actual + "'"); + EXPECT_EQ(expected, "'" + actual + "'"); } -void MetricManagerTest::testJsonOutput() +TEST_F(MetricManagerTest, test_json_output) { FastOS_ThreadPool pool(256 * 1024); FakeTimer* timer = new FakeTimer(1000); @@ -752,7 +723,7 @@ void MetricManagerTest::testJsonOutput() "consumer[0].tags[0] snaptest\n", pool); - mm.takeSnapshots(mm.getMetricLock(), 1000); + takeSnapshots(mm, 1000); // Adding metrics to have some values in them mySet.val6.addValue(2); @@ -763,9 +734,9 @@ void MetricManagerTest::testJsonOutput() mySet.val10.b.val1.addValue(1); timer->_time = 1300; - mm.takeSnapshots(mm.getMetricLock(), 1300); + takeSnapshots(mm, 1300); - // Create json output + // Create json output vespalib::asciistream as; vespalib::JsonStream jsonStream(as); JsonWriter writer(jsonStream); @@ -776,7 +747,7 @@ void MetricManagerTest::testJsonOutput() } jsonStream.finalize(); std::string jsonData = as.str(); - // Parse it back + // Parse it back using namespace vespalib::slime; vespalib::Slime slime; size_t parsed = JsonFormat::decode(vespalib::Memory(jsonData), slime); @@ -786,51 +757,37 @@ void MetricManagerTest::testJsonOutput() std::ostringstream ost; ost << "Failed to parse JSON: '\n" << jsonData << "'\n:" << buffer.get().make_string() << "\n"; - CPPUNIT_ASSERT_EQUAL_MSG(ost.str(), jsonData.size(), parsed); + EXPECT_EQ(jsonData.size(), parsed) << ost.str(); } - // Verify some content - CPPUNIT_ASSERT_EQUAL_MSG(jsonData, 1000.0, - slime.get()["snapshot"]["from"].asDouble()); - CPPUNIT_ASSERT_EQUAL_MSG(jsonData, 1300.0, - slime.get()["snapshot"]["to"].asDouble()); - - CPPUNIT_ASSERT_EQUAL_MSG(jsonData, vespalib::string("temp.val6"), - slime.get()["values"][0]["name"].asString().make_string()); - CPPUNIT_ASSERT_EQUAL_MSG(jsonData, vespalib::string("val6 desc"), - slime.get()["values"][0]["description"].asString().make_string()); - CPPUNIT_ASSERT_EQUAL_MSG(jsonData, 2.0, - slime.get()["values"][0]["values"]["average"].asDouble()); - CPPUNIT_ASSERT_EQUAL_MSG(jsonData, 1.0, - slime.get()["values"][0]["values"]["count"].asDouble()); - CPPUNIT_ASSERT_EQUAL_MSG(jsonData, 2.0, - slime.get()["values"][0]["values"]["min"].asDouble()); - CPPUNIT_ASSERT_EQUAL_MSG(jsonData, 2.0, - slime.get()["values"][0]["values"]["max"].asDouble()); - CPPUNIT_ASSERT_EQUAL_MSG(jsonData, 2.0, - slime.get()["values"][0]["values"]["last"].asDouble()); - - CPPUNIT_ASSERT_EQUAL_MSG(jsonData, vespalib::string("temp.multisub.sum.valsum"), - slime.get()["values"][10]["name"].asString().make_string()); - CPPUNIT_ASSERT_EQUAL_MSG(jsonData, vespalib::string("valsum desc"), - slime.get()["values"][10]["description"].asString().make_string()); - CPPUNIT_ASSERT_EQUAL_MSG(jsonData, 10.0, - slime.get()["values"][10]["values"]["average"].asDouble()); - CPPUNIT_ASSERT_EQUAL_MSG(jsonData, 3.0, - slime.get()["values"][10]["values"]["count"].asDouble()); - CPPUNIT_ASSERT_EQUAL_MSG(jsonData, 1.0, - slime.get()["values"][10]["values"]["min"].asDouble()); - CPPUNIT_ASSERT_EQUAL_MSG(jsonData, 7.0, - slime.get()["values"][10]["values"]["max"].asDouble()); - CPPUNIT_ASSERT_EQUAL_MSG(jsonData, 10.0, - slime.get()["values"][10]["values"]["last"].asDouble()); - + // Verify some content + EXPECT_EQ(1000.0, slime.get()["snapshot"]["from"].asDouble()) << jsonData; + EXPECT_EQ(1300.0, slime.get()["snapshot"]["to"].asDouble()) << jsonData; + EXPECT_EQ(vespalib::string("temp.val6"), + slime.get()["values"][0]["name"].asString().make_string()) << jsonData; + EXPECT_EQ(vespalib::string("val6 desc"), + slime.get()["values"][0]["description"].asString().make_string()) << jsonData; + EXPECT_EQ(2.0, slime.get()["values"][0]["values"]["average"].asDouble()) << jsonData; + EXPECT_EQ(1.0, slime.get()["values"][0]["values"]["count"].asDouble()) << jsonData; + EXPECT_EQ(2.0, slime.get()["values"][0]["values"]["min"].asDouble()) << jsonData; + EXPECT_EQ(2.0, slime.get()["values"][0]["values"]["max"].asDouble()) << jsonData; + EXPECT_EQ(2.0, slime.get()["values"][0]["values"]["last"].asDouble()) << jsonData; + + EXPECT_EQ(vespalib::string("temp.multisub.sum.valsum"), + slime.get()["values"][10]["name"].asString().make_string()) << jsonData; + EXPECT_EQ(vespalib::string("valsum desc"), + slime.get()["values"][10]["description"].asString().make_string()) << jsonData; + EXPECT_EQ(10.0, slime.get()["values"][10]["values"]["average"].asDouble()) << jsonData; + EXPECT_EQ(3.0, slime.get()["values"][10]["values"]["count"].asDouble()) << jsonData; + EXPECT_EQ(1.0, slime.get()["values"][10]["values"]["min"].asDouble()) << jsonData; + EXPECT_EQ(7.0, slime.get()["values"][10]["values"]["max"].asDouble()) << jsonData; + EXPECT_EQ(10.0, slime.get()["values"][10]["values"]["last"].asDouble()) << jsonData; metrics::StateApiAdapter adapter(mm); vespalib::string normal = adapter.getMetrics("snapper"); - CPPUNIT_ASSERT_EQUAL(vespalib::string(jsonData), normal); + EXPECT_EQ(vespalib::string(jsonData), normal); vespalib::string total = adapter.getTotalMetrics("snapper"); - CPPUNIT_ASSERT(total.size() > 0); - CPPUNIT_ASSERT(total != normal); + EXPECT_GT(total.size(), 0); + EXPECT_NE(total, normal); } namespace { @@ -940,12 +897,10 @@ public: const Metric::Tags& dimensions) { // Works to do this outside of main test body because cppunit uses // exceptions for its failures. - CPPUNIT_ASSERT_EQUAL_MSG(_jsonText, name, nthMetricName(metricIndex)); - CPPUNIT_ASSERT_EQUAL_MSG(_jsonText, dimensions.size(), - nthMetricDimensionCount(metricIndex)); + EXPECT_EQ(name, nthMetricName(metricIndex)) << _jsonText; + EXPECT_EQ(dimensions.size(), nthMetricDimensionCount(metricIndex)) << _jsonText; for (auto& dim : dimensions) { - CPPUNIT_ASSERT_EQUAL_MSG(_jsonText, std::string(dim.value()), - nthMetricDimension(metricIndex, dim.key())); + EXPECT_EQ(std::string(dim.value()), nthMetricDimension(metricIndex, dim.key())) << _jsonText; } } }; @@ -973,10 +928,9 @@ DimensionTestMetricSet::DimensionTestMetricSet(MetricSet* owner) { } DimensionTestMetricSet::~DimensionTestMetricSet() { } -} // anon ns +} -void -MetricManagerTest::jsonOutputSupportsMultipleDimensions() +TEST_F(MetricManagerTest, json_output_supports_multiple_dimensions) { DimensionTestMetricSet mset; MetricSnapshotTestFixture fixture(*this, mset); @@ -1008,10 +962,9 @@ struct NestedDimensionTestMetricSet : MetricSet } }; -} // anon ns +} -void -MetricManagerTest::jsonOutputCanNestDimensionsFromMultipleMetricSets() +TEST_F(MetricManagerTest, json_output_can_nest_dimensions_from_multiple_metric_sets) { NestedDimensionTestMetricSet mset; MetricSnapshotTestFixture fixture(*this, mset); @@ -1060,10 +1013,9 @@ SameNamesTestMetricSet::SameNamesTestMetricSet() { } SameNamesTestMetricSet::~SameNamesTestMetricSet() { } -} // anon ns +} -void -MetricManagerTest::jsonOutputCanHaveMultipleSetsWithSameName() +TEST_F(MetricManagerTest, json_output_can_have_multiple_sets_with_same_name) { SameNamesTestMetricSet mset; MetricSnapshotTestFixture fixture(*this, mset); @@ -1082,7 +1034,7 @@ MetricManagerTest::jsonOutputCanHaveMultipleSetsWithSameName() {{"foo", "baz"}, {"fancy", "stuff"}}); } -void MetricManagerTest::testTextOutput() +TEST_F(MetricManagerTest, test_text_output) { FastOS_ThreadPool pool(256 * 1024); FakeTimer* timer = new FakeTimer(1000); @@ -1133,11 +1085,10 @@ void MetricManagerTest::testTextOutput() // Not bothering to match all the nitty gritty details as it will test // more than it needs to. Just be here in order to check on XML output // easily if needed. - CPPUNIT_ASSERT_EQUAL(expected, actual); + EXPECT_EQ(expected, actual); } -void -MetricManagerTest::textOutputSupportsDimensions() +TEST_F(MetricManagerTest, text_output_supports_dimensions) { NestedDimensionTestMetricSet mset; MetricSnapshotTestFixture fixture(*this, mset); @@ -1153,7 +1104,7 @@ MetricManagerTest::textOutputSupportsDimensions() "average=2 last=2 min=2 max=2 count=1 total=2\n" "outer{fancy:stuff}.temp{bar:hyperbar,foo:megafoo}.val2" "{baz:superbaz} count=1"); - CPPUNIT_ASSERT_EQUAL(expected, actual); + EXPECT_EQ(expected, actual); } namespace { @@ -1171,7 +1122,7 @@ namespace { }; } -void MetricManagerTest::testUpdateHooks() +TEST_F(MetricManagerTest, test_update_hooks) { std::ostringstream output; FastOS_ThreadPool pool(256 * 1024); @@ -1194,7 +1145,7 @@ void MetricManagerTest::testUpdateHooks() // All hooks should get called during initialization - // Initialize metric manager to get snapshots created. + // Initialize metric manager to get snapshots created. output << "Running init\n"; mm.init("raw:" "consumer[2]\n" @@ -1290,7 +1241,7 @@ void MetricManagerTest::testUpdateHooks() "1450: AIL called\n" ); std::string actual(output.str()); - CPPUNIT_ASSERT_EQUAL(expected, actual); + EXPECT_EQ(expected, actual); } -} // metrics +} diff --git a/metrics/src/tests/stresstest.cpp b/metrics/src/tests/stresstest.cpp index e20045ad2d4..4a6d2f4a2ea 100644 --- a/metrics/src/tests/stresstest.cpp +++ b/metrics/src/tests/stresstest.cpp @@ -1,26 +1,16 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/metrics/metrics.h> #include <vespa/metrics/loadmetric.hpp> -#include <vespa/metrics/summetric.hpp> #include <vespa/metrics/metricmanager.h> -#include <vespa/vdstestlib/cppunit/macros.h> +#include <vespa/metrics/metrics.h> +#include <vespa/metrics/summetric.hpp> +#include <vespa/vespalib/gtest/gtest.h> #include <vespa/log/log.h> LOG_SETUP(".metrics.test.stress"); namespace metrics { -struct StressTest : public CppUnit::TestFixture { - void testStress(); - - CPPUNIT_TEST_SUITE(StressTest); - CPPUNIT_TEST(testStress); - CPPUNIT_TEST_SUITE_END(); -}; - -CPPUNIT_TEST_SUITE_REGISTRATION(StressTest); - namespace { struct InnerMetricSet : public MetricSet { const LoadTypeSet& _loadTypes; @@ -88,49 +78,49 @@ OuterMetricSet::OuterMetricSet(const LoadTypeSet& lt, MetricSet* owner) OuterMetricSet::~OuterMetricSet() { } - struct Hammer : public document::Runnable { - using UP = std::unique_ptr<Hammer>; +struct Hammer : public document::Runnable { + using UP = std::unique_ptr<Hammer>; - OuterMetricSet& _metrics; - const LoadTypeSet& _loadTypes; - LoadType _nonexistingLoadType; + OuterMetricSet& _metrics; + const LoadTypeSet& _loadTypes; + LoadType _nonexistingLoadType; + + Hammer(OuterMetricSet& metrics, const LoadTypeSet& lt, + FastOS_ThreadPool& threadPool) + : _metrics(metrics), _loadTypes(lt), + _nonexistingLoadType(123, "nonexisting") + { + start(threadPool); + } + ~Hammer() { + stop(); + join(); + //std::cerr << "Loadgiver thread joined\n"; + } - Hammer(OuterMetricSet& metrics, const LoadTypeSet& lt, - FastOS_ThreadPool& threadPool) - : _metrics(metrics), _loadTypes(lt), - _nonexistingLoadType(123, "nonexisting") - { - start(threadPool); - } - ~Hammer() { - stop(); - join(); - //std::cerr << "Loadgiver thread joined\n"; + void run() override { + uint64_t i = 0; + while (running()) { + ++i; + setMetrics(i, _metrics._inner1); + setMetrics(i + 3, _metrics._inner2); + const LoadType& loadType(_loadTypes[i % _loadTypes.size()]); + setMetrics(i + 5, _metrics._load[loadType]); } + } - void run() override { - uint64_t i = 0; - while (running()) { - ++i; - setMetrics(i, _metrics._inner1); - setMetrics(i + 3, _metrics._inner2); - const LoadType& loadType(_loadTypes[i % _loadTypes.size()]); - setMetrics(i + 5, _metrics._load[loadType]); - } - } + void setMetrics(uint64_t val, InnerMetricSet& set) { + set._count.inc(val); + set._value1.addValue(val); + set._value2.addValue(val + 10); + set._load[_loadTypes[val % _loadTypes.size()]].addValue(val); + } +}; - void setMetrics(uint64_t val, InnerMetricSet& set) { - set._count.inc(val); - set._value1.addValue(val); - set._value2.addValue(val + 10); - set._load[_loadTypes[val % _loadTypes.size()]].addValue(val); - } - }; } -void -StressTest::testStress() +TEST(StressTest, test_stress) { LoadTypeSet loadTypes; loadTypes.push_back(LoadType(0, "default")); @@ -158,4 +148,4 @@ StressTest::testStress() // std::cerr << ost.str() << "\n"; } -} // metrics +} diff --git a/metrics/src/tests/testrunner.cpp b/metrics/src/tests/testrunner.cpp deleted file mode 100644 index 9f871997873..00000000000 --- a/metrics/src/tests/testrunner.cpp +++ /dev/null @@ -1,13 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include <vespa/vdstestlib/cppunit/cppunittestrunner.h> - -#include <vespa/log/log.h> -LOG_SETUP("storagecppunittests"); - -int -main(int argc, const char *argv[]) -{ - vdstestlib::CppUnitTestRunner testRunner; - return testRunner.run(argc, argv); -} diff --git a/node-maintainer/OWNERS b/node-maintainer/OWNERS deleted file mode 100644 index e030acdbc5b..00000000000 --- a/node-maintainer/OWNERS +++ /dev/null @@ -1 +0,0 @@ -freva diff --git a/node-maintainer/README.md b/node-maintainer/README.md deleted file mode 100644 index 33df558effb..00000000000 --- a/node-maintainer/README.md +++ /dev/null @@ -1,21 +0,0 @@ -<!-- Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> -# Node Admin Maintenance - -Executes maintenance jobs, such as deleting old logs, processing and reporting coredumps, on behalf of node-admin. -Node admin maintenance runs as a separate JVM from node-admin to make it possible to run it as root if needed. - -## Node Verification -Node verification for both hardware and spec. Hardware is verified by performing different benchmarking tasks, -while spec is verified by comparing information reported by the OS with the spec from node repository. - -### Execute examples -Spec verification and hardware benchmarks must be executed with config server host name as parameter - -SpecVerifier: -- sudo java -cp node-maintainer-jar-with-dependencies.jar com.yahoo.vespa.hosted.node.verification.spec.SpecVerifier cfg.1.hostname,cfg.2.hostname,... - -HardwareBenchmarker: -- sudo java -cp node-maintainer-jar-with-dependencies.jar com.yahoo.vespa.hosted.node.verification.hardware.HardwareBenchmarker cfg.1.hostname,cfg.2.hostname,... - -In "verification" you can find a TODO file where we have listed things we did not have time to implement. -Both programs have README explaining closer what it does. diff --git a/node-maintainer/pom.xml b/node-maintainer/pom.xml deleted file mode 100644 index 3c51bdc05be..00000000000 --- a/node-maintainer/pom.xml +++ /dev/null @@ -1,121 +0,0 @@ -<?xml version="1.0"?> -<!-- Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> -<project xmlns="http://maven.apache.org/POM/4.0.0" - xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" - xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 - http://maven.apache.org/xsd/maven-4.0.0.xsd"> - <modelVersion>4.0.0</modelVersion> - <parent> - <groupId>com.yahoo.vespa</groupId> - <artifactId>parent</artifactId> - <version>7-SNAPSHOT</version> - <relativePath>../parent/pom.xml</relativePath> - </parent> - - <artifactId>node-maintainer</artifactId> - <version>7-SNAPSHOT</version> - <packaging>jar</packaging> - <name>${project.artifactId}</name> - - <dependencies> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>defaults</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>vespajlib</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>config</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>org.apache.httpcomponents</groupId> - <artifactId>httpclient</artifactId> - </dependency> - <dependency> - <groupId>com.fasterxml.jackson.core</groupId> - <artifactId>jackson-databind</artifactId> - </dependency> - <dependency> - <groupId>io.airlift</groupId> - <artifactId>airline</artifactId> - <version>0.6</version> - </dependency> - - <dependency> - <groupId>org.hamcrest</groupId> - <artifactId>hamcrest-junit</artifactId> - <version>2.0.0.0</version> - <scope>test</scope> - </dependency> - <dependency> - <groupId>junit</groupId> - <artifactId>junit</artifactId> - <scope>test</scope> - </dependency> - <dependency> - <groupId>org.mockito</groupId> - <artifactId>mockito-core</artifactId> - <scope>test</scope> - </dependency> - <dependency> - <groupId>com.fasterxml.jackson.core</groupId> - <artifactId>jackson-annotations</artifactId> - <version>2.8.3</version> - </dependency> - <dependency> - <groupId>com.google.guava</groupId> - <artifactId>guava</artifactId> - </dependency> - </dependencies> - - <build> - <plugins> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-compiler-plugin</artifactId> - </plugin> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-surefire-plugin</artifactId> - <configuration> - <redirectTestOutputToFile>${test.hide}</redirectTestOutputToFile> - <environmentVariables> - <VESPA_HOME>/opt/vespa</VESPA_HOME> - </environmentVariables> - </configuration> - </plugin> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-shade-plugin</artifactId> - <configuration> - <finalName>${project.artifactId}-jar-with-dependencies</finalName> - <filters> - <filter> - <!-- Don't include signature files from bouncycastle in uber jar. --> - <artifact>*:*</artifact> - <excludes> - <exclude>META-INF/*.SF</exclude> - <exclude>META-INF/*.DSA</exclude> - <exclude>META-INF/*.RSA</exclude> - </excludes> - </filter> - </filters> - </configuration> - <executions> - <execution> - <phase>package</phase> - <goals> - <goal>shade</goal> - </goals> - </execution> - </executions> - </plugin> - </plugins> - </build> -</project> diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/maintainer/CoreCollector.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/maintainer/CoreCollector.java deleted file mode 100644 index fef5a695db6..00000000000 --- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/maintainer/CoreCollector.java +++ /dev/null @@ -1,200 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.maintainer; - -import com.yahoo.collections.Pair; -import com.yahoo.system.ProcessExecuter; - -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.Arrays; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.logging.Level; -import java.util.logging.Logger; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -import static com.yahoo.vespa.defaults.Defaults.getDefaults; - -/** - * Takes in a compressed (lz4) or uncompressed core dump and collects relevant metadata. - * - * @author freva - */ -public class CoreCollector { - private static final String GDB_PATH = getDefaults().underVespaHome("bin64/gdb"); - private static final String LZ4_PATH = getDefaults().underVespaHome("bin64/lz4"); - private static final Pattern CORE_GENERATOR_PATH_PATTERN = Pattern.compile("^Core was generated by `(?<path>.*?)'.$"); - private static final Pattern EXECFN_PATH_PATTERN = Pattern.compile("^.* execfn: '(?<path>.*?)'"); - private static final Pattern FROM_PATH_PATTERN = Pattern.compile("^.* from '(?<path>.*?)'"); - private static final Pattern TOTAL_MEMORY_PATTERN = Pattern.compile("^MemTotal:\\s*(?<totalMem>\\d+) kB$", Pattern.MULTILINE); - - private static final Logger logger = Logger.getLogger(CoreCollector.class.getName()); - private final ProcessExecuter processExecuter; - - public CoreCollector(ProcessExecuter processExecuter) { - this.processExecuter = processExecuter; - } - - List<String> readInstallState(Path installStatePath) throws IOException { - Pair<Integer, String> result = processExecuter.exec(new String[]{"cat", installStatePath.toString()}); - - if (result.getFirst() != 0) { - throw new RuntimeException("Failed to read install state file at: " + installStatePath + ", result: " + result); - } - return Arrays.asList(result.getSecond().split("\n")); - } - - List<String> readRpmPackages() throws IOException { - Pair<Integer, String> result = processExecuter.exec(new String[]{"rpm", "-qa"}); - - if (result.getFirst() != 0) { - throw new RuntimeException("Failed to read RPM packages " + result); - } - return Arrays.asList(result.getSecond().split("\n")); - } - - Path readBinPathFallback(Path coredumpPath) throws IOException { - String command = GDB_PATH + " -n -batch -core " + coredumpPath + " | grep \'^Core was generated by\'"; - String[] wrappedCommand = new String[] {"/bin/sh", "-c", command}; - Pair<Integer, String> result = processExecuter.exec(wrappedCommand); - - Matcher matcher = CORE_GENERATOR_PATH_PATTERN.matcher(result.getSecond()); - if (! matcher.find()) { - throw new RuntimeException(String.format("Failed to extract binary path from GDB, result: %s, command: %s", - result, Arrays.toString(wrappedCommand))); - } - return Paths.get(matcher.group("path").split(" ")[0]); - } - - Path readBinPath(Path coredumpPath) throws IOException { - String[] command = new String[] {"file", coredumpPath.toString()}; - try { - Pair<Integer, String> result = processExecuter.exec(command); - - if (result.getFirst() != 0) { - throw new RuntimeException("file command failed with " + result); - } - - Matcher execfnMatcher = EXECFN_PATH_PATTERN.matcher(result.getSecond()); - if (execfnMatcher.find()) { - return Paths.get(execfnMatcher.group("path").split(" ")[0]); - } - - Matcher fromMatcher = FROM_PATH_PATTERN.matcher(result.getSecond()); - if (fromMatcher.find()) { - return Paths.get(fromMatcher.group("path").split(" ")[0]); - } - } catch (Throwable e) { - logger.log(Level.WARNING, String.format("Failed getting bin path, command: %s. " + - "Trying fallback instead", Arrays.toString(command)), e); - } - - return readBinPathFallback(coredumpPath); - } - - List<String> readBacktrace(Path coredumpPath, Path binPath, boolean allThreads) throws IOException { - String threads = allThreads ? "thread apply all bt" : "bt"; - String[] command = new String[]{GDB_PATH, "-n", "-ex", threads, "-batch", binPath.toString(), coredumpPath.toString()}; - Pair<Integer, String> result = processExecuter.exec(command); - if (result.getFirst() != 0) { - throw new RuntimeException("Failed to read backtrace " + result + ", Command: " + Arrays.toString(command)); - } - return Arrays.asList(result.getSecond().split("\n")); - } - - Map<String, Object> collect(Path coredumpPath, Optional<Path> installStatePath) { - Map<String, Object> data = new LinkedHashMap<>(); - try { - coredumpPath = compressCoredump(coredumpPath); - } catch (IOException e) { - logger.log(Level.WARNING, "Failed compressing/decompressing core dump", e); - } - - try { - Path binPath = readBinPath(coredumpPath); - - data.put("bin_path", binPath.toString()); - data.put("backtrace", readBacktrace(coredumpPath, binPath, false)); - data.put("backtrace_all_threads", readBacktrace(coredumpPath, binPath, true)); - } catch (Throwable e) { - logger.log(Level.WARNING, "Failed to extract backtrace", e); - } - - installStatePath.ifPresent(installState -> { - try { - data.put("install_state", readInstallState(installState)); - } catch (Exception e) { - logger.log(Level.WARNING, "Failed to read install state", e); - } - - try { - data.put("rpm_packages", readRpmPackages()); - } catch (Exception e) { - logger.log(Level.WARNING, "Failed to read RPM packages", e); - } - }); - - try { - deleteDecompressedCoredump(coredumpPath); - } catch (IOException e) { - logger.log(Level.WARNING, "Failed to delete decompressed core dump", e); - } - return data; - } - - - /** - * This method will either compress or decompress the core dump if the input path is to a decompressed or - * compressed core dump, respectively. - * - * @return Path to the decompressed core dump - */ - private Path compressCoredump(Path coredumpPath) throws IOException { - if (! coredumpPath.toString().endsWith(".lz4")) { - processExecuter.exec( - new String[]{LZ4_PATH, "-f", coredumpPath.toString(), coredumpPath.toString() + ".lz4"}); - return coredumpPath; - - } else { - if (!diskSpaceAvailable(coredumpPath)) { - throw new RuntimeException("Not decompressing " + coredumpPath + " due to not enough disk space available"); - } - - Path decompressedPath = Paths.get(coredumpPath.toString().replaceFirst("\\.lz4$", "")); - Pair<Integer, String> result = processExecuter.exec( - new String[] {LZ4_PATH, "-f", "-d", coredumpPath.toString(), decompressedPath.toString()}); - if (result.getFirst() != 0) { - throw new RuntimeException("Failed to decompress file " + coredumpPath + ": " + result); - } - return decompressedPath; - } - } - - /** - * Delete the core dump unless: - * - The file is compressed - * - There is no compressed file (i.e. it was not decompressed in the first place) - */ - void deleteDecompressedCoredump(Path coredumpPath) throws IOException { - if (! coredumpPath.toString().endsWith(".lz4") && Paths.get(coredumpPath.toString() + ".lz4").toFile().exists()) { - Files.delete(coredumpPath); - } - } - - private boolean diskSpaceAvailable(Path path) throws IOException { - // TODO: If running inside container, check against container memory size, not for the enitre host - String memInfo = new String(Files.readAllBytes(Paths.get("/proc/meminfo"))); - return path.toFile().getFreeSpace() > parseTotalMemorySize(memInfo); - } - - int parseTotalMemorySize(String memInfo) { - Matcher matcher = TOTAL_MEMORY_PATTERN.matcher(memInfo); - if (!matcher.find()) throw new RuntimeException("Could not parse meminfo: " + memInfo); - return Integer.valueOf(matcher.group("totalMem")); - } -} diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/maintainer/CoredumpHandler.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/maintainer/CoredumpHandler.java deleted file mode 100644 index 0c118e2dc6c..00000000000 --- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/maintainer/CoredumpHandler.java +++ /dev/null @@ -1,173 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.maintainer; - -import com.fasterxml.jackson.databind.ObjectMapper; -import org.apache.http.HttpHeaders; -import org.apache.http.client.methods.CloseableHttpResponse; -import org.apache.http.client.methods.HttpPost; -import org.apache.http.entity.StringEntity; -import org.apache.http.impl.client.CloseableHttpClient; -import org.apache.http.util.EntityUtils; - -import java.io.BufferedReader; -import java.io.IOException; -import java.io.InputStreamReader; -import java.nio.file.Files; -import java.nio.file.Path; -import java.time.Duration; -import java.util.Comparator; -import java.util.HashMap; -import java.util.Map; -import java.util.Optional; -import java.util.UUID; -import java.util.logging.Level; -import java.util.logging.Logger; -import java.util.stream.Collectors; - -/** - * Finds coredumps, collects metadata and reports them - * - * @author freva - */ -class CoredumpHandler { - - static final String PROCESSING_DIRECTORY_NAME = "processing"; - static final String METADATA_FILE_NAME = "metadata.json"; - - private final Logger logger = Logger.getLogger(CoredumpHandler.class.getName()); - private final ObjectMapper objectMapper = new ObjectMapper(); - - private final CloseableHttpClient httpClient; - private final CoreCollector coreCollector; - private final Path coredumpsPath; - private final Path doneCoredumpsPath; - private final Map<String, Object> nodeAttributes; - private final Optional<Path> installStatePath; - private final String feedEndpoint; - - public CoredumpHandler(CloseableHttpClient httpClient, CoreCollector coreCollector, Path coredumpsPath, Path doneCoredumpsPath, - Map<String, Object> nodeAttributes, Optional<Path> installStatePath, String feedEndpoint) { - this.httpClient = httpClient; - this.coreCollector = coreCollector; - this.coredumpsPath = coredumpsPath; - this.doneCoredumpsPath = doneCoredumpsPath; - this.nodeAttributes = nodeAttributes; - this.installStatePath = installStatePath; - this.feedEndpoint = feedEndpoint; - } - - public void processAll() throws IOException { - removeJavaCoredumps(); - handleNewCoredumps(); - removeOldCoredumps(); - } - - private void removeJavaCoredumps() throws IOException { - if (! coredumpsPath.toFile().isDirectory()) return; - FileHelper.deleteFiles(coredumpsPath, Duration.ZERO, Optional.of("^java_pid.*\\.hprof$"), false); - } - - private void removeOldCoredumps() throws IOException { - if (! doneCoredumpsPath.toFile().isDirectory()) return; - FileHelper.deleteDirectories(doneCoredumpsPath, Duration.ofDays(10), Optional.empty()); - } - - private void handleNewCoredumps() { - Path processingCoredumps = enqueueCoredumps(); - processAndReportCoredumps(processingCoredumps); - } - - - /** - * Moves a coredump to a new directory under the processing/ directory. Limit to only processing - * one coredump at the time, starting with the oldest. - */ - Path enqueueCoredumps() { - Path processingCoredumpsPath = coredumpsPath.resolve(PROCESSING_DIRECTORY_NAME); - processingCoredumpsPath.toFile().mkdirs(); - if (!FileHelper.listContentsOfDirectory(processingCoredumpsPath).isEmpty()) return processingCoredumpsPath; - - FileHelper.listContentsOfDirectory(coredumpsPath).stream() - .filter(path -> path.toFile().isFile() && ! path.getFileName().toString().startsWith(".")) - .min((Comparator.comparingLong(o -> o.toFile().lastModified()))) - .ifPresent(coredumpPath -> { - try { - enqueueCoredumpForProcessing(coredumpPath, processingCoredumpsPath); - } catch (Throwable e) { - logger.log(Level.WARNING, "Failed to process coredump " + coredumpPath, e); - } - }); - - return processingCoredumpsPath; - } - - void processAndReportCoredumps(Path processingCoredumpsPath) { - doneCoredumpsPath.toFile().mkdirs(); - - FileHelper.listContentsOfDirectory(processingCoredumpsPath).stream() - .filter(path -> path.toFile().isDirectory()) - .forEach(coredumpDirectory -> { - try { - String metadata = collectMetadata(coredumpDirectory, nodeAttributes); - report(coredumpDirectory, metadata); - finishProcessing(coredumpDirectory); - } catch (Throwable e) { - logger.log(Level.WARNING, "Failed to report coredump " + coredumpDirectory, e); - } - }); - } - - Path enqueueCoredumpForProcessing(Path coredumpPath, Path processingCoredumpsPath) throws IOException { - // Make coredump readable - coredumpPath.toFile().setReadable(true, false); - - // Create new directory for this coredump and move it into it - Path folder = processingCoredumpsPath.resolve(UUID.randomUUID().toString()); - folder.toFile().mkdirs(); - return Files.move(coredumpPath, folder.resolve(coredumpPath.getFileName())); - } - - String collectMetadata(Path coredumpDirectory, Map<String, Object> nodeAttributes) throws IOException { - Path metadataPath = coredumpDirectory.resolve(METADATA_FILE_NAME); - if (!Files.exists(metadataPath)) { - Path coredumpPath = FileHelper.listContentsOfDirectory(coredumpDirectory).stream().findFirst() - .orElseThrow(() -> new RuntimeException("No coredump file found in processing directory " + coredumpDirectory)); - Map<String, Object> metadata = coreCollector.collect(coredumpPath, installStatePath); - metadata.putAll(nodeAttributes); - - Map<String, Object> fields = new HashMap<>(); - fields.put("fields", metadata); - - String metadataFields = objectMapper.writeValueAsString(fields); - Files.write(metadataPath, metadataFields.getBytes()); - return metadataFields; - } else { - return new String(Files.readAllBytes(metadataPath)); - } - } - - void report(Path coredumpDirectory, String metadata) throws IOException { - // Use core dump UUID as document ID - String documentId = coredumpDirectory.getFileName().toString(); - - HttpPost post = new HttpPost(feedEndpoint + "/" + documentId); - post.setHeader(HttpHeaders.CONTENT_TYPE, "application/json"); - post.setEntity(new StringEntity(metadata)); - - try (CloseableHttpResponse response = httpClient.execute(post)) { - if (response.getStatusLine().getStatusCode() / 100 != 2) { - String result = new BufferedReader(new InputStreamReader(response.getEntity().getContent())) - .lines().collect(Collectors.joining("\n")); - throw new RuntimeException("POST to " + post.getURI() + " failed with HTTP: " + - response.getStatusLine().getStatusCode() + " [" + result + "]"); - } - EntityUtils.consume(response.getEntity()); - } - logger.info("Successfully reported coredump " + documentId); - } - - void finishProcessing(Path coredumpDirectory) throws IOException { - Files.move(coredumpDirectory, doneCoredumpsPath.resolve(coredumpDirectory.getFileName())); - } - -} diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/maintainer/FileHelper.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/maintainer/FileHelper.java deleted file mode 100644 index 7b93e7ad98d..00000000000 --- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/maintainer/FileHelper.java +++ /dev/null @@ -1,177 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.maintainer; - -import java.io.IOException; -import java.io.UncheckedIOException; -import java.nio.file.Files; -import java.nio.file.LinkOption; -import java.nio.file.NoSuchFileException; -import java.nio.file.Path; -import java.nio.file.attribute.FileTime; -import java.time.Duration; -import java.time.Instant; -import java.util.Collections; -import java.util.Comparator; -import java.util.List; -import java.util.Optional; -import java.util.logging.Logger; -import java.util.regex.Pattern; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -/** - * @author freva - */ -public class FileHelper { - private static final Logger logger = Logger.getLogger(FileHelper.class.getSimpleName()); - - /** - * (Recursively) deletes files if they match all the criteria, also deletes empty directories. - * - * @param basePath Base path from where to start the search - * @param maxAge Delete files older (last modified date) than maxAge - * @param fileNameRegex Delete files where filename matches fileNameRegex - * @param recursive Delete files in sub-directories (with the same criteria) - */ - public static void deleteFiles(Path basePath, Duration maxAge, Optional<String> fileNameRegex, boolean recursive) throws IOException { - Pattern fileNamePattern = fileNameRegex.map(Pattern::compile).orElse(null); - - for (Path path : listContentsOfDirectory(basePath)) { - if (Files.isDirectory(path)) { - if (recursive) { - deleteFiles(path, maxAge, fileNameRegex, true); - if (listContentsOfDirectory(path).isEmpty() && !Files.deleteIfExists(path)) { - logger.warning("Could not delete directory: " + path.toAbsolutePath()); - } - } - } else if (isPatternMatchingFilename(fileNamePattern, path) && - isTimeSinceLastModifiedMoreThan(path, maxAge)) { - if (! Files.deleteIfExists(path)) { - logger.warning("Could not delete file: " + path.toAbsolutePath()); - } - } - } - } - - /** - * Deletes all files in target directory except the n most recent (by modified date) - * - * @param basePath Base path to delete from - * @param nMostRecentToKeep Number of most recent files to keep - */ - static void deleteFilesExceptNMostRecent(Path basePath, int nMostRecentToKeep) throws IOException { - if (nMostRecentToKeep < 1) { - throw new IllegalArgumentException("Number of files to keep must be a positive number"); - } - - List<Path> pathsInDeleteDir = listContentsOfDirectory(basePath).stream() - .filter(Files::isRegularFile) - .sorted(Comparator.comparing(FileHelper::getLastModifiedTime)) - .skip(nMostRecentToKeep) - .collect(Collectors.toList()); - - for (Path path : pathsInDeleteDir) { - if (!Files.deleteIfExists(path)) { - logger.warning("Could not delete file: " + path.toAbsolutePath()); - } - } - } - - static void deleteFilesLargerThan(Path basePath, long sizeInBytes) throws IOException { - for (Path path : listContentsOfDirectory(basePath)) { - if (Files.isDirectory(path)) { - deleteFilesLargerThan(path, sizeInBytes); - } else { - if (Files.size(path) > sizeInBytes && !Files.deleteIfExists(path)) { - logger.warning("Could not delete file: " + path.toAbsolutePath()); - } - } - } - } - - /** - * Deletes directories and their contents if they match all the criteria - * - * @param basePath Base path to delete the directories from - * @param maxAge Delete directories older (last modified date) than maxAge - * @param dirNameRegex Delete directories where directory name matches dirNameRegex - */ - public static void deleteDirectories(Path basePath, Duration maxAge, Optional<String> dirNameRegex) throws IOException { - Pattern dirNamePattern = dirNameRegex.map(Pattern::compile).orElse(null); - - for (Path path : listContentsOfDirectory(basePath)) { - if (Files.isDirectory(path) && isPatternMatchingFilename(dirNamePattern, path)) { - boolean mostRecentFileModifiedBeforeMaxAge = getMostRecentlyModifiedFileIn(path) - .map(mostRecentlyModified -> isTimeSinceLastModifiedMoreThan(mostRecentlyModified, maxAge)) - .orElse(true); - - if (mostRecentFileModifiedBeforeMaxAge) { - deleteFiles(path, Duration.ZERO, Optional.empty(), true); - if (listContentsOfDirectory(path).isEmpty() && !Files.deleteIfExists(path)) { - logger.warning("Could not delete directory: " + path.toAbsolutePath()); - } - } - } - } - } - - /** - * Similar to rm -rf file: - * - It's not an error if file doesn't exist - * - If file is a directory, it and all content is removed - * - For symlinks: Only the symlink is removed, not what the symlink points to - */ - public static void recursiveDelete(Path basePath) throws IOException { - if (Files.isDirectory(basePath)) { - for (Path path : listContentsOfDirectory(basePath)) { - recursiveDelete(path); - } - } - - Files.deleteIfExists(basePath); - } - - public static void moveIfExists(Path from, Path to) throws IOException { - if (Files.exists(from)) { - Files.move(from, to); - } - } - - private static Optional<Path> getMostRecentlyModifiedFileIn(Path basePath) throws IOException { - return Files.walk(basePath).max(Comparator.comparing(FileHelper::getLastModifiedTime)); - } - - private static boolean isTimeSinceLastModifiedMoreThan(Path path, Duration duration) { - Instant nowMinusDuration = Instant.now().minus(duration); - Instant lastModified = getLastModifiedTime(path).toInstant(); - - // Return true also if they are equal for test stability - // (lastModified <= nowMinusDuration) is the same as !(lastModified > nowMinusDuration) - return !lastModified.isAfter(nowMinusDuration); - } - - private static boolean isPatternMatchingFilename(Pattern pattern, Path path) { - return pattern == null || pattern.matcher(path.getFileName().toString()).find(); - } - - /** - * @return list all files in a directory, returns empty list if directory does not exist - */ - public static List<Path> listContentsOfDirectory(Path basePath) { - try (Stream<Path> directoryStream = Files.list(basePath)) { - return directoryStream.collect(Collectors.toList()); - } catch (NoSuchFileException ignored) { - return Collections.emptyList(); - } catch (IOException e) { - throw new UncheckedIOException("Failed to list contents of directory " + basePath.toAbsolutePath(), e); - } - } - - static FileTime getLastModifiedTime(Path path) { - try { - return Files.getLastModifiedTime(path, LinkOption.NOFOLLOW_LINKS); - } catch (IOException e) { - throw new UncheckedIOException("Failed to get last modified time of " + path.toAbsolutePath(), e); - } - } -} diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/maintainer/Maintainer.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/maintainer/Maintainer.java deleted file mode 100644 index 3bf62e60481..00000000000 --- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/maintainer/Maintainer.java +++ /dev/null @@ -1,203 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.maintainer; - -import com.yahoo.log.LogSetup; -import com.yahoo.slime.ArrayTraverser; -import com.yahoo.slime.Inspector; -import com.yahoo.slime.Type; -import com.yahoo.system.ProcessExecuter; -import com.yahoo.vespa.config.SlimeUtils; -import org.apache.http.client.config.RequestConfig; -import org.apache.http.impl.client.CloseableHttpClient; -import org.apache.http.impl.client.HttpClientBuilder; - -import java.io.IOException; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.time.Duration; -import java.util.HashMap; -import java.util.Map; -import java.util.Optional; -import java.util.concurrent.atomic.AtomicInteger; - -/** - * @author freva - */ -public class Maintainer { - - private static final CoreCollector coreCollector = new CoreCollector(new ProcessExecuter()); - - public static void main(String[] args) { - LogSetup.initVespaLogging("node-maintainer"); - if (args.length != 1) { - throw new RuntimeException("Expected only 1 argument - a JSON list of maintainer jobs to execute"); - } - - Inspector object = SlimeUtils.jsonToSlime(args[0].getBytes()).get(); - if (object.type() != Type.ARRAY) { - throw new IllegalArgumentException("Expected a list of maintainer jobs to execute"); - } - - // Variable must be effectively final to be used in lambda expression - AtomicInteger numberOfJobsFailed = new AtomicInteger(0); - object.traverse((ArrayTraverser) (int i, Inspector item) -> { - try { - String type = getFieldOrFail(item, "type").asString(); - Inspector arguments = getFieldOrFail(item, "arguments"); - parseMaintenanceJob(type, arguments); - } catch (Exception e) { - System.err.println("Failed executing job: " + item.toString()); - e.printStackTrace(); - numberOfJobsFailed.incrementAndGet(); - } - }); - - if (numberOfJobsFailed.get() > 0) { - System.err.println(numberOfJobsFailed.get() + " of jobs has failed"); - System.exit(1); - } - } - - private static void parseMaintenanceJob(String type, Inspector arguments) { - if (arguments.type() != Type.OBJECT) { - throw new IllegalArgumentException("Expected a 'arguments' to be an object"); - } - - switch (type) { - case "delete-files": - parseDeleteFilesJob(arguments); - break; - - case "delete-directories": - parseDeleteDirectoriesJob(arguments); - break; - - case "recursive-delete": - parseRecursiveDelete(arguments); - break; - - case "move-files": - parseMoveFiles(arguments); - break; - - case "handle-core-dumps": - parseHandleCoreDumps(arguments); - break; - - default: - throw new IllegalArgumentException("Unknown job: " + type); - } - } - - private static void parseDeleteFilesJob(Inspector arguments) { - Path basePath = Paths.get(getFieldOrFail(arguments, "basePath").asString()); - Duration maxAge = Duration.ofSeconds(getFieldOrFail(arguments, "maxAgeSeconds").asLong()); - Optional<String> fileNameRegex = SlimeUtils.optionalString(arguments.field("fileNameRegex")); - boolean recursive = getFieldOrFail(arguments, "recursive").asBool(); - try { - FileHelper.deleteFiles(basePath, maxAge, fileNameRegex, recursive); - } catch (IOException e) { - throw new RuntimeException("Failed deleting files under " + basePath.toAbsolutePath() + - fileNameRegex.map(regex -> ", matching '" + regex + "'").orElse("") + - ", " + (recursive ? "" : "not ") + "recursively" + - " and older than " + maxAge, e); - } - } - - private static void parseDeleteDirectoriesJob(Inspector arguments) { - Path basePath = Paths.get(getFieldOrFail(arguments, "basePath").asString()); - Duration maxAge = Duration.ofSeconds(getFieldOrFail(arguments, "maxAgeSeconds").asLong()); - Optional<String> dirNameRegex = SlimeUtils.optionalString(arguments.field("dirNameRegex")); - try { - FileHelper.deleteDirectories(basePath, maxAge, dirNameRegex); - } catch (IOException e) { - throw new RuntimeException("Failed deleting directories under " + basePath.toAbsolutePath() + - dirNameRegex.map(regex -> ", matching '" + regex + "'").orElse("") + - " and older than " + maxAge, e); - } - } - - private static void parseRecursiveDelete(Inspector arguments) { - Path basePath = Paths.get(getFieldOrFail(arguments, "path").asString()); - try { - FileHelper.recursiveDelete(basePath); - } catch (IOException e) { - throw new RuntimeException("Failed deleting " + basePath.toAbsolutePath(), e); - } - } - - private static void parseMoveFiles(Inspector arguments) { - Path from = Paths.get(getFieldOrFail(arguments, "from").asString()); - Path to = Paths.get(getFieldOrFail(arguments, "to").asString()); - - try { - FileHelper.moveIfExists(from, to); - } catch (IOException e) { - throw new RuntimeException("Failed moving from " + from.toAbsolutePath() + ", to " + to.toAbsolutePath(), e); - } - } - - private static void parseHandleCoreDumps(Inspector arguments) { - Path coredumpsPath = Paths.get(getFieldOrFail(arguments, "coredumpsPath").asString()); - Path doneCoredumpsPath = Paths.get(getFieldOrFail(arguments, "doneCoredumpsPath").asString()); - Map<String, Object> attributesMap = parseMap(arguments); - Optional<Path> installStatePath = SlimeUtils.optionalString(arguments.field("installStatePath")).map(Paths::get); - String feedEndpoint = getFieldOrFail(arguments, "feedEndpoint").asString(); - - try (CloseableHttpClient httpClient = createHttpClient(Duration.ofSeconds(5))) { - CoredumpHandler coredumpHandler = new CoredumpHandler(httpClient, coreCollector, coredumpsPath, - doneCoredumpsPath, attributesMap, installStatePath, - feedEndpoint); - coredumpHandler.processAll(); - } catch (IOException e) { - throw new RuntimeException("Failed processing coredumps at " + coredumpsPath.toAbsolutePath() + - ", moving fished dumps to " + doneCoredumpsPath.toAbsolutePath(), e); - } - } - - private static Map<String, Object> parseMap(Inspector object) { - Map<String, Object> map = new HashMap<>(); - getFieldOrFail(object, "attributes").traverse((String key, Inspector value) -> { - switch (value.type()) { - case BOOL: - map.put(key, value.asBool()); - break; - case LONG: - map.put(key, value.asLong()); - break; - case DOUBLE: - map.put(key, value.asDouble()); - break; - case STRING: - map.put(key, value.asString()); - break; - default: - throw new IllegalArgumentException("Invalid attribute for key '" + key + "', value " + value); - } - }); - return map; - } - - private static Inspector getFieldOrFail(Inspector object, String key) { - Inspector out = object.field(key); - if (out.type() == Type.NIX) { - throw new IllegalArgumentException("Key '" + key + "' was not found!"); - } - return out; - } - - private static CloseableHttpClient createHttpClient(Duration timeout) { - int timeoutInMillis = (int) timeout.toMillis(); - return HttpClientBuilder.create() - .setUserAgent("node-maintainer") - .setDefaultRequestConfig(RequestConfig.custom() - .setConnectTimeout(timeoutInMillis) - .setConnectionRequestTimeout(timeoutInMillis) - .setSocketTimeout(timeoutInMillis) - .build()) - .setMaxConnTotal(100) - .setMaxConnPerRoute(10) - .build(); - } - -} diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/Main.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/Main.java deleted file mode 100644 index a33dd07dff2..00000000000 --- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/Main.java +++ /dev/null @@ -1,84 +0,0 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.verification; - -import com.fasterxml.jackson.databind.ObjectMapper; -import com.yahoo.log.LogLevel; -import com.yahoo.log.LogSetup; -import com.yahoo.vespa.hosted.node.verification.commons.CommandExecutor; -import com.yahoo.vespa.hosted.node.verification.commons.report.HardwareDivergenceReport; -import com.yahoo.vespa.hosted.node.verification.hardware.HardwareBenchmarker; -import com.yahoo.vespa.hosted.node.verification.spec.SpecVerifier; -import io.airlift.command.Cli; -import io.airlift.command.Help; -import io.airlift.command.Option; - -import java.io.IOException; -import java.util.logging.Level; -import java.util.logging.Logger; - -/** - * @author freva - */ -public class Main { - private static final String EXECUTABLE_NAME = "node-verifier"; - private static final Logger logger = Logger.getLogger(Main.class.getName()); - private static final ObjectMapper om = new ObjectMapper(); - - public static void main(String[] args) { - LogSetup.initVespaLogging(EXECUTABLE_NAME); - - try { - System.out.println(execute(args, new CommandExecutor())); - } catch (Exception e) { - logger.log(LogLevel.ERROR, "Something went wrong", e); - System.exit(1); - } - } - - @SuppressWarnings("unchecked") - public static String execute(String[] args, CommandExecutor commandExecutor) throws IOException { - Cli.CliBuilder<Object> builder = Cli.builder(EXECUTABLE_NAME) - .withDescription("Verifies that node meets the expected specification and benchmarks") - .withDefaultCommand(Help.class) - .withCommands(Help.class, SpecVerifier.class, HardwareBenchmarker.class); - - Object command = builder.build().parse(args); - if (command instanceof VerifierCommand) { - HardwareDivergenceReport report = ((VerifierCommand) command).getPreviousHardwareDivergence(); - ((VerifierCommand) command).run(report, commandExecutor); - return hardwareDivergenceReportToString(report); - } else if (command instanceof Runnable) { - ((Runnable) command).run(); - return ""; - } - - throw new RuntimeException("Unknown command class " + command.getClass().getName()); - } - - public static abstract class VerifierCommand { - @Option(name = {"-h", "--divergence"}, description = "JSON of the previous hardware divergence report") - private String hardwareDivergence; - - private HardwareDivergenceReport getPreviousHardwareDivergence() { - if (hardwareDivergence == null) { - return new HardwareDivergenceReport(); - } - try { - return om.readValue(hardwareDivergence, HardwareDivergenceReport.class); - } catch (IOException e) { - logger.log(Level.WARNING, "Failed to parse hardware divergence:\n" + hardwareDivergence, e.getMessage()); - return new HardwareDivergenceReport(); - } - } - - protected abstract void run(HardwareDivergenceReport hardwareDivergenceReport, CommandExecutor commandExecutor); - } - - private static String hardwareDivergenceReportToString(HardwareDivergenceReport hardwareDivergenceReport) throws IOException { - if (hardwareDivergenceReport.isHardwareDivergenceReportEmpty()) { - return "null"; - } else { - return om.writeValueAsString(hardwareDivergenceReport); - } - } -} diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/TODO.md b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/TODO.md deleted file mode 100644 index e7869adb66a..00000000000 --- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/TODO.md +++ /dev/null @@ -1,31 +0,0 @@ -<!-- Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> -#TODO -Here we have listed things we did not have time to do during the summer, but that we think can be implemented later. - -##Spec -- The net interface speed is at the moment approved if it is over 1 000MB/s. Since some nodes are supposed to have -10 000MB/s a feature could be to either add information about interface speed in node repo or use the flavor to decide -if the interface speed is correct or not. -- In HardwareNodeComparator the spec found in node repo and on the node are compared. We set a threshold on 5%, meaning -if a the value from the node is more than 5% away from what is said in node repo (+-), then we say it is a bad value. -If this threshold is too high or too low, it has to be changed in this class. - -##Benchmark -- BenchmarkResultInspector is the class that decides whether a benchmark result is ok or if the result should be - reported. The values that decides this are not given very much thought and it could be an idea to check these. -- Benchmark is not running on docker hosts since there is not yet found a solution to only start benchmarking at reboot. -Spec verification runs every hour on node-admin, but since node-admin reboots too often, benchmarking is at the moment -not running here. - -##Reporting / Node repo -- Since HardwareDivergenceReport is printed as a json string and then reported to noderepo as a string, a feature could -be to not store HardwareDivergence as a string in node repo, as it is now, but as a json. This will be mostly changes -outside of this code, but then the member variable HardwareDivergence in NodeRepoJsonModel have to be change from string -to HardwareDivergenceReport which will cause some work in the Reporter class. -- No actions are now taken if there is anything wrong with the node. The information is only uploaded to node repo. -Here it is room for improvements and some possible solutions are: - - Continue only reporting to node repo, but have a program that scans through all nodes and makes a list of those - with errors in HardwareDivergence. - - In addition to uploading the report to node repo, have actions based on what kind of errors and what kind of state - the node has. - - Automatically create Jira tickets when something is reported wrong on a node. diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/CommandExecutor.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/CommandExecutor.java deleted file mode 100644 index 21e54c8c010..00000000000 --- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/CommandExecutor.java +++ /dev/null @@ -1,50 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.verification.commons; - -import org.apache.commons.exec.CommandLine; -import org.apache.commons.exec.DefaultExecutor; -import org.apache.commons.exec.PumpStreamHandler; - -import java.io.BufferedReader; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.StringReader; -import java.util.ArrayList; -import java.util.List; - -/** - * Wrapper for executing terminal commands - * - * @author olaaun - * @author sgrostad - */ -public class CommandExecutor { - - public List<String> executeCommand(String command) throws IOException { - ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); - List<String> results = new ArrayList<>(); - writeToOutputStream(outputStream, command); - writeOutputStreamToResults(outputStream, results); - return results; - } - - private void writeToOutputStream(ByteArrayOutputStream outputStream, String command) throws IOException { - CommandLine cmdLine = new CommandLine("/bin/bash"); - cmdLine.addArgument("-c", false); - cmdLine.addArgument(command, false); - DefaultExecutor executor = new DefaultExecutor(); - PumpStreamHandler streamHandler = new PumpStreamHandler(outputStream); - executor.setStreamHandler(streamHandler); - executor.execute(cmdLine); - } - - private void writeOutputStreamToResults(ByteArrayOutputStream outputStream, List<String> results) throws IOException { - String out = outputStream.toString(); - BufferedReader br = new BufferedReader(new StringReader(out)); - String line; - while ((line = br.readLine()) != null) { - results.add(line); - } - } - -} diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/noderepo/IPAddressVerifier.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/noderepo/IPAddressVerifier.java deleted file mode 100644 index fe1f8d72d50..00000000000 --- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/noderepo/IPAddressVerifier.java +++ /dev/null @@ -1,184 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.verification.commons.noderepo; - -import com.yahoo.vespa.hosted.node.verification.commons.report.SpecVerificationReport; - -import javax.naming.NamingEnumeration; -import javax.naming.NamingException; -import javax.naming.directory.Attribute; -import javax.naming.directory.Attributes; -import javax.naming.directory.DirContext; -import javax.naming.directory.InitialDirContext; -import java.net.InetAddress; -import java.net.UnknownHostException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Enumeration; -import java.util.Hashtable; -import java.util.List; -import java.util.logging.Level; -import java.util.logging.Logger; - -/** - * Created by olaa on 14/07/2017. - * Verifies that the IP addresses of a node points to the correct hostname - * - * @author olaaun - * @author sgrostad - */ -public class IPAddressVerifier { - - private static final Logger logger = Logger.getLogger(IPAddressVerifier.class.getName()); - - private final String expectedHostname; - private final boolean skipLookup; - private final boolean skipReverseLookup; - - public IPAddressVerifier(String expectedHostname, boolean skipLookup, boolean skipReverseLookup) { - this.expectedHostname = expectedHostname; - this.skipLookup = skipLookup; - this.skipReverseLookup = skipReverseLookup; - } - - public void reportFaultyIpAddresses(NodeSpec nodeSpec, SpecVerificationReport specVerificationReport) { - String[] faultyIpAddresses = getFaultyIpAddresses(nodeSpec); - if (faultyIpAddresses.length > 0) { - specVerificationReport.setFaultyIpAddresses(faultyIpAddresses); - } - } - - public String[] getFaultyIpAddresses(NodeSpec nodeSpec) { - List<String> faultyIpAddresses = new ArrayList<>(); - if (expectedHostname == null || expectedHostname.equals("")) - return new String[0]; - - if (!isValidIpv4(nodeSpec.getIpv4Address())) { - faultyIpAddresses.add(nodeSpec.getIpv4Address()); - } - if (!isValidIpv6(nodeSpec.getIpv6Address())) { - faultyIpAddresses.add(nodeSpec.getIpv6Address()); - } - return faultyIpAddresses.toArray(new String[0]); - } - - private boolean hostnameResolvesToIpAddress(String ipAddress) { - if (skipLookup) { - return true; - } - - InetAddress addressFromIpAddress; - try { - addressFromIpAddress = InetAddress.getByName(ipAddress); - } catch (UnknownHostException e) { - logger.log(Level.WARNING, "Failed to parse IP address " + ipAddress, e); - return false; - } - - List<InetAddress> addressesFromHostname; - try { - addressesFromHostname = mockableGetAllByName(expectedHostname); - } catch (UnknownHostException e) { - logger.log(Level.WARNING, "Failed to get IP addresses of hostname " + expectedHostname, e); - return false; - } - - if (addressesFromHostname.stream().noneMatch(addressFromIpAddress::equals)) { - logger.log(Level.WARNING, "Hostname " + expectedHostname + " resolved to " + addressesFromHostname + - " which does not contain the IP address " + addressFromIpAddress); - return false; - } - - return true; - } - - private boolean ipAddressResolvesToHostname(String ipAddressLookupFormat) { - if (skipReverseLookup) { - return true; - } - - try { - String hostnameFromIpAddress = reverseLookUp(ipAddressLookupFormat); - if (hostnameFromIpAddress.equals(expectedHostname)) { - return true; - } - - logger.log(Level.WARNING, "IP address " + ipAddressLookupFormat + " resolved to " + - hostnameFromIpAddress + ", not " + expectedHostname); - } catch (NamingException e) { - logger.log(Level.WARNING, "Could not get hostname of IP address " + ipAddressLookupFormat, e); - } - - return false; - } - - private boolean isValidIpv4(String ipv4Address) { - if (ipv4Address == null) { - return true; - } - - return hostnameResolvesToIpAddress(ipv4Address) && - ipAddressResolvesToHostname(convertIpv4ToLookupFormat(ipv4Address)); - } - - private boolean isValidIpv6(String ipv6Address) { - if (ipv6Address == null) { - return true; - } - - return hostnameResolvesToIpAddress(ipv6Address) && - ipAddressResolvesToHostname(convertIpv6ToLookupFormat(ipv6Address)); - } - - String reverseLookUp(String ipAddress) throws NamingException { - Hashtable<String, String> env = new Hashtable<>(); - env.put("java.naming.factory.initial", "com.sun.jndi.dns.DnsContextFactory"); - DirContext ctx = new InitialDirContext(env); - Attributes attrs = ctx.getAttributes(ipAddress, new String[]{"PTR"}); - for (NamingEnumeration<? extends Attribute> ae = attrs.getAll(); ae.hasMoreElements(); ) { - Attribute attr = ae.next(); - Enumeration<?> vals = attr.getAll(); - if (vals.hasMoreElements()) { - String hostname = vals.nextElement().toString(); - ctx.close(); - return hostname.substring(0, hostname.length() - 1); - } - } - ctx.close(); - return ""; - } - - String convertIpv6ToLookupFormat(String ipAddress) { - StringBuilder newIpAddress = new StringBuilder(); - String doubleColonReplacement = "0.0.0.0.0.0.0.0.0.0.0.0."; - String domain = "ip6.arpa"; - String[] hextets = ipAddress.split(":"); - for (int i = hextets.length - 1; i >= 0; i--) { - String reversedHextet = new StringBuilder(hextets[i]).reverse().toString(); - if (reversedHextet.equals("")) { - newIpAddress.append(doubleColonReplacement); - continue; - } - String trailingZeroes = "0000"; - String paddedHextet = (reversedHextet + trailingZeroes).substring(0, trailingZeroes.length()); - String punctuatedHextet = paddedHextet.replaceAll(".", "$0."); - newIpAddress.append(punctuatedHextet); - } - newIpAddress.append(domain); - return newIpAddress.toString(); - } - - String convertIpv4ToLookupFormat(String ipAddress) { - String domain = "in-addr.arpa"; - String[] octets = ipAddress.split("\\."); - StringBuilder convertedIpAddress = new StringBuilder(); - for (int i = octets.length - 1; i >= 0; i--) { - convertedIpAddress.append(octets[i]).append("."); - } - convertedIpAddress.append(domain); - return convertedIpAddress.toString(); - } - - List<InetAddress> mockableGetAllByName(String hostname) throws UnknownHostException { - return Arrays.asList(InetAddress.getAllByName(hostname)); - } -} diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/noderepo/NodeJsonConverter.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/noderepo/NodeJsonConverter.java deleted file mode 100644 index 622d46ca587..00000000000 --- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/noderepo/NodeJsonConverter.java +++ /dev/null @@ -1,38 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.verification.commons.noderepo; - -import com.yahoo.vespa.hosted.node.verification.spec.retrievers.HardwareInfo; - -/** - * Converts a NodeSpec object to a HardwareInfo object. - * - * @author olaaun - * @author sgrostad - */ -public class NodeJsonConverter { - - private static void setInterfaceSpeed(NodeSpec nodeSpec, HardwareInfo nodeRepoHardwareInfo) { - nodeRepoHardwareInfo.setInterfaceSpeedMbs(nodeSpec.getBandwidth()); - } - - private static void setIpv6Interface(NodeSpec nodeSpec, HardwareInfo nodeRepoHardwareInfo) { - if (nodeSpec.getIpv6Address() != null) { - nodeRepoHardwareInfo.setIpv6Interface(true); - } - } - - private static void setIpv4Interface(NodeSpec nodeSpec, HardwareInfo nodeRepoHardwareInfo) { - if (nodeSpec.getIpv4Address() != null) { - nodeRepoHardwareInfo.setIpv4Interface(true); - } - } - - public static HardwareInfo convertJsonModelToHardwareInfo(NodeSpec nodeSpec) { - HardwareInfo nodeRepoHardwareInfo = nodeSpec.copyToHardwareInfo(); - setInterfaceSpeed(nodeSpec, nodeRepoHardwareInfo); - setIpv4Interface(nodeSpec, nodeRepoHardwareInfo); - setIpv6Interface(nodeSpec, nodeRepoHardwareInfo); - return nodeRepoHardwareInfo; - } - -} diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/noderepo/NodeSpec.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/noderepo/NodeSpec.java deleted file mode 100644 index 16be1c39a74..00000000000 --- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/noderepo/NodeSpec.java +++ /dev/null @@ -1,62 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.verification.commons.noderepo; - -import com.google.common.net.InetAddresses; -import com.yahoo.vespa.hosted.node.verification.spec.retrievers.HardwareInfo; -import com.yahoo.vespa.hosted.node.verification.spec.retrievers.HardwareInfo.DiskType; - -import java.net.Inet4Address; -import java.net.Inet6Address; -import java.net.InetAddress; -import java.util.stream.Stream; - -/** - * Object with the information about a node - * - * @author freva - */ -public class NodeSpec { - private final double minDiskAvailableGb; - private final double minMainMemoryAvailableGb; - private final double minCpuCores; - private final boolean fastDisk; - private final double bandwidth; - private final String[] ipAddresses; - - public NodeSpec(double minDiskAvailableGb, double minMainMemoryAvailableGb, double minCpuCores, boolean fastDisk, - double bandwidth, String[] ipAddresses) { - this.minDiskAvailableGb = minDiskAvailableGb; - this.minMainMemoryAvailableGb = minMainMemoryAvailableGb; - this.minCpuCores = minCpuCores; - this.fastDisk = fastDisk; - this.bandwidth = bandwidth; - this.ipAddresses = ipAddresses; - } - - public HardwareInfo copyToHardwareInfo() { - HardwareInfo hardwareInfo = new HardwareInfo(); - hardwareInfo.setMinMainMemoryAvailableGb(this.minMainMemoryAvailableGb); - hardwareInfo.setMinDiskAvailableGb(this.minDiskAvailableGb); - hardwareInfo.setMinCpuCores((int) Math.round(this.minCpuCores)); - hardwareInfo.setDiskType(this.fastDisk ? DiskType.FAST : DiskType.SLOW); - hardwareInfo.setInterfaceSpeedMbs(bandwidth); - hardwareInfo.setIpv6Connection(getIpv6Address() != null); - return hardwareInfo; - } - - public String getIpv6Address() { - return Stream.of(ipAddresses) - .map(InetAddresses::forString) - .filter(ip -> ip instanceof Inet6Address) - .findFirst().map(InetAddress::getHostAddress).orElse(null); - } - - public String getIpv4Address() { - return Stream.of(ipAddresses) - .map(InetAddresses::forString) - .filter(ip -> ip instanceof Inet4Address) - .findFirst().map(InetAddress::getHostAddress).orElse(null); - } - - public double getBandwidth() { return bandwidth ; } -} diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/parser/OutputParser.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/parser/OutputParser.java deleted file mode 100644 index 55bf1dd0e0b..00000000000 --- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/parser/OutputParser.java +++ /dev/null @@ -1,47 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.verification.commons.parser; - -import java.util.ArrayList; -import java.util.List; -import java.util.Optional; -import java.util.regex.Pattern; - -/** - * Parses terminal command output, and returns results based on ParseInstructions - * - * @author sgrostad - * @author olaaaun - */ -public class OutputParser { - - public static List<ParseResult> parseOutput(ParseInstructions parseInstructions, List<String> commandOutput) { - List<ParseResult> results = new ArrayList<>(); - int searchElementIndex = parseInstructions.getSearchElementIndex(); - int valueElementIndex = parseInstructions.getValueElementIndex(); - List<String> searchWords = parseInstructions.getSearchWords(); - for (String line : commandOutput) { - String[] lineSplit = line.trim().split(parseInstructions.getSplitRegex()); - if (lineSplit.length <= Math.max(searchElementIndex, valueElementIndex)) { - continue; - } - String searchWordCandidate = lineSplit[searchElementIndex].trim(); - boolean searchWordCandidateMatch = matchingSearchWord(searchWords, searchWordCandidate); - if (searchWordCandidateMatch) { - String value = lineSplit[valueElementIndex]; - results.add(new ParseResult(searchWordCandidate, value.trim())); - } - } - return results; - } - - public static Optional<ParseResult> parseSingleOutput(ParseInstructions parseInstructions, List<String> commandOutput) { - List<ParseResult> parseResults = parseOutput(parseInstructions, commandOutput); - if (parseResults.isEmpty()) return Optional.empty(); - return Optional.ofNullable(parseResults.get(0)); - } - - private static boolean matchingSearchWord(List<String> searchWords, String searchWordCandidate) { - return searchWords.stream().anyMatch(w -> Pattern.compile(w).matcher(searchWordCandidate).matches()); - } - -} diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/parser/ParseInstructions.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/parser/ParseInstructions.java deleted file mode 100644 index 262bfcba0b9..00000000000 --- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/parser/ParseInstructions.java +++ /dev/null @@ -1,42 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.verification.commons.parser; - -import java.util.List; - -/** - * Contains instructions of how a command line output should be parsed - * - * @author sgrostad - * @author olaaaun - */ -public class ParseInstructions { - - private final int searchElementIndex; - private final int valueElementIndex; - private final String splitRegex; - private final List<String> searchWords; - - public ParseInstructions(int searchElementIndex, int returnElementNum, String splitRegex, List<String> searchWords) { - this.searchElementIndex = searchElementIndex; - this.valueElementIndex = returnElementNum; - this.splitRegex = splitRegex; - this.searchWords = searchWords; - } - - public int getSearchElementIndex() { - return searchElementIndex; - } - - public int getValueElementIndex() { - return valueElementIndex; - } - - public String getSplitRegex() { - return splitRegex; - } - - public List<String> getSearchWords() { - return searchWords; - } - -} diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/parser/ParseResult.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/parser/ParseResult.java deleted file mode 100644 index a89cfd89f31..00000000000 --- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/parser/ParseResult.java +++ /dev/null @@ -1,52 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.verification.commons.parser; - -import java.util.Objects; - -/** - * Contains the results from parsing a command line output - * - * @author sgrostad - * @author olaaun - */ -public class ParseResult { - - private final String searchWord; - private final String value; - - public ParseResult(String searchWord, String value) { - this.searchWord = searchWord; - this.value = value; - } - - public String getSearchWord() { - return searchWord; - } - - public String getValue() { - return value; - } - - @Override - public boolean equals(Object obj) { - if (this == obj) return true; - if (obj instanceof ParseResult) { - ParseResult parseResult = (ParseResult) obj; - if (this.searchWord.equals(parseResult.getSearchWord()) && this.value.equals(parseResult.getValue())) { - return true; - } - } - return false; - } - - @Override - public int hashCode() { - return Objects.hash(searchWord, value); - } - - @Override - public String toString() { - return "Search word: " + searchWord + ", Value: " + value; - } - -} diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/report/BenchmarkReport.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/report/BenchmarkReport.java deleted file mode 100644 index ce4e3685a25..00000000000 --- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/report/BenchmarkReport.java +++ /dev/null @@ -1,73 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.verification.commons.report; - -import com.fasterxml.jackson.annotation.JsonIgnore; -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectMapper; - -/** - * JSON-mapped class for reporting benchmark results to node repo - * - * @author sgrostad - * @author olaaun - */ -@JsonInclude(JsonInclude.Include.NON_NULL) -public class BenchmarkReport { - - @JsonProperty - private Double cpuCyclesPerSec; - @JsonProperty - private Double diskSpeedMbs; - @JsonProperty - private Double memoryWriteSpeedGBs; - @JsonProperty - private Double memoryReadSpeedGBs; - - public void setCpuCyclesPerSec(double cpuCyclesPerSec) { - this.cpuCyclesPerSec = cpuCyclesPerSec; - } - - public void setDiskSpeedMbs(Double diskSpeedMbs) { - this.diskSpeedMbs = diskSpeedMbs != null ? diskSpeedMbs : -1; - } - - - public void setMemoryWriteSpeedGBs(Double memoryWriteSpeedGBs) { - this.memoryWriteSpeedGBs = memoryWriteSpeedGBs != null ? memoryWriteSpeedGBs : -1; - } - - public void setMemoryReadSpeedGBs(Double memoryReadSpeedGBs) { - this.memoryReadSpeedGBs = memoryReadSpeedGBs != null ? memoryReadSpeedGBs : -1; - } - - public Double getCpuCyclesPerSec() { - return cpuCyclesPerSec; - } - - public Double getDiskSpeedMbs() { - return diskSpeedMbs; - } - - public Double getMemoryWriteSpeedGBs() { - return memoryWriteSpeedGBs; - } - - public Double getMemoryReadSpeedGBs() { - return memoryReadSpeedGBs; - } - - @JsonIgnore - public boolean isAllBenchmarksOK() { - ObjectMapper om = new ObjectMapper(); - try { - String jsonReport = om.writeValueAsString(this); - return jsonReport.length() == 2; - } catch (JsonProcessingException e) { - e.printStackTrace(); - return false; - } - } - -} diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/report/HardwareDivergenceReport.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/report/HardwareDivergenceReport.java deleted file mode 100644 index 972733abdd2..00000000000 --- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/report/HardwareDivergenceReport.java +++ /dev/null @@ -1,43 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.verification.commons.report; - -import com.fasterxml.jackson.annotation.JsonIgnore; -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonProperty; - -/** - * JSON-wrapped report for node repo - * - * @author sgrostad - * @author olaaun - */ -@JsonInclude(JsonInclude.Include.NON_NULL) -public class HardwareDivergenceReport { - - @JsonProperty - SpecVerificationReport specVerificationReport; - - @JsonProperty - BenchmarkReport benchmarkReport; - - public void setSpecVerificationReport(SpecVerificationReport specVerificationReport) { - if (specVerificationReport.isValidSpec()) { - this.specVerificationReport = null; - } else { - this.specVerificationReport = specVerificationReport; - } - } - - public void setBenchmarkReport(BenchmarkReport benchmarkReport) { - if (benchmarkReport.isAllBenchmarksOK()) { - this.benchmarkReport = null; - } else { - this.benchmarkReport = benchmarkReport; - } - } - - @JsonIgnore - public boolean isHardwareDivergenceReportEmpty() { - return specVerificationReport == null && benchmarkReport == null; - } -} diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/report/SpecVerificationReport.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/report/SpecVerificationReport.java deleted file mode 100644 index 0494eed5dc5..00000000000 --- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/report/SpecVerificationReport.java +++ /dev/null @@ -1,78 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.verification.commons.report; - -import com.fasterxml.jackson.annotation.JsonIgnore; -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.yahoo.vespa.hosted.node.verification.spec.retrievers.HardwareInfo; - - -/** - * Stores results of comparing node repo spec and actual hardware info. - * In case of divergent values, set the corresponding attribute to the actual hardware info value. - * Attributes of equal value remain null. - * - * @author sgrostad - * @author olaaun - */ -@JsonInclude(JsonInclude.Include.NON_NULL) -public class SpecVerificationReport { - - @JsonProperty - private Double actualMemoryAvailable; - @JsonProperty - private HardwareInfo.DiskType actualDiskType; - @JsonProperty - private Double actualDiskSpaceAvailable; - @JsonProperty - private Double actualInterfaceSpeed; - @JsonProperty - private Integer actualcpuCores; - @JsonProperty - private String[] faultyIpAddresses; - @JsonProperty - private Boolean actualIpv6Connection; - - public void setActualIpv6Connection(boolean actualIpv6Connection) { - this.actualIpv6Connection = actualIpv6Connection; - } - - public void setActualMemoryAvailable(Double actualMemoryAvailable) { - this.actualMemoryAvailable = actualMemoryAvailable; - } - - public void setActualDiskType(HardwareInfo.DiskType actualFastDisk) { - this.actualDiskType = actualFastDisk; - } - - public void setActualDiskSpaceAvailable(Double actualDiskSpaceAvailable) { - this.actualDiskSpaceAvailable = actualDiskSpaceAvailable; - } - - public void setActualcpuCores(int actualcpuCores) { - this.actualcpuCores = actualcpuCores; - } - - public void setActualInterfaceSpeed(Double actualInterfaceSpeed) { - this.actualInterfaceSpeed = actualInterfaceSpeed; - } - - public void setFaultyIpAddresses(String[] faultyIpAddresses) { - this.faultyIpAddresses = faultyIpAddresses; - } - - @JsonIgnore - public boolean isValidSpec() { - ObjectMapper om = new ObjectMapper(); - try { - String jsonReport = om.writeValueAsString(this); - return jsonReport.length() == 2; - } catch (JsonProcessingException e) { - e.printStackTrace(); - return false; - } - } - -} diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/BenchmarkResultInspector.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/BenchmarkResultInspector.java deleted file mode 100644 index 5e63706f556..00000000000 --- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/BenchmarkResultInspector.java +++ /dev/null @@ -1,42 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.verification.hardware; - -import com.yahoo.vespa.hosted.node.verification.commons.report.BenchmarkReport; -import com.yahoo.vespa.hosted.node.verification.hardware.benchmarks.BenchmarkResults; - -/** - * Responsible for checking the benchmarks results, and adding unreasonable results to BenchmarkReport - * - * @author sgrostad - * @author olaaun - */ -public class BenchmarkResultInspector { - - private static final double CPU_FREQUENCY_LOWER_LIMIT = 0.5; - private static final double MEMORY_WRITE_SPEED_LOWER_LIMIT = 1D; - private static final double MEMORY_READ_SPEED_LOWER_LIMIT = 1D; - private static final double DISK_SPEED_LOWER_LIMIT = 50D; - - public static BenchmarkReport makeBenchmarkReport(BenchmarkResults benchmarkResults) { - BenchmarkReport benchmarkReport = new BenchmarkReport(); - double cpuCyclesPerSec = benchmarkResults.getCpuCyclesPerSec(); - if (cpuCyclesPerSec < CPU_FREQUENCY_LOWER_LIMIT) { - benchmarkReport.setCpuCyclesPerSec(cpuCyclesPerSec); - } - double memoryWriteSpeed = benchmarkResults.getMemoryWriteSpeedGBs(); -// TODO: Temporarily disabled due to Meltdown/Spectre performance impact, see VESPA-11051 -// if (memoryWriteSpeed < MEMORY_WRITE_SPEED_LOWER_LIMIT) { -// benchmarkReport.setMemoryWriteSpeedGBs(memoryWriteSpeed); -// } -// double memoryReadSpeed = benchmarkResults.getMemoryReadSpeedGBs(); -// if (memoryReadSpeed < MEMORY_READ_SPEED_LOWER_LIMIT) { -// benchmarkReport.setMemoryReadSpeedGBs(memoryReadSpeed); -// } - double diskSpeed = benchmarkResults.getDiskSpeedMbs(); - if (diskSpeed < DISK_SPEED_LOWER_LIMIT) { - benchmarkReport.setDiskSpeedMbs(diskSpeed); - } - return benchmarkReport; - } - -} diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/HardwareBenchmarker.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/HardwareBenchmarker.java deleted file mode 100644 index 9b3d8f45e1d..00000000000 --- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/HardwareBenchmarker.java +++ /dev/null @@ -1,43 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.verification.hardware; - -import com.yahoo.vespa.hosted.node.verification.Main; -import com.yahoo.vespa.hosted.node.verification.commons.CommandExecutor; -import com.yahoo.vespa.hosted.node.verification.commons.report.BenchmarkReport; -import com.yahoo.vespa.hosted.node.verification.commons.report.HardwareDivergenceReport; -import com.yahoo.vespa.hosted.node.verification.hardware.benchmarks.Benchmark; -import com.yahoo.vespa.hosted.node.verification.hardware.benchmarks.BenchmarkResults; -import com.yahoo.vespa.hosted.node.verification.hardware.benchmarks.CPUBenchmark; -import com.yahoo.vespa.hosted.node.verification.hardware.benchmarks.DiskBenchmark; -import com.yahoo.vespa.hosted.node.verification.hardware.benchmarks.MemoryBenchmark; -import io.airlift.command.Command; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; - -/** - * Benchmarks different hardware components and creates report - */ -@Command(name = "benchmark", description = "Run node benchmarks") -public class HardwareBenchmarker extends Main.VerifierCommand { - - @Override - public void run(HardwareDivergenceReport hardwareDivergenceReport, CommandExecutor commandExecutor) { - BenchmarkReport benchmarkReport = hardwareBenchmarks(commandExecutor); - - hardwareDivergenceReport.setBenchmarkReport(benchmarkReport); - } - - private BenchmarkReport hardwareBenchmarks(CommandExecutor commandExecutor) { - BenchmarkResults benchmarkResults = new BenchmarkResults(); - List<Benchmark> benchmarks = new ArrayList<>(Arrays.asList( - new DiskBenchmark(benchmarkResults, commandExecutor), - new CPUBenchmark(benchmarkResults, commandExecutor), - new MemoryBenchmark(benchmarkResults, commandExecutor))); - for (Benchmark benchmark : benchmarks) { - benchmark.doBenchmark(); - } - return BenchmarkResultInspector.makeBenchmarkReport(benchmarkResults); - } -} diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/README.md b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/README.md deleted file mode 100644 index defa24bfcab..00000000000 --- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/README.md +++ /dev/null @@ -1,14 +0,0 @@ -<!-- Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> -# Hardware Verification -Verification of behaviour and performance of hardware. Benchmarks cpu frequency, disk write speed and memory write/read speed. -A report is sent to the node repository if any of the results are below an accepted threshold. - -## Code Walkthrough -The main class, HardwareBenchmarker, calls every benchmark in the benchmark package. - -The results of these benchmarks are passed through -the BenchmarkResultInspector, which creates a BenchmarkReport containing the values below the accepted threshold. - -ReportSender is then called such that -the old HardwareDivergence report is retrieved from node repo and updated with the new results. ReportSender prints the new HardwareReport such that it -can be updated in node repo by chef or other. diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/Benchmark.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/Benchmark.java deleted file mode 100644 index 1e5b512f492..00000000000 --- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/Benchmark.java +++ /dev/null @@ -1,15 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.verification.hardware.benchmarks; - -/** - * @author sgrostad - * @author olaaun - */ -public interface Benchmark { - - /** - * Should perform benchmark for some part of the hardware, and store the result in BenchmarkResults instance passed to class - */ - void doBenchmark(); - -} diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/BenchmarkResults.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/BenchmarkResults.java deleted file mode 100644 index dfa436460e1..00000000000 --- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/BenchmarkResults.java +++ /dev/null @@ -1,50 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.verification.hardware.benchmarks; - -/** - * Stores results from benchmarks - * - * @author sgrostad - * @author olaaun - */ -// TODO: This should be immutable -public class BenchmarkResults { - - private double cpuCyclesPerSec; - private double diskSpeedMbs; - private double memoryWriteSpeedGBs; - private double memoryReadSpeedGBs; - - public double getMemoryWriteSpeedGBs() { - return memoryWriteSpeedGBs; - } - - public void setMemoryWriteSpeedGBs(double memoryWriteSpeedGBs) { - this.memoryWriteSpeedGBs = memoryWriteSpeedGBs; - } - - public double getMemoryReadSpeedGBs() { - return memoryReadSpeedGBs; - } - - public void setMemoryReadSpeedGBs(double memoryReadSpeedGBs) { - this.memoryReadSpeedGBs = memoryReadSpeedGBs; - } - - public double getCpuCyclesPerSec() { - return cpuCyclesPerSec; - } - - public void setCpuCyclesPerSec(double cpuCycles) { - this.cpuCyclesPerSec = cpuCycles; - } - - public double getDiskSpeedMbs() { - return diskSpeedMbs; - } - - public void setDiskSpeedMbs(double diskSpeedMbs) { - this.diskSpeedMbs = diskSpeedMbs; - } - -} diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/CPUBenchmark.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/CPUBenchmark.java deleted file mode 100644 index 57763177a5b..00000000000 --- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/CPUBenchmark.java +++ /dev/null @@ -1,119 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.verification.hardware.benchmarks; - -import com.yahoo.vespa.hosted.node.verification.commons.CommandExecutor; -import com.yahoo.vespa.hosted.node.verification.commons.parser.OutputParser; -import com.yahoo.vespa.hosted.node.verification.commons.parser.ParseInstructions; -import com.yahoo.vespa.hosted.node.verification.commons.parser.ParseResult; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.logging.Level; -import java.util.logging.Logger; - -/** - * Responsible for benchmarking CPU clock frequency, and storing the result in a BenchmarkResults instance - * - * @author sgrostad - * @author olaaun - */ -public class CPUBenchmark implements Benchmark { - - private static final String CPU_BENCHMARK_COMMAND = "perf stat -e cycles dd if=/dev/zero of=/dev/null count=2000000 2>&1 | grep 'cycles\\|seconds'"; - private static final String CYCLES_SEARCH_WORD = "cycles"; - private static final String SECONDS_SEARCH_WORD = "seconds"; - private static final String SPLIT_REGEX_STRING = "\\s+"; - private static final int SEARCH_ELEMENT_INDEX = 1; - private static final int RETURN_ELEMENT_INDEX = 0; - private static final Logger logger = Logger.getLogger(CPUBenchmark.class.getName()); - private final BenchmarkResults benchmarkResults; - - private final CommandExecutor commandExecutor; - - public CPUBenchmark(BenchmarkResults benchmarkResults, CommandExecutor commandExecutor) { - this.benchmarkResults = benchmarkResults; - this.commandExecutor = commandExecutor; - } - - @Override - public void doBenchmark() { - try { - List<String> commandOutput = commandExecutor.executeCommand(CPU_BENCHMARK_COMMAND); - List<ParseResult> parseResults = parseCpuCyclesPerSec(commandOutput); - setCpuCyclesPerSec(parseResults); - } catch (IOException e) { - logger.log(Level.WARNING, "Failed to perform CPU benchmark", e); - } - } - - List<ParseResult> parseCpuCyclesPerSec(List<String> commandOutput) { - List<String> searchWords = new ArrayList<>(Arrays.asList(CYCLES_SEARCH_WORD, SECONDS_SEARCH_WORD)); - ParseInstructions parseInstructions = new ParseInstructions(SEARCH_ELEMENT_INDEX, RETURN_ELEMENT_INDEX, SPLIT_REGEX_STRING, searchWords); - return OutputParser.parseOutput(parseInstructions, commandOutput); - } - - - void setCpuCyclesPerSec(List<ParseResult> parseResults) { - double cpuCyclesPerSec = getCyclesPerSecond(parseResults); - if (cpuCyclesPerSec > 0) { - benchmarkResults.setCpuCyclesPerSec(cpuCyclesPerSec); - } - } - - private double getCyclesPerSecond(List<ParseResult> parseResults) { - double cycles = -1; - double seconds = -1; - for (ParseResult parseResult : parseResults) { - switch (parseResult.getSearchWord()) { - case CYCLES_SEARCH_WORD: - cycles = makeCyclesDouble(parseResult.getValue()); - break; - case SECONDS_SEARCH_WORD: - seconds = makeSecondsDouble(parseResult.getValue()); - break; - default: - throw new RuntimeException("Invalid ParseResult searchWord: " + parseResult.getSearchWord()); - } - } - if (cycles > 0 && seconds > 0) { - return convertToGHz(cycles, seconds); - } - return -1; - } - - double makeCyclesDouble(String cycles) { - cycles = cycles.replaceAll("[^\\d]", ""); - if (checkIfNumber(cycles)) { - return Double.parseDouble(cycles); - } - return -1; - } - - double makeSecondsDouble(String seconds) { - seconds = seconds.replaceAll(",", "."); - if (checkIfNumber(seconds)) { - return Double.parseDouble(seconds); - } - return -1; - } - - boolean checkIfNumber(String numberCandidate) { - if (numberCandidate == null || numberCandidate.equals("")) { - return false; - } - try { - Double.parseDouble(numberCandidate); - } catch (NumberFormatException e) { - return false; - } - return true; - } - - double convertToGHz(double cycles, double seconds) { - double giga = 1000000000.0; - return (cycles / seconds) / giga; - } - -} diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/DiskBenchmark.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/DiskBenchmark.java deleted file mode 100644 index a7fc5809058..00000000000 --- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/DiskBenchmark.java +++ /dev/null @@ -1,102 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.verification.hardware.benchmarks; - -import com.yahoo.vespa.hosted.node.verification.commons.CommandExecutor; -import com.yahoo.vespa.hosted.node.verification.commons.parser.OutputParser; -import com.yahoo.vespa.hosted.node.verification.commons.parser.ParseInstructions; -import com.yahoo.vespa.hosted.node.verification.commons.parser.ParseResult; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.Optional; -import java.util.logging.Level; -import java.util.logging.Logger; - - -/** - * Responsible for benchmarking disk write speed, and storing the result in a BenchmarkResults instance - * - * @author olaaun - * @author sgrostad - */ -public class DiskBenchmark implements Benchmark { - - private static final String DISK_BENCHMARK_COMMAND = "(dd if=/dev/zero of=/home/y/tmp/output conv=fdatasync bs=4G count=4; rm -f /home/y/tmp/output;) 2>&1 | grep bytes | awk '{ print $8 \" \" $9 }'"; - private static final String KILO_BYTE_SEARCH_WORD = "kB/s"; - private static final String MEGA_BYTE_SEARCH_WORD = "MB/s"; - private static final String GIGA_BYTE_SEARCH_WORD = "GB/s"; - private static final String SPLIT_REGEX_STRING = " "; - private static final int SEARCH_ELEMENT_INDEX = 1; - private static final int RETURN_ELEMENT_INDEX = 0; - private static final Logger logger = Logger.getLogger(DiskBenchmark.class.getName()); - private final BenchmarkResults benchmarkResults; - private final CommandExecutor commandExecutor; - - public DiskBenchmark(BenchmarkResults benchmarkResults, CommandExecutor commandExecutor) { - this.benchmarkResults = benchmarkResults; - this.commandExecutor = commandExecutor; - } - - @Override - public void doBenchmark() { - try { - List<String> commandOutput = commandExecutor.executeCommand(DISK_BENCHMARK_COMMAND); - Optional<ParseResult> parseResult = parseDiskSpeed(commandOutput); - setDiskSpeed(parseResult); - } catch (IOException e) { - logger.log(Level.WARNING, "Failed to perform disk benchmark", e); - } - } - - Optional<ParseResult> parseDiskSpeed(List<String> commandOutput) { - List<String> searchWords = new ArrayList<>(Arrays.asList(KILO_BYTE_SEARCH_WORD, MEGA_BYTE_SEARCH_WORD, GIGA_BYTE_SEARCH_WORD)); - ParseInstructions parseInstructions = new ParseInstructions(SEARCH_ELEMENT_INDEX, RETURN_ELEMENT_INDEX, SPLIT_REGEX_STRING, searchWords); - return OutputParser.parseSingleOutput(parseInstructions, commandOutput); - } - - void setDiskSpeed(Optional<ParseResult> parseResult) { - benchmarkResults.setDiskSpeedMbs(parseResult.map(this::getDiskSpeedInMBs).orElse(0.0d)); - } - - double getDiskSpeedInMBs(ParseResult parseResult) { - double diskSpeedMBs = 0; - double convertKBsToMBs = 1 / 1000.0; - double convertGBsToMBs = 1000.0; - double convertMbsToMBs = 1.0; - String diskSpeed = parseResult.getValue(); - if (checkSpeedValidity(diskSpeed)) { - switch (parseResult.getSearchWord()) { - case KILO_BYTE_SEARCH_WORD: - diskSpeedMBs = convertToMBs(diskSpeed, convertKBsToMBs); - break; - case MEGA_BYTE_SEARCH_WORD: - diskSpeedMBs = convertToMBs(diskSpeed, convertMbsToMBs); - break; - case GIGA_BYTE_SEARCH_WORD: - diskSpeedMBs = convertToMBs(diskSpeed, convertGBsToMBs); - break; - default: - throw new RuntimeException("Invalid ParseResult searchWord: " + parseResult.getSearchWord()); - } - } - - return diskSpeedMBs; - } - - boolean checkSpeedValidity(String speed) { - try { - Double.parseDouble(speed); - } catch (NullPointerException | NumberFormatException e) { - return false; - } - return true; - } - - Double convertToMBs(String speed, double numberToConvert) { - double speedMbs = Double.parseDouble(speed); - return speedMbs * numberToConvert; - } - -} diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/MemoryBenchmark.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/MemoryBenchmark.java deleted file mode 100644 index 7a72999a1e4..00000000000 --- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/MemoryBenchmark.java +++ /dev/null @@ -1,102 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.verification.hardware.benchmarks; - -import com.yahoo.vespa.hosted.node.verification.commons.CommandExecutor; -import com.yahoo.vespa.hosted.node.verification.commons.parser.OutputParser; -import com.yahoo.vespa.hosted.node.verification.commons.parser.ParseInstructions; -import com.yahoo.vespa.hosted.node.verification.commons.parser.ParseResult; - -import java.io.IOException; -import java.util.Collections; -import java.util.List; -import java.util.Optional; -import java.util.logging.Level; -import java.util.logging.Logger; - -/** - * Responsible for benchmarking memory read/write speed, and storing the result in a BenchmarkResults instance - * - * @author sgrostad - * @author olaaun - */ -public class MemoryBenchmark implements Benchmark { - - private static final String MEM_BENCHMARK_CREATE_FOLDER = "mkdir -p RAM_test"; - private static final String MEM_BENCHMARK_MOUNT_TMPFS = "mount tmpfs -t tmpfs RAM_test/"; - private static final String MEM_BENCHMARK_UNMOUNT_TMPFS = "umount RAM_test"; - private static final String MEM_BENCHMARK_DELETE_FOLDER = "rm -rf RAM_test"; - private static final String MEM_BENCHMARK_WRITE_SPEED = "dd if=/dev/zero of=RAM_test/data_tmp bs=1M count=512"; - private static final String MEM_BENCHMARK_READ_SPEED = "dd if=RAM_test/data_tmp of=/dev/null bs=1M count=512"; - private static final String READ_AND_WRITE_SEARCH_WORD_GB = "GB/s"; - private static final String READ_AND_WRITE_SEARCH_WORD_MB = "MB/s"; - private static final String SPLIT_REGEX_STRING = " "; - private static final int SEARCH_ELEMENT_INDEX = 8; - private static final int RETURN_ELEMENT_INDEX = 7; - private static final Logger logger = Logger.getLogger(MemoryBenchmark.class.getName()); - private final BenchmarkResults benchmarkResults; - private final CommandExecutor commandExecutor; - - public MemoryBenchmark(BenchmarkResults benchmarkResults, CommandExecutor commandExecutor) { - this.benchmarkResults = benchmarkResults; - this.commandExecutor = commandExecutor; - } - - @Override - public void doBenchmark() { - try { - setupMountPoint(); - - parseMemorySpeed(commandExecutor.executeCommand(MEM_BENCHMARK_WRITE_SPEED)) - .ifPresent(benchmarkResults::setMemoryWriteSpeedGBs); - - parseMemorySpeed(commandExecutor.executeCommand(MEM_BENCHMARK_READ_SPEED)) - .ifPresent(benchmarkResults::setMemoryReadSpeedGBs); - } catch (IOException e) { - logger.log(Level.WARNING, "Failed to perform memory benchmark", e); - } finally { - breakDownMountPoint(); - } - } - - private void setupMountPoint() throws IOException { - commandExecutor.executeCommand(MEM_BENCHMARK_CREATE_FOLDER); - commandExecutor.executeCommand(MEM_BENCHMARK_MOUNT_TMPFS); - } - - private void breakDownMountPoint() { - try { - commandExecutor.executeCommand(MEM_BENCHMARK_UNMOUNT_TMPFS); - } catch (IOException e) { - logger.log(Level.WARNING, "Failed to unmount tmpfs folder", e); - } - try { - commandExecutor.executeCommand(MEM_BENCHMARK_DELETE_FOLDER); - } catch (IOException e) { - logger.log(Level.WARNING, "Failed to delete memory benchmark folder", e); - } - } - - protected Optional<Double> parseMemorySpeed(List<String> commandOutput) { - Optional<ParseResult> parseResultGb = parseMemorySpeed(commandOutput, READ_AND_WRITE_SEARCH_WORD_GB); - if (parseResultGb.isPresent()) return parseDouble(parseResultGb.get().getValue()); - - Optional<ParseResult> parseResultMb = parseMemorySpeed(commandOutput, READ_AND_WRITE_SEARCH_WORD_MB); - if (parseResultMb.isPresent()) return parseDouble(parseResultMb.get().getValue()).map(v -> v / 1000.0d); - - return Optional.empty(); - } - - private Optional<ParseResult> parseMemorySpeed(List<String> commandOutput, String searchWord) { - List<String> searchWords = Collections.singletonList(searchWord); - ParseInstructions parseInstructions = new ParseInstructions(SEARCH_ELEMENT_INDEX, RETURN_ELEMENT_INDEX, SPLIT_REGEX_STRING, searchWords); - return OutputParser.parseSingleOutput(parseInstructions, commandOutput); - } - - private Optional<Double> parseDouble(String benchmarkOutput) { - try { - return Optional.of(Double.parseDouble(benchmarkOutput)); - } catch (NumberFormatException | NullPointerException e) { - return Optional.empty(); - } - } -} diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/HardwareNodeComparator.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/HardwareNodeComparator.java deleted file mode 100644 index 57d83694709..00000000000 --- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/HardwareNodeComparator.java +++ /dev/null @@ -1,84 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.verification.spec; - -import com.yahoo.vespa.hosted.node.verification.commons.report.SpecVerificationReport; -import com.yahoo.vespa.hosted.node.verification.spec.retrievers.HardwareInfo; -import com.yahoo.vespa.hosted.node.verification.spec.retrievers.HardwareInfo.DiskType; - -/** - * Compares two HardwareInfo objects and stores divergent values in a SpecVerificationReport - * - * @author olaaun - * @author sgrostad - */ -public class HardwareNodeComparator { - - private static final double PERCENTAGE_THRESHOLD = 0.05; - - public static SpecVerificationReport compare(HardwareInfo nodeRepoHardwareInfo, HardwareInfo actualHardware) { - SpecVerificationReport specVerificationReport = new SpecVerificationReport(); - if (nodeRepoHardwareInfo == null || actualHardware == null) { - return specVerificationReport; - } - setReportMetrics(nodeRepoHardwareInfo, actualHardware, specVerificationReport); - return specVerificationReport; - } - - private static void setReportMetrics(HardwareInfo nodeRepoHardwareInfo, HardwareInfo actualHardware, SpecVerificationReport specVerificationReport) { - setMemoryMetrics(nodeRepoHardwareInfo, actualHardware, specVerificationReport); - setCpuMetrics(nodeRepoHardwareInfo, actualHardware, specVerificationReport); - setDiskTypeMetrics(nodeRepoHardwareInfo, actualHardware, specVerificationReport); - setDiskSpaceMetrics(nodeRepoHardwareInfo, actualHardware, specVerificationReport); - setNetMetrics(nodeRepoHardwareInfo, actualHardware, specVerificationReport); - } - - private static void setMemoryMetrics(HardwareInfo nodeRepoHardwareInfo, HardwareInfo actualHardware, SpecVerificationReport specVerificationReport) { - double expectedMemory = nodeRepoHardwareInfo.getMinMainMemoryAvailableGb(); - double actualMemory = actualHardware.getMinMainMemoryAvailableGb(); - if (belowThreshold(expectedMemory, actualMemory, PERCENTAGE_THRESHOLD)) { - specVerificationReport.setActualMemoryAvailable(actualMemory); - } - } - - private static void setCpuMetrics(HardwareInfo nodeRepoHardwareInfo, HardwareInfo actualHardware, SpecVerificationReport specVerificationReport) { - int expectedCpuCores = nodeRepoHardwareInfo.getMinCpuCores(); - int actualCpuCores = actualHardware.getMinCpuCores(); - if (expectedCpuCores != actualCpuCores) { - specVerificationReport.setActualcpuCores(actualCpuCores); - } - } - - private static void setDiskTypeMetrics(HardwareInfo nodeRepoHardwareInfo, HardwareInfo actualHardware, SpecVerificationReport specVerificationReport) { - DiskType expectedFastDisk = nodeRepoHardwareInfo.getDiskType(); - DiskType actualFastDisk = actualHardware.getDiskType(); - if (expectedFastDisk != null && actualFastDisk != null && expectedFastDisk != actualFastDisk) { - specVerificationReport.setActualDiskType(actualFastDisk); - } - } - - private static void setDiskSpaceMetrics(HardwareInfo nodeRepoHardwareInfo, HardwareInfo actualHardware, SpecVerificationReport specVerificationReport) { - double expectedDiskSpace = nodeRepoHardwareInfo.getMinDiskAvailableGb(); - double actualDiskSpace = actualHardware.getMinDiskAvailableGb(); - if (belowThreshold(expectedDiskSpace, actualDiskSpace, PERCENTAGE_THRESHOLD)) { - specVerificationReport.setActualDiskSpaceAvailable(actualDiskSpace); - } - } - - private static void setNetMetrics(HardwareInfo nodeRepoHardwareInfo, HardwareInfo actualHardware, SpecVerificationReport specVerificationReport) { - double expectedInterfaceSpeed = nodeRepoHardwareInfo.getInterfaceSpeedMbs(); - double actualInterfaceSpeed = actualHardware.getInterfaceSpeedMbs(); - if (expectedInterfaceSpeed > actualInterfaceSpeed) { - specVerificationReport.setActualInterfaceSpeed(actualInterfaceSpeed); - } - - if (nodeRepoHardwareInfo.isIpv6Connection() && !actualHardware.isIpv6Connection()) { - specVerificationReport.setActualIpv6Connection(actualHardware.isIpv6Connection()); - } - } - - private static boolean belowThreshold(double expected, double actual, double thresholdPercentage) { - double lowerThresholdPercentage = 1 - thresholdPercentage; - return actual < expected * lowerThresholdPercentage; - } - -} diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/README.md b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/README.md deleted file mode 100644 index d7129553416..00000000000 --- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/README.md +++ /dev/null @@ -1,15 +0,0 @@ -<!-- Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> -# Spec Verification -Verifies that the spec information in node repo coincides with what found on the node and reports back to node repo. - -## Code "walkthrough" -The main class SpecVerifier uses the "noderepo" package in "commons" to retrieve node spec from node repo. - -It must be called with one parameter, the config server hostname. It finds hostname using "HostURLGenerator" in the "commons" package. - -It then retrieves all the hardware information at the node with the "retrievers" package and stores the values as a -"HardwareInfo" object. - -SpecVerifier then uses HardwareNodeComparator to compare spec from node repo and the node itself. It generates a -SpecVerificationReport and uses Reporter in "commons" to retrieve the old HardwareDivergence report from node repo and update it. Reporter then prints the new -updated HardwareDivergence report such that it can be updated in node repo by chef or other. diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/SpecVerifier.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/SpecVerifier.java deleted file mode 100644 index 3f0391193ca..00000000000 --- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/SpecVerifier.java +++ /dev/null @@ -1,83 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.verification.spec; - -import com.google.common.base.Strings; -import com.yahoo.vespa.defaults.Defaults; -import com.yahoo.vespa.hosted.node.verification.Main; -import com.yahoo.vespa.hosted.node.verification.commons.CommandExecutor; -import com.yahoo.vespa.hosted.node.verification.commons.noderepo.IPAddressVerifier; -import com.yahoo.vespa.hosted.node.verification.commons.noderepo.NodeJsonConverter; -import com.yahoo.vespa.hosted.node.verification.commons.noderepo.NodeSpec; -import com.yahoo.vespa.hosted.node.verification.commons.report.HardwareDivergenceReport; -import com.yahoo.vespa.hosted.node.verification.commons.report.SpecVerificationReport; -import com.yahoo.vespa.hosted.node.verification.spec.retrievers.HardwareInfo; -import com.yahoo.vespa.hosted.node.verification.spec.retrievers.HardwareInfoRetriever; -import io.airlift.command.Command; -import io.airlift.command.Option; - -import java.util.Optional; - -/** - * Creates two HardwareInfo objects, one with spec from node repository and one from spec retrieved at the node. - * Compares the objects and returns the result. - * - * @author olaaun - * @author sgrostad - */ -@Command(name = "specification", description = "Verify that node's actual hardware and configuration matches the expected") -public class SpecVerifier extends Main.VerifierCommand { - - @Option(name = {"-d", "--disk"}, required = true, description = "Expected disk size in GB") - private double diskAvailableGb; - - @Option(name = {"-m", "--memory"}, required = true, description = "Expected main memory size in GB") - private double mainMemoryAvailableGb; - - @Option(name = {"-c", "--cpu_cores"}, required = true, description = "Expected number of CPU cores") - private double cpuCores; - - @Option(name = {"-s", "--is_ssd"}, required = true, description = "Set to true if disk is SSD", allowedValues = {"true", "false"}) - private String fastDisk; - - @Option(name = {"-b", "--bandwidth"}, required = true, description = "Expected network interface speed in Mbit/s") - private double bandwidth; - - @Option(name = {"-i", "--ips"}, description = "Comma separated list of IP addresses assigned to this node") - private String ipAddresses; - - @Option(name = {"--skip-lookup"}, required = false, description = "Skip verification of hostname -> IP addresses") - private boolean skipLookup = false; - - @Option(name = {"--skip-reverse-lookup"}, required = false, description = "Skip verification of IP addresses -> hostname") - private boolean skipReverseLookup = false; - - @Override - public void run(HardwareDivergenceReport hardwareDivergenceReport, CommandExecutor commandExecutor) { - String[] ips = Optional.ofNullable(ipAddresses) - .filter(s -> !Strings.isNullOrEmpty(s)) - .map(s -> s.split(",")) - .orElse(new String[0]); - - NodeSpec nodeSpec = new NodeSpec(diskAvailableGb, mainMemoryAvailableGb, cpuCores, Boolean.valueOf(fastDisk), bandwidth, ips); - SpecVerificationReport specVerificationReport = verifySpec(nodeSpec, commandExecutor); - - hardwareDivergenceReport.setSpecVerificationReport(specVerificationReport); - } - - private SpecVerificationReport verifySpec(NodeSpec nodeSpec, CommandExecutor commandExecutor) { - VerifierSettings verifierSettings = new VerifierSettings(true); - HardwareInfo actualHardware = HardwareInfoRetriever.retrieve(commandExecutor, verifierSettings); - return makeVerificationReport(actualHardware, nodeSpec, skipLookup, skipReverseLookup); - } - - private static SpecVerificationReport makeVerificationReport( - HardwareInfo actualHardware, - NodeSpec nodeSpec, - boolean skipLookup, - boolean skipReverseLookup) { - SpecVerificationReport specVerificationReport = HardwareNodeComparator.compare(NodeJsonConverter.convertJsonModelToHardwareInfo(nodeSpec), actualHardware); - IPAddressVerifier ipAddressVerifier = new IPAddressVerifier(Defaults.getDefaults().vespaHostname(), skipLookup, skipReverseLookup); - ipAddressVerifier.reportFaultyIpAddresses(nodeSpec, specVerificationReport); - return specVerificationReport; - } -} diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/VerifierSettings.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/VerifierSettings.java deleted file mode 100644 index 3bc1a447ea8..00000000000 --- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/VerifierSettings.java +++ /dev/null @@ -1,32 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.verification.spec; - -import com.yahoo.vespa.hosted.node.verification.commons.noderepo.NodeSpec; - -/** - * Contains information on what spec should be verified or not. - * - * @author sgrostad - * @author olaaun - */ -public class VerifierSettings { - - private final boolean checkIPv6; - - public VerifierSettings() { - this.checkIPv6 = true; - } - - public VerifierSettings(boolean checkIPv6) { - this.checkIPv6 = checkIPv6; - } - - public VerifierSettings(NodeSpec nodeSpec) { - checkIPv6 = nodeSpec.getIpv6Address() != null; - } - - public boolean isCheckIPv6() { - return checkIPv6; - } - -} diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/CPURetriever.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/CPURetriever.java deleted file mode 100644 index 6416babb79b..00000000000 --- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/CPURetriever.java +++ /dev/null @@ -1,62 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.verification.spec.retrievers; - -import com.yahoo.vespa.hosted.node.verification.commons.CommandExecutor; -import com.yahoo.vespa.hosted.node.verification.commons.parser.OutputParser; -import com.yahoo.vespa.hosted.node.verification.commons.parser.ParseInstructions; -import com.yahoo.vespa.hosted.node.verification.commons.parser.ParseResult; - -import java.io.IOException; -import java.util.Collections; -import java.util.List; -import java.util.logging.Level; -import java.util.logging.Logger; - -/** - * Retrieves number of CPU cores, and stores the result in a HardwareInfo instance - * - * @author olaaun - * @author sgrostad - */ -public class CPURetriever implements HardwareRetriever { - - private static final String CPU_INFO_COMMAND = "cat /proc/cpuinfo"; - private static final String SEARCH_WORD = "cpu MHz"; - private static final String REGEX_SPLIT = "\\s+:\\s"; - private static final int SEARCH_ELEMENT_INDEX = 0; - private static final int RETURN_ELEMENT_INDEX = 1; - private static final Logger logger = Logger.getLogger(CPURetriever.class.getName()); - private final HardwareInfo hardwareInfo; - private final CommandExecutor commandExecutor; - - public CPURetriever(HardwareInfo hardwareInfo, CommandExecutor commandExecutor) { - this.hardwareInfo = hardwareInfo; - this.commandExecutor = commandExecutor; - } - - @Override - public void updateInfo() { - try { - List<String> commandOutput = commandExecutor.executeCommand(CPU_INFO_COMMAND); - List<ParseResult> parseResults = parseCPUInfoFile(commandOutput); - setCpuCores(parseResults); - } catch (IOException e) { - logger.log(Level.WARNING, "Failed to retrieve CPU info", e); - } - } - - List<ParseResult> parseCPUInfoFile(List<String> commandOutput) { - List<String> searchWords = Collections.singletonList(SEARCH_WORD); - ParseInstructions parseInstructions = new ParseInstructions(SEARCH_ELEMENT_INDEX, RETURN_ELEMENT_INDEX, REGEX_SPLIT, searchWords); - return OutputParser.parseOutput(parseInstructions, commandOutput); - } - - void setCpuCores(List<ParseResult> parseResults) { - hardwareInfo.setMinCpuCores(countCpuCores(parseResults)); - } - - private int countCpuCores(List<ParseResult> parseResults) { - return parseResults.size(); - } - -} diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/DiskRetriever.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/DiskRetriever.java deleted file mode 100644 index 2eb6bf80bc5..00000000000 --- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/DiskRetriever.java +++ /dev/null @@ -1,108 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.verification.spec.retrievers; - -import com.yahoo.vespa.hosted.node.verification.commons.CommandExecutor; -import com.yahoo.vespa.hosted.node.verification.commons.parser.OutputParser; -import com.yahoo.vespa.hosted.node.verification.commons.parser.ParseInstructions; -import com.yahoo.vespa.hosted.node.verification.commons.parser.ParseResult; -import com.yahoo.vespa.hosted.node.verification.spec.retrievers.HardwareInfo.DiskType; - -import java.io.IOException; -import java.util.Collections; -import java.util.List; -import java.util.logging.Level; -import java.util.logging.Logger; - -/** - * Retrieves disk space and type, and stores the result in a HardwareInfo instance - * - * @author olaaun - * @author sgrostad - */ -public class DiskRetriever implements HardwareRetriever { - - private static final String DISK_CHECK_TYPE = "lsblk -d -o name,rota"; - private static final String DISK_CHECK_SIZE = "pvdisplay --units G | grep 'PV Size'"; - private static final String DISK_NAME = "sda"; - private static final String DISK_TYPE_REGEX_SPLIT = "\\s+"; - private static final int DISK_TYPE_SEARCH_ELEMENT_INDEX = 0; - private static final int DISK_TYPE_RETURN_ELEMENT_INDEX = 1; - private static final String DISK_SIZE_SEARCH_WORD = "Size"; - private static final String DISK_SIZE_REGEX_SPLIT = "\\s+"; - private static final int DISK_SIZE_SEARCH_ELEMENT_INDEX = 1; - private static final int DISK_SIZE_RETURN_ELEMENT_INDEX = 2; - private static final Logger logger = Logger.getLogger(DiskRetriever.class.getName()); - private final HardwareInfo hardwareInfo; - private final CommandExecutor commandExecutor; - - - public DiskRetriever(HardwareInfo hardwareInfo, CommandExecutor commandExecutor) { - this.hardwareInfo = hardwareInfo; - this.commandExecutor = commandExecutor; - } - - @Override - public void updateInfo() { - updateDiskType(); - updateDiskSize(); - } - - void updateDiskType() { - try { - List<String> commandOutput = commandExecutor.executeCommand(DISK_CHECK_TYPE); - ParseResult parseResult = parseDiskType(commandOutput); - setDiskType(parseResult); - } catch (IOException e) { - logger.log(Level.WARNING, "Failed to retrieve disk type", e); - } - } - - void updateDiskSize() { - try { - List<String> commandOutput = commandExecutor.executeCommand(DISK_CHECK_SIZE); - List<ParseResult> parseResult = parseDiskSize(commandOutput); - setDiskSize(parseResult); - } catch (IOException e) { - logger.log(Level.WARNING, "Failed to retrieve disk size", e); - } - } - - ParseResult parseDiskType(List<String> commandOutput) throws IOException { - List<String> searchWords = Collections.singletonList(DISK_NAME); - ParseInstructions parseInstructions = new ParseInstructions(DISK_TYPE_SEARCH_ELEMENT_INDEX, DISK_TYPE_RETURN_ELEMENT_INDEX, DISK_TYPE_REGEX_SPLIT, searchWords); - return OutputParser.parseSingleOutput(parseInstructions, commandOutput) - .orElseThrow(() -> new IOException("Parsing for disk type failed")); - } - - void setDiskType(ParseResult parseResult) { - hardwareInfo.setDiskType(DiskType.UNKNOWN); - String fastDiskSymbol = "0"; - String nonFastDiskSymbol = "1"; - if (parseResult.getValue().equals(fastDiskSymbol)) { - hardwareInfo.setDiskType(DiskType.FAST); - } else if (parseResult.getValue().equals(nonFastDiskSymbol)) { - hardwareInfo.setDiskType(DiskType.SLOW); - } - } - - List<ParseResult> parseDiskSize(List<String> commandOutput) { - List<String> searchWords = Collections.singletonList(DISK_SIZE_SEARCH_WORD); - ParseInstructions parseInstructions = new ParseInstructions(DISK_SIZE_SEARCH_ELEMENT_INDEX, DISK_SIZE_RETURN_ELEMENT_INDEX, DISK_SIZE_REGEX_SPLIT, searchWords); - return OutputParser.parseOutput(parseInstructions, commandOutput); - } - - private void setDiskSize(List<ParseResult> parseResults) { - double diskSize = 0; - try { - for (ParseResult parseResult : parseResults) { - String sizeValue = parseResult.getValue().replaceAll("[^\\d.]", ""); - diskSize += Double.parseDouble(sizeValue); - } - } catch (NumberFormatException | NullPointerException e) { - logger.log(Level.WARNING, "Parse results contained an invalid PV size - ", parseResults); - } finally { - hardwareInfo.setMinDiskAvailableGb(diskSize); - } - } - -} diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/HardwareInfo.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/HardwareInfo.java deleted file mode 100644 index 1c17d73523b..00000000000 --- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/HardwareInfo.java +++ /dev/null @@ -1,88 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.verification.spec.retrievers; - -/** - * All information the different retrievers retrieve is stored as a HardwareInfo object. - * - * @author olaaun - * @author sgrostad - */ -// TODO: This should be immutable -public class HardwareInfo { - - private double minDiskAvailableGb; - private double minMainMemoryAvailableGb; - private int minCpuCores; - private boolean ipv4Interface; - private boolean ipv6Interface; - private boolean ipv6Connection; - private double interfaceSpeedMbs; - private DiskType diskType; - - public double getInterfaceSpeedMbs() { - return interfaceSpeedMbs; - } - - public void setInterfaceSpeedMbs(double interfaceSpeedMbs) { - this.interfaceSpeedMbs = interfaceSpeedMbs; - } - - public double getMinDiskAvailableGb() { - return minDiskAvailableGb; - } - - public void setMinDiskAvailableGb(double minDiskAvailableGb) { - this.minDiskAvailableGb = minDiskAvailableGb; - } - - public boolean getIpv6Interface() { - return ipv6Interface; - } - - public void setIpv6Interface(boolean ipv6Interface) { - this.ipv6Interface = ipv6Interface; - } - - public boolean getIpv4Interface() { - return ipv4Interface; - } - - public void setIpv4Interface(boolean ipv4Interface) { - this.ipv4Interface = ipv4Interface; - } - - public boolean isIpv6Connection() { - return ipv6Connection; - } - - public void setIpv6Connection(boolean ipv6Connection) { - this.ipv6Connection = ipv6Connection; - } - - public double getMinMainMemoryAvailableGb() { - return minMainMemoryAvailableGb; - } - - public void setMinMainMemoryAvailableGb(double minMainMemoryAvailableGb) { - this.minMainMemoryAvailableGb = minMainMemoryAvailableGb; - } - - public void setDiskType(DiskType diskType) { - this.diskType = diskType; - } - - public DiskType getDiskType() { - return diskType; - } - - public int getMinCpuCores() { - return minCpuCores; - } - - public void setMinCpuCores(int minCpuCores) { - this.minCpuCores = minCpuCores; - } - - public enum DiskType {SLOW, FAST, UNKNOWN} - -} diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/HardwareInfoRetriever.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/HardwareInfoRetriever.java deleted file mode 100644 index ba29f07baeb..00000000000 --- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/HardwareInfoRetriever.java +++ /dev/null @@ -1,32 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.verification.spec.retrievers; - -import com.yahoo.vespa.hosted.node.verification.commons.CommandExecutor; -import com.yahoo.vespa.hosted.node.verification.spec.VerifierSettings; - -import java.util.ArrayList; -import java.util.List; - -/** - * Makes a HardwareInfo object and calls all the retrievers for this object. - * - * @author olaaun - * @author sgrostad - */ -public class HardwareInfoRetriever { - - public static HardwareInfo retrieve(CommandExecutor commandExecutor, VerifierSettings verifierSettings) { - HardwareInfo hardwareInfo = new HardwareInfo(); - List<HardwareRetriever> infoList = new ArrayList<>(); - infoList.add(new CPURetriever(hardwareInfo, commandExecutor)); - infoList.add(new MemoryRetriever(hardwareInfo, commandExecutor)); - infoList.add(new DiskRetriever(hardwareInfo, commandExecutor)); - infoList.add(new NetRetriever(hardwareInfo, commandExecutor, verifierSettings)); - - for (HardwareRetriever hardwareInfoType : infoList) { - hardwareInfoType.updateInfo(); - } - return hardwareInfo; - } - -} diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/HardwareRetriever.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/HardwareRetriever.java deleted file mode 100644 index fe02262e80f..00000000000 --- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/HardwareRetriever.java +++ /dev/null @@ -1,15 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.verification.spec.retrievers; - -/** - * @author olaaun - * @author sgrostad - */ -interface HardwareRetriever { - - /** - * Should retrieve spec from some part of the hardware, and store the result in hardwareinfo instance passed to class - */ - void updateInfo(); - -} diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/MemoryRetriever.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/MemoryRetriever.java deleted file mode 100644 index 2f5bbfc48dd..00000000000 --- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/MemoryRetriever.java +++ /dev/null @@ -1,67 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.verification.spec.retrievers; - -import com.yahoo.vespa.hosted.node.verification.commons.CommandExecutor; -import com.yahoo.vespa.hosted.node.verification.commons.parser.OutputParser; -import com.yahoo.vespa.hosted.node.verification.commons.parser.ParseInstructions; -import com.yahoo.vespa.hosted.node.verification.commons.parser.ParseResult; - -import java.io.IOException; -import java.util.Collections; -import java.util.List; -import java.util.logging.Level; -import java.util.logging.Logger; - -/** - * Retrieves memory size, and stores the result in a HardwareInfo instance - * - * @author olaaun - * @author sgrostad - */ -public class MemoryRetriever implements HardwareRetriever { - - private static final String MEMORY_INFO_COMMAND = "cat /proc/meminfo"; - private static final String SEARCH_WORD = "MemTotal"; - private static final String REGEX_SPLIT = ":\\s"; - private static final int SEARCH_ELEMENT_INDEX = 0; - private static final int RETURN_ELEMENT_INDEX = 1; - private static final Logger logger = Logger.getLogger(MemoryRetriever.class.getName()); - private final HardwareInfo hardwareInfo; - private final CommandExecutor commandExecutor; - - public MemoryRetriever(HardwareInfo hardwareInfo, CommandExecutor commandExecutor) { - this.hardwareInfo = hardwareInfo; - this.commandExecutor = commandExecutor; - } - - @Override - public void updateInfo() { - try { - List<String> commandOutput = commandExecutor.executeCommand(MEMORY_INFO_COMMAND); - ParseResult parseResult = parseMemInfoFile(commandOutput); - updateMemoryInfo(parseResult); - } catch (IOException e) { - logger.log(Level.WARNING, "Failed to retrieve memory info. ", e); - } - } - - ParseResult parseMemInfoFile(List<String> commandOutput) throws IOException { - List<String> searchWords = Collections.singletonList(SEARCH_WORD); - ParseInstructions parseInstructions = new ParseInstructions(SEARCH_ELEMENT_INDEX, RETURN_ELEMENT_INDEX, REGEX_SPLIT, searchWords); - return OutputParser.parseSingleOutput(parseInstructions, commandOutput) - .orElseThrow(() -> new IOException("Failed to parse memory info file.")); - } - - void updateMemoryInfo(ParseResult parseResult) { - double memory = convertKBToGB(parseResult.getValue()); - hardwareInfo.setMinMainMemoryAvailableGb(memory); - } - - double convertKBToGB(String totMem) { - String[] split = totMem.split(" "); - double value = Double.parseDouble(split[0]); - double kiloToGiga = 1000000.0; - return value / kiloToGiga; - } - -} diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/NetRetriever.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/NetRetriever.java deleted file mode 100644 index 9f147f79934..00000000000 --- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/NetRetriever.java +++ /dev/null @@ -1,158 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.verification.spec.retrievers; - -import com.yahoo.vespa.hosted.node.verification.commons.CommandExecutor; -import com.yahoo.vespa.hosted.node.verification.commons.parser.OutputParser; -import com.yahoo.vespa.hosted.node.verification.commons.parser.ParseInstructions; -import com.yahoo.vespa.hosted.node.verification.commons.parser.ParseResult; -import com.yahoo.vespa.hosted.node.verification.spec.VerifierSettings; -import org.apache.commons.exec.ExecuteException; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.List; -import java.util.logging.Level; -import java.util.logging.Logger; - -/** - * Retrieves IPv4/IPv6 interface, and checks interface speed. If node should have IPv6, tries to ping6. - * The results are stored in a HardwareInfo instance - * - * @author olaaun - * @author sgrostad - */ -public class NetRetriever implements HardwareRetriever { - - // Interface commands ignores lo-, veth- and docker interfaces - private static final String NET_FIND_INTERFACE = "/sbin/ifconfig | awk 'BEGIN {RS=\"\\n\\n\"; } { if ( $1 != \"lo\" && !match($1, \"^veth\") && !match($1, \"^docker\")) {print} }'"; - private static final String NET_CHECK_INTERFACE_SPEED = "for i in $(/sbin/ifconfig | awk 'BEGIN {RS=\"\\n\\n\"; } { if ( $1 != \"lo\" && !match($1, \"^veth\") && !match($1, \"^docker\")) {print $1} }'); do /sbin/ethtool $i; done;"; - private static final String SEARCH_WORD_INTERFACE_IP4 = "inet"; - private static final String SEARCH_WORD_INTERFACE_IPV6 = "inet6"; - private static final String SEARCH_WORD_INTERFACE_SPEED = "Speed"; - private static final String INTERFACE_NAME_REGEX_SPLIT = "\\s+"; - private static final int INTERFACE_SEARCH_ELEMENT_INDEX = 0; - private static final int INTERFACE_RETURN_ELEMENT_INDEX = 0; - private static final String INTERFACE_SPEED_REGEX_SPLIT = ":"; - private static final int INTERFACE_SPEED_SEARCH_ELEMENT_INDEX = 0; - private static final int INTERFACE_SPEED_RETURN_ELEMENT_INDEX = 1; - private static final String PING_NET_COMMAND = "ping6 -c 1 -q www.yahoo.com | grep -oP '\\d+(?=% packet loss)'"; - private static final String PING_SEARCH_WORD = "\\d+\\.?\\d*"; - private static final String PING_SPLIT_REGEX_STRING = "\\s+"; - private static final int PING_SEARCH_ELEMENT_INDEX = 0; - private static final int PING_RETURN_ELEMENT_INDEX = 0; - private static final Logger logger = Logger.getLogger(NetRetriever.class.getName()); - private final HardwareInfo hardwareInfo; - private final CommandExecutor commandExecutor; - private final VerifierSettings verifierSettings; - - - public NetRetriever(HardwareInfo hardwareInfo, CommandExecutor commandExecutor, VerifierSettings verifierSettings) { - this.hardwareInfo = hardwareInfo; - this.commandExecutor = commandExecutor; - this.verifierSettings = verifierSettings; - } - - @Override - public void updateInfo() { - List<ParseResult> parseResults = findInterface(); - findInterfaceSpeed(parseResults); - if (verifierSettings.isCheckIPv6()) { - testPingResponse(parseResults); - } else { - hardwareInfo.setIpv6Connection(true); - } - updateHardwareInfoWithNet(parseResults); - } - - List<ParseResult> findInterface() { - List<ParseResult> parseResults = new ArrayList<>(); - try { - List<String> commandOutput = commandExecutor.executeCommand(NET_FIND_INTERFACE); - parseResults = parseNetInterface(commandOutput); - - } catch (IOException e) { - logger.log(Level.WARNING, "Failed to retrieve net interface. ", e); - } - return parseResults; - } - - List<ParseResult> parseNetInterface(List<String> commandOutput) { - List<String> searchWords = new ArrayList<>(Arrays.asList(SEARCH_WORD_INTERFACE_IP4, SEARCH_WORD_INTERFACE_IPV6)); - ParseInstructions parseInstructions = new ParseInstructions(INTERFACE_SEARCH_ELEMENT_INDEX, INTERFACE_RETURN_ELEMENT_INDEX, INTERFACE_NAME_REGEX_SPLIT, searchWords); - return OutputParser.parseOutput(parseInstructions, commandOutput); - } - - void findInterfaceSpeed(List<ParseResult> parseResults) { - try { - List<String> commandOutput = commandExecutor.executeCommand(NET_CHECK_INTERFACE_SPEED); - ParseResult parseResult = parseInterfaceSpeed(commandOutput); - parseResults.add(parseResult); - } catch (IOException e) { - logger.log(Level.WARNING, "Failed to retrieve interface speed. ", e); - } - } - - ParseResult parseInterfaceSpeed(List<String> commandOutput) throws IOException { - List<String> searchWords = Collections.singletonList(SEARCH_WORD_INTERFACE_SPEED); - ParseInstructions parseInstructions = new ParseInstructions(INTERFACE_SPEED_SEARCH_ELEMENT_INDEX, INTERFACE_SPEED_RETURN_ELEMENT_INDEX, INTERFACE_SPEED_REGEX_SPLIT, searchWords); - return OutputParser.parseSingleOutput(parseInstructions, commandOutput) - .orElseThrow(() -> new IOException("Failed to parse interface speed output.")); - } - - private void testPingResponse(List<ParseResult> parseResults) { - try { - List<String> commandOutput = commandExecutor.executeCommand(PING_NET_COMMAND); - parseResults.add(parsePingResponse(commandOutput)); - } catch (ExecuteException e) { - logger.log(Level.WARNING, "Failed to execute ping6", e); - } catch (IOException e) { - logger.log(Level.WARNING, e.getMessage()); - } - } - - ParseResult parsePingResponse(List<String> commandOutput) throws IOException { - List<String> searchWords = Collections.singletonList(PING_SEARCH_WORD); - ParseInstructions parseInstructions = new ParseInstructions(PING_SEARCH_ELEMENT_INDEX, PING_RETURN_ELEMENT_INDEX, PING_SPLIT_REGEX_STRING, searchWords); - return new ParseResult(PING_SEARCH_WORD, OutputParser.parseSingleOutput(parseInstructions, commandOutput) - .orElseThrow(()->new IOException("Failed to parse ping output.")).getValue()); - } - - void updateHardwareInfoWithNet(List<ParseResult> parseResults) { - hardwareInfo.setIpv6Interface(false); - hardwareInfo.setIpv4Interface(false); - for (ParseResult parseResult : parseResults) { - switch (parseResult.getSearchWord()) { - case SEARCH_WORD_INTERFACE_IP4: - hardwareInfo.setIpv4Interface(true); - break; - case SEARCH_WORD_INTERFACE_IPV6: - hardwareInfo.setIpv6Interface(true); - break; - case SEARCH_WORD_INTERFACE_SPEED: - double speed = convertInterfaceSpeed(parseResult.getValue()); - hardwareInfo.setInterfaceSpeedMbs(speed); - break; - case PING_SEARCH_WORD: - setIpv6Connectivity(parseResult); - break; - default: - throw new RuntimeException("Invalid ParseResult search word: " + parseResult.getSearchWord()); - } - } - } - - double convertInterfaceSpeed(String speed) { - return Double.parseDouble(speed.replaceAll("[^\\d.]", "")); - } - - void setIpv6Connectivity(ParseResult parseResult) { - String pingResponse = parseResult.getValue(); - String packetLoss = pingResponse.replaceAll("[^\\d.]", ""); - if (packetLoss.equals("")) return; - if (Double.parseDouble(packetLoss) > 99) return; - hardwareInfo.setIpv6Connection(true); - } - -} diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/maintainer/CoreCollectorTest.java b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/maintainer/CoreCollectorTest.java deleted file mode 100644 index b32f6d076a7..00000000000 --- a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/maintainer/CoreCollectorTest.java +++ /dev/null @@ -1,251 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.maintainer; - -import com.yahoo.collections.Pair; -import static com.yahoo.vespa.defaults.Defaults.getDefaults; -import com.yahoo.system.ProcessExecuter; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TemporaryFolder; - -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.Set; -import java.util.stream.Collectors; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.fail; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -/** - * @author freva - */ -public class CoreCollectorTest { - private final ProcessExecuter processExecuter = mock(ProcessExecuter.class); - private final CoreCollector coreCollector = new CoreCollector(processExecuter); - - private final Path INSTALL_STATE_PATH = Paths.get("/path/to/install.state"); - private final Path TEST_CORE_PATH = Paths.get("/tmp/core.1234"); - private final Path TEST_BIN_PATH = Paths.get("/usr/bin/program"); - private final List<String> GDB_BACKTRACE = Arrays.asList("[New Thread 2703]", - "Core was generated by `/usr/bin/program\'.", "Program terminated with signal 11, Segmentation fault.", - "#0 0x00000000004004d8 in main (argv=0x1) at main.c:4", "4\t printf(argv[3]);", - "#0 0x00000000004004d8 in main (argv=0x1) at main.c:4"); - - private final List<String> INSTALL_STATE = Arrays.asList("package: some_package-0.0.2", - "variable 'value'", - "ca_file /path/to/ca.pem"); - - private final List<String> RPM_PACKAGES = Arrays.asList("some_package-0.0.2", - "another_package-1.0.6", - "last_pkg-3.10_102"); - - @Rule - public TemporaryFolder folder= new TemporaryFolder(); - - private void mockExec(String[] cmd, String output) throws IOException { - mockExec(cmd, output, ""); - } - - private void mockExec(String[] cmd, String output, String error) throws IOException { - when(processExecuter.exec(cmd)).thenReturn(new Pair<>(error.isEmpty() ? 0 : 1, output + error)); - } - - static final String GDB_PATH = getDefaults().underVespaHome("bin64/gdb"); - - @Test - public void extractsBinaryPathTest() throws IOException { - final String[] cmd = {"file", TEST_CORE_PATH.toString()}; - - mockExec(cmd, - "/tmp/core.1234: ELF 64-bit LSB core file x86-64, version 1 (SYSV), SVR4-style, from " + - "'/usr/bin/program'"); - assertEquals(TEST_BIN_PATH, coreCollector.readBinPath(TEST_CORE_PATH)); - - mockExec(cmd, - "/tmp/core.1234: ELF 64-bit LSB core file x86-64, version 1 (SYSV), SVR4-style, from " + - "'/usr/bin/program --foo --bar baz'"); - assertEquals(TEST_BIN_PATH, coreCollector.readBinPath(TEST_CORE_PATH)); - - mockExec(cmd, - "/tmp/core.1234: ELF 64-bit LSB core file x86-64, version 1 (SYSV), SVR4-style, from " + - "'/usr/bin//program'"); - assertEquals(TEST_BIN_PATH, coreCollector.readBinPath(TEST_CORE_PATH)); - - mockExec(cmd, - "/tmp/core.1234: ELF 64-bit LSB core file x86-64, version 1 (SYSV), SVR4-style, " + - "from 'program', real uid: 0, effective uid: 0, real gid: 0, effective gid: 0, " + - "execfn: '/usr/bin/program', platform: 'x86_64"); - assertEquals(TEST_BIN_PATH, coreCollector.readBinPath(TEST_CORE_PATH)); - - - Path fallbackResponse = Paths.get("/response/from/fallback"); - mockExec(new String[]{"/bin/sh", "-c", GDB_PATH + " -n -batch -core /tmp/core.1234 | grep '^Core was generated by'"}, - "Core was generated by `/response/from/fallback'."); - mockExec(cmd, - "/tmp/core.1234: ELF 64-bit LSB core file x86-64, version 1 (SYSV), SVR4-style"); - assertEquals(fallbackResponse, coreCollector.readBinPath(TEST_CORE_PATH)); - - mockExec(cmd, "", "Error code 1234"); - assertEquals(fallbackResponse, coreCollector.readBinPath(TEST_CORE_PATH)); - } - - @Test - public void extractsBinaryPathUsingGdbTest() throws IOException { - final String[] cmd = new String[]{"/bin/sh", "-c", - GDB_PATH + " -n -batch -core /tmp/core.1234 | grep '^Core was generated by'"}; - - mockExec(cmd, "Core was generated by `/usr/bin/program-from-gdb --identity foo/search/cluster.content_'."); - assertEquals(Paths.get("/usr/bin/program-from-gdb"), coreCollector.readBinPathFallback(TEST_CORE_PATH)); - - mockExec(cmd, "", "Error 123"); - try { - coreCollector.readBinPathFallback(TEST_CORE_PATH); - fail("Expected not to be able to get bin path"); - } catch (RuntimeException e) { - assertEquals("Failed to extract binary path from GDB, result: (1,Error 123), command: " + - "[/bin/sh, -c, /opt/vespa/bin64/gdb -n -batch -core /tmp/core.1234 | grep '^Core was generated by']", e.getMessage()); - } - } - - @Test - public void extractsBacktraceUsingGdb() throws IOException { - mockExec(new String[]{GDB_PATH, "-n", "-ex", "bt", "-batch", "/usr/bin/program", "/tmp/core.1234"}, - String.join("\n", GDB_BACKTRACE)); - assertEquals(GDB_BACKTRACE, coreCollector.readBacktrace(TEST_CORE_PATH, TEST_BIN_PATH, false)); - - mockExec(new String[]{GDB_PATH, "-n", "-ex", "bt", "-batch", "/usr/bin/program", "/tmp/core.1234"}, - "", "Failure"); - try { - coreCollector.readBacktrace(TEST_CORE_PATH, TEST_BIN_PATH, false); - fail("Expected not to be able to read backtrace"); - } catch (RuntimeException e) { - assertEquals("Failed to read backtrace (1,Failure), Command: " + - "[/opt/vespa/bin64/gdb, -n, -ex, bt, -batch, /usr/bin/program, /tmp/core.1234]", e.getMessage()); - } - } - - @Test - public void extractsBacktraceFromAllThreadsUsingGdb() throws IOException { - mockExec(new String[]{GDB_PATH, "-n", "-ex", "thread apply all bt", "-batch", - "/usr/bin/program", "/tmp/core.1234"}, - String.join("\n", GDB_BACKTRACE)); - assertEquals(GDB_BACKTRACE, coreCollector.readBacktrace(TEST_CORE_PATH, TEST_BIN_PATH, true)); - } - - @Test - public void collectsDataTest() throws IOException { - mockExec(new String[]{"file", TEST_CORE_PATH.toString()}, - "/tmp/core.1234: ELF 64-bit LSB core file x86-64, version 1 (SYSV), SVR4-style, from " + - "'/usr/bin/program'"); - mockExec(new String[]{GDB_PATH, "-n", "-ex", "bt", "-batch", "/usr/bin/program", "/tmp/core.1234"}, - String.join("\n", GDB_BACKTRACE)); - mockExec(new String[]{GDB_PATH, "-n", "-ex", "thread apply all bt", "-batch", - "/usr/bin/program", "/tmp/core.1234"}, - String.join("\n", GDB_BACKTRACE)); - mockExec(new String[]{"cat", INSTALL_STATE_PATH.toString()}, String.join("\n", INSTALL_STATE)); - mockExec(new String[]{"rpm", "-qa"}, String.join("\n", RPM_PACKAGES)); - - Map<String, Object> expectedData = new HashMap<>(); - expectedData.put("bin_path", TEST_BIN_PATH.toString()); - expectedData.put("backtrace", new ArrayList<>(GDB_BACKTRACE)); - expectedData.put("backtrace_all_threads", new ArrayList<>(GDB_BACKTRACE)); - expectedData.put("install_state", new ArrayList<>(INSTALL_STATE)); - expectedData.put("rpm_packages", new ArrayList<>(RPM_PACKAGES)); - assertEquals(expectedData, coreCollector.collect(TEST_CORE_PATH, Optional.of(INSTALL_STATE_PATH))); - } - - @Test - public void collectsPartialIfUnableToDetermineDumpingProgramTest() throws IOException { - // We fail to get backtrace and RPM packages, but install state works, make sure it is returned - mockExec(new String[]{"cat", INSTALL_STATE_PATH.toString()}, String.join("\n", INSTALL_STATE)); - - Map<String, Object> expectedData = new HashMap<>(); - expectedData.put("install_state", new ArrayList<>(INSTALL_STATE)); - assertEquals(expectedData, coreCollector.collect(TEST_CORE_PATH, Optional.of(INSTALL_STATE_PATH))); - } - - @Test - public void collectsPartialIfBacktraceFailsTest() throws IOException { - mockExec(new String[]{"file", TEST_CORE_PATH.toString()}, - "/tmp/core.1234: ELF 64-bit LSB core file x86-64, version 1 (SYSV), SVR4-style, from " + - "'/usr/bin/program'"); - mockExec(new String[]{GDB_PATH + " -n -ex bt -batch /usr/bin/program /tmp/core.1234"}, - "", "Failure"); - - Map<String, Object> expectedData = new HashMap<>(); - expectedData.put("bin_path", TEST_BIN_PATH.toString()); - assertEquals(expectedData, coreCollector.collect(TEST_CORE_PATH, Optional.empty())); - } - - @Test - public void parseTotalMemoryTestTest() { - String memInfo = "MemTotal: 100000000 kB\nMemUsed: 1000000 kB\n"; - assertEquals(100000000, coreCollector.parseTotalMemorySize(memInfo)); - - String badMemInfo = "This string has no memTotal value"; - try { - coreCollector.parseTotalMemorySize(badMemInfo); - fail("Expected to fail on parsing"); - } catch (RuntimeException e) { - assertEquals("Could not parse meminfo: " + badMemInfo, e.getMessage()); - } - } - - @Test - public void testDeleteUncompressedFiles() throws IOException { - final String documentId = "UIDD-ABCD-EFGH"; - final String coreDumpFilename = "core.dump"; - - Path coredumpPath = folder.newFolder("crash").toPath() - .resolve(CoredumpHandler.PROCESSING_DIRECTORY_NAME) - .resolve(documentId); - coredumpPath.toFile().mkdirs(); - coredumpPath.resolve(coreDumpFilename).toFile().createNewFile(); - - Set<Path> expectedContentsOfCoredump = new HashSet<>(Arrays.asList( - coredumpPath.resolve(CoredumpHandler.METADATA_FILE_NAME), - coredumpPath.resolve(coreDumpFilename + ".lz4"))); - expectedContentsOfCoredump.forEach(path -> { - try { - path.toFile().createNewFile(); - } catch (IOException ignored) { ignored.printStackTrace();} - }); - coreCollector.deleteDecompressedCoredump(coredumpPath.resolve(coreDumpFilename)); - - assertEquals(expectedContentsOfCoredump, Files.list(coredumpPath).collect(Collectors.toSet())); - } - - @Test - public void testDeleteUncompressedFilesWithoutLz4() throws IOException { - final String documentId = "UIDD-ABCD-EFGH"; - final String coreDumpFilename = "core.dump"; - - Path coredumpPath = folder.newFolder("crash").toPath() - .resolve(CoredumpHandler.PROCESSING_DIRECTORY_NAME) - .resolve(documentId); - coredumpPath.toFile().mkdirs(); - - Set<Path> expectedContentsOfCoredump = new HashSet<>(Arrays.asList( - coredumpPath.resolve(CoredumpHandler.METADATA_FILE_NAME), - coredumpPath.resolve(coreDumpFilename))); - expectedContentsOfCoredump.forEach(path -> { - try { - path.toFile().createNewFile(); - } catch (IOException ignored) { ignored.printStackTrace();} - }); - coreCollector.deleteDecompressedCoredump(coredumpPath.resolve(coreDumpFilename)); - - assertEquals(expectedContentsOfCoredump, Files.list(coredumpPath).collect(Collectors.toSet())); - } -} diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/maintainer/CoredumpHandlerTest.java b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/maintainer/CoredumpHandlerTest.java deleted file mode 100644 index a0125e78ca2..00000000000 --- a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/maintainer/CoredumpHandlerTest.java +++ /dev/null @@ -1,248 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.maintainer; - -import org.apache.http.HttpHeaders; -import org.apache.http.HttpVersion; -import org.apache.http.client.methods.CloseableHttpResponse; -import org.apache.http.client.methods.HttpPost; -import org.apache.http.entity.StringEntity; -import org.apache.http.impl.client.CloseableHttpClient; -import org.apache.http.message.BasicStatusLine; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TemporaryFolder; -import org.mockito.ArgumentCaptor; - -import java.io.BufferedReader; -import java.io.IOException; -import java.io.InputStreamReader; -import java.net.URI; -import java.net.URISyntaxException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.time.Instant; -import java.util.Arrays; -import java.util.Collections; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.Set; -import java.util.stream.Collectors; - -import static org.junit.Assert.assertEquals; -import static org.mockito.Matchers.any; -import static org.mockito.Matchers.eq; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.verifyZeroInteractions; -import static org.mockito.Mockito.when; - -/** - * @author freva - */ -public class CoredumpHandlerTest { - - private final CloseableHttpClient httpClient = mock(CloseableHttpClient.class); - private final CoreCollector coreCollector = mock(CoreCollector.class); - private static final Map<String, Object> attributes = new LinkedHashMap<>(); - private static final Map<String, Object> metadata = new LinkedHashMap<>(); - private static final String expectedMetadataFileContents = "{\"fields\":{" + - "\"bin_path\":\"/bin/bash\"," + - "\"backtrace\":[\"call 1\",\"function 2\",\"something something\"]," + - "\"hostname\":\"host123.yahoo.com\"," + - "\"vespa_version\":\"6.48.4\"," + - "\"kernel_version\":\"2.6.32-573.22.1.el6.YAHOO.20160401.10.x86_64\"," + - "\"docker_image\":\"vespa/ci:6.48.4\"}}"; - private static final String feedEndpoint = "http://feed-endpoint.hostname.tld/feed"; - - static { - attributes.put("hostname", "host123.yahoo.com"); - attributes.put("vespa_version", "6.48.4"); - attributes.put("kernel_version", "2.6.32-573.22.1.el6.YAHOO.20160401.10.x86_64"); - attributes.put("docker_image", "vespa/ci:6.48.4"); - - metadata.put("bin_path", "/bin/bash"); - metadata.put("backtrace", Arrays.asList("call 1", "function 2", "something something")); - } - - @Rule - public TemporaryFolder folder = new TemporaryFolder(); - - private CoredumpHandler coredumpHandler; - private Path crashPath; - private Path donePath; - - @Before - public void setup() throws IOException { - crashPath = folder.newFolder("crash").toPath(); - donePath = folder.newFolder("done").toPath(); - - coredumpHandler = new CoredumpHandler(httpClient, coreCollector, crashPath, donePath, attributes, - Optional.empty(), feedEndpoint); - } - - @Test - public void ignoresIncompleteCoredumps() throws IOException { - Path coredumpPath = createCoredump(".core.dump", Instant.now()); - Path processingPath = coredumpHandler.enqueueCoredumps(); - - // The 'processing' directory should be empty - assertFolderContents(processingPath); - - // The 'crash' directory should have 'processing' and the incomplete core dump in it - assertFolderContents(crashPath, processingPath.getFileName().toString(), coredumpPath.getFileName().toString()); - } - - @Test - public void startProcessingTest() throws IOException { - Path coredumpPath = createCoredump("core.dump", Instant.now()); - Path processingPath = crashPath.resolve("processing_dir"); - coredumpHandler.enqueueCoredumpForProcessing(coredumpPath, processingPath); - - // Contents of 'crash' should be only the 'processing' directory - assertFolderContents(crashPath, processingPath.getFileName().toString()); - - // The 'processing' directory should have 1 directory inside for the core.dump we just created - List<Path> processedCoredumps = Files.list(processingPath).collect(Collectors.toList()); - assertEquals(processedCoredumps.size(), 1); - - // Inside the coredump directory, there should be 1 file: core.dump - assertFolderContents(processedCoredumps.get(0), coredumpPath.getFileName().toString()); - } - - @Test - public void limitToProcessingOneCoredumpAtTheTimeTest() throws IOException { - final String oldestCoredump = "core.dump0"; - final Instant startTime = Instant.now(); - createCoredump(oldestCoredump, startTime.minusSeconds(3600)); - createCoredump("core.dump1", startTime.minusSeconds(1000)); - createCoredump("core.dump2", startTime); - Path processingPath = coredumpHandler.enqueueCoredumps(); - - List<Path> processingCoredumps = Files.list(processingPath).collect(Collectors.toList()); - assertEquals(1, processingCoredumps.size()); - - // Make sure that the 1 coredump that we are processing is the oldest one - Set<String> filenamesInProcessingDirectory = Files.list(processingCoredumps.get(0)) - .map(file -> file.getFileName().toString()) - .collect(Collectors.toSet()); - assertEquals(Collections.singleton(oldestCoredump), filenamesInProcessingDirectory); - - // Running enqueueCoredumps should not start processing any new coredumps as we already are processing one - coredumpHandler.enqueueCoredumps(); - assertEquals(processingCoredumps, Files.list(processingPath).collect(Collectors.toList())); - filenamesInProcessingDirectory = Files.list(processingCoredumps.get(0)) - .map(file -> file.getFileName().toString()) - .collect(Collectors.toSet()); - assertEquals(Collections.singleton(oldestCoredump), filenamesInProcessingDirectory); - } - - @Test - public void coredumpMetadataCollectAndWriteTest() throws IOException { - createCoredump("core.dump", Instant.now()); - Path processingPath = coredumpHandler.enqueueCoredumps(); - Path processingCoredumpPath = Files.list(processingPath).findFirst().orElseThrow(() -> - new RuntimeException("Expected to find directory with coredump in processing dir")); - when(coreCollector.collect(eq(processingCoredumpPath.resolve("core.dump")), any())).thenReturn(metadata); - - // Inside 'processing' directory, there should be a new directory containing 'core.dump' file - String returnedMetadata = coredumpHandler.collectMetadata(processingCoredumpPath, attributes); - String metadataFileContents = new String(Files.readAllBytes( - processingCoredumpPath.resolve(CoredumpHandler.METADATA_FILE_NAME))); - assertEquals(expectedMetadataFileContents, metadataFileContents); - assertEquals(expectedMetadataFileContents, returnedMetadata); - } - - @Test - public void coredumpMetadataReadIfExistsTest() throws IOException { - final String documentId = "UIDD-ABCD-EFGH"; - Path metadataPath = createProcessedCoredump(documentId); - - verifyZeroInteractions(coreCollector); - String returnedMetadata = coredumpHandler.collectMetadata(metadataPath.getParent(), attributes); - assertEquals(expectedMetadataFileContents, returnedMetadata); - } - - @Test - public void reportSuccessCoredumpTest() throws IOException, URISyntaxException { - final String documentId = "UIDD-ABCD-EFGH"; - Path coredumpPath = createProcessedCoredump(documentId); - - setNextHttpResponse(200, Optional.empty()); - coredumpHandler.report(coredumpPath.getParent(), expectedMetadataFileContents); - validateNextHttpPost(documentId, expectedMetadataFileContents); - } - - @Test - public void reportFailCoredumpTest() throws IOException, URISyntaxException { - final String documentId = "UIDD-ABCD-EFGH"; - Path metadataPath = createProcessedCoredump(documentId); - - setNextHttpResponse(500, Optional.of("Internal server error")); - coredumpHandler.processAndReportCoredumps(crashPath.resolve(CoredumpHandler.PROCESSING_DIRECTORY_NAME)); - validateNextHttpPost(documentId, expectedMetadataFileContents); - - // The coredump should not have been moved out of 'processing' and into 'done' as the report failed - assertFolderContents(donePath); - assertFolderContents(metadataPath.getParent(), CoredumpHandler.METADATA_FILE_NAME); - } - - @Test - public void finishProcessingTest() throws IOException { - final String documentId = "UIDD-ABCD-EFGH"; - - Path coredumpPath = createProcessedCoredump(documentId); - coredumpHandler.finishProcessing(coredumpPath.getParent()); - - // The coredump should've been moved out of 'processing' and into 'done' - assertFolderContents(crashPath.resolve(CoredumpHandler.PROCESSING_DIRECTORY_NAME)); - assertFolderContents(donePath.resolve(documentId), CoredumpHandler.METADATA_FILE_NAME); - } - - - private static void assertFolderContents(Path pathToFolder, String... filenames) throws IOException { - Set<Path> expectedContentsOfFolder = Arrays.stream(filenames) - .map(pathToFolder::resolve) - .collect(Collectors.toSet()); - Set<Path> actualContentsOfFolder = Files.list(pathToFolder).collect(Collectors.toSet()); - assertEquals(expectedContentsOfFolder, actualContentsOfFolder); - } - - private Path createCoredump(String coredumpName, Instant lastModified) throws IOException { - Path coredumpPath = crashPath.resolve(coredumpName); - coredumpPath.toFile().createNewFile(); - coredumpPath.toFile().setLastModified(lastModified.toEpochMilli()); - return coredumpPath; - } - - private Path createProcessedCoredump(String documentId) throws IOException { - Path coredumpPath = crashPath - .resolve(CoredumpHandler.PROCESSING_DIRECTORY_NAME) - .resolve(documentId) - .resolve(CoredumpHandler.METADATA_FILE_NAME); - coredumpPath.getParent().toFile().mkdirs(); - return Files.write(coredumpPath, expectedMetadataFileContents.getBytes()); - } - - private void setNextHttpResponse(int code, Optional<String> message) throws IOException { - CloseableHttpResponse response = mock(CloseableHttpResponse.class); - when(response.getStatusLine()).thenReturn(new BasicStatusLine(HttpVersion.HTTP_1_1, code, null)); - if (message.isPresent()) when(response.getEntity()).thenReturn(new StringEntity(message.get())); - - when(httpClient.execute(any())).thenReturn(response); - } - - private void validateNextHttpPost(String documentId, String expectedBody) throws IOException, URISyntaxException { - ArgumentCaptor<HttpPost> capturedPost = ArgumentCaptor.forClass(HttpPost.class); - verify(httpClient).execute(capturedPost.capture()); - - URI expectedURI = new URI(feedEndpoint + "/" + documentId); - assertEquals(expectedURI, capturedPost.getValue().getURI()); - assertEquals("application/json", capturedPost.getValue().getHeaders(HttpHeaders.CONTENT_TYPE)[0].getValue()); - assertEquals(expectedBody, - new BufferedReader(new InputStreamReader(capturedPost.getValue().getEntity().getContent())).readLine()); - } - -} diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/maintainer/FileHelperTest.java b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/maintainer/FileHelperTest.java deleted file mode 100644 index eefdcf4adad..00000000000 --- a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/maintainer/FileHelperTest.java +++ /dev/null @@ -1,325 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.maintainer; - -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TemporaryFolder; - -import java.io.File; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.time.Duration; -import java.util.Arrays; -import java.util.Collections; -import java.util.Optional; - -import static org.junit.Assert.assertArrayEquals; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; - -/** - * @author freva - */ -public class FileHelperTest { - @Rule - public TemporaryFolder folder = new TemporaryFolder(); - - @Before - public void initFiles() throws IOException { - for (int i=0; i<10; i++) { - File temp = folder.newFile("test_" + i + ".json"); - temp.setLastModified(System.currentTimeMillis() - i*Duration.ofSeconds(130).toMillis()); - } - - for (int i=0; i<7; i++) { - File temp = folder.newFile("test_" + i + "_file.test"); - temp.setLastModified(System.currentTimeMillis() - i*Duration.ofSeconds(250).toMillis()); - } - - for (int i=0; i<5; i++) { - File temp = folder.newFile(i + "-abc" + ".json"); - temp.setLastModified(System.currentTimeMillis() - i*Duration.ofSeconds(80).toMillis()); - } - - File temp = folder.newFile("week_old_file.json"); - temp.setLastModified(System.currentTimeMillis() - Duration.ofDays(8).toMillis()); - } - - @Test - public void testDeleteAll() throws IOException { - FileHelper.deleteFiles(folder.getRoot().toPath(), Duration.ZERO, Optional.empty(), false); - - assertEquals(0, getContentsOfDirectory(folder.getRoot()).length); - } - - @Test - public void testDeletePrefix() throws IOException { - FileHelper.deleteFiles(folder.getRoot().toPath(), Duration.ZERO, Optional.of("^test_"), false); - - assertEquals(6, getContentsOfDirectory(folder.getRoot()).length); // 5 abc files + 1 week_old_file - } - - @Test - public void testDeleteSuffix() throws IOException { - FileHelper.deleteFiles(folder.getRoot().toPath(), Duration.ZERO, Optional.of(".json$"), false); - - assertEquals(7, getContentsOfDirectory(folder.getRoot()).length); - } - - @Test - public void testDeletePrefixAndSuffix() throws IOException { - FileHelper.deleteFiles(folder.getRoot().toPath(), Duration.ZERO, Optional.of("^test_.*\\.json$"), false); - - assertEquals(13, getContentsOfDirectory(folder.getRoot()).length); // 5 abc files + 7 test_*_file.test files + week_old_file - } - - @Test - public void testDeleteOld() throws IOException { - FileHelper.deleteFiles(folder.getRoot().toPath(), Duration.ofSeconds(600), Optional.empty(), false); - - assertEquals(13, getContentsOfDirectory(folder.getRoot()).length); // All 23 - 6 (from test_*_.json) - 3 (from test_*_file.test) - 1 week old file - } - - @Test - public void testDeleteWithAllParameters() throws IOException { - FileHelper.deleteFiles(folder.getRoot().toPath(), Duration.ofSeconds(200), Optional.of("^test_.*\\.json$"), false); - - assertEquals(15, getContentsOfDirectory(folder.getRoot()).length); // All 23 - 8 (from test_*_.json) - } - - @Test - public void testDeleteWithSubDirectoriesNoRecursive() throws IOException { - initSubDirectories(); - FileHelper.deleteFiles(folder.getRoot().toPath(), Duration.ZERO, Optional.of("^test_.*\\.json$"), false); - - // 6 test_*.json from test_folder1/ - // + 9 test_*.json and 4 abc_*.json from test_folder2/ - // + 13 test_*.json from test_folder2/subSubFolder2/ - // + 7 test_*_file.test and 5 *-abc.json and 1 week_old_file from root - // + test_folder1/ and test_folder2/ and test_folder2/subSubFolder2/ themselves - assertEquals(48, getNumberOfFilesAndDirectoriesIn(folder.getRoot())); - } - - @Test - public void testDeleteWithSubDirectoriesRecursive() throws IOException { - initSubDirectories(); - FileHelper.deleteFiles(folder.getRoot().toPath(), Duration.ZERO, Optional.of("^test_.*\\.json$"), true); - - // 4 abc_*.json from test_folder2/ - // + 7 test_*_file.test and 5 *-abc.json and 1 week_old_file from root - // + test_folder2/ itself - assertEquals(18, getNumberOfFilesAndDirectoriesIn(folder.getRoot())); - } - - @Test - public void testDeleteFilesWhereFilenameRegexAlsoMatchesDirectories() throws IOException { - initSubDirectories(); - - FileHelper.deleteFiles(folder.getRoot().toPath(), Duration.ZERO, Optional.of("^test_"), false); - - assertEquals(8, getContentsOfDirectory(folder.getRoot()).length); // 5 abc files + 1 week_old_file + 2 directories - } - - @Test - public void testGetContentsOfNonExistingDirectory() { - Path fakePath = Paths.get("/some/made/up/dir/"); - assertEquals(Collections.emptyList(), FileHelper.listContentsOfDirectory(fakePath)); - } - - @Test(expected=IllegalArgumentException.class) - public void testDeleteFilesExceptNMostRecentWithNegativeN() throws IOException { - FileHelper.deleteFilesExceptNMostRecent(folder.getRoot().toPath(), -5); - } - - @Test - public void testDeleteFilesExceptFiveMostRecent() throws IOException { - FileHelper.deleteFilesExceptNMostRecent(folder.getRoot().toPath(), 5); - - assertEquals(5, getContentsOfDirectory(folder.getRoot()).length); - - String[] oldestFiles = {"test_5_file.test", "test_6_file.test", "test_8.json", "test_9.json", "week_old_file.json"}; - String[] remainingFiles = Arrays.stream(getContentsOfDirectory(folder.getRoot())) - .map(File::getName) - .sorted() - .toArray(String[]::new); - - assertArrayEquals(oldestFiles, remainingFiles); - } - - @Test - public void testDeleteFilesExceptNMostRecentWithLargeN() throws IOException { - String[] filesPreDelete = folder.getRoot().list(); - - FileHelper.deleteFilesExceptNMostRecent(folder.getRoot().toPath(), 50); - - assertArrayEquals(filesPreDelete, folder.getRoot().list()); - } - - @Test - public void testDeleteFilesLargerThan10B() throws IOException { - initSubDirectories(); - - File temp1 = new File(folder.getRoot(), "small_file"); - writeNBytesToFile(temp1, 50); - - File temp2 = new File(folder.getRoot(), "some_file"); - writeNBytesToFile(temp2, 20); - - File temp3 = new File(folder.getRoot(), "test_folder1/some_other_file"); - writeNBytesToFile(temp3, 75); - - FileHelper.deleteFilesLargerThan(folder.getRoot().toPath(), 10); - - assertEquals(58, getNumberOfFilesAndDirectoriesIn(folder.getRoot())); - assertFalse(temp1.exists() || temp2.exists() || temp3.exists()); - } - - @Test - public void testDeleteDirectories() throws IOException { - initSubDirectories(); - - FileHelper.deleteDirectories(folder.getRoot().toPath(), Duration.ZERO, Optional.of(".*folder2")); - - //23 files in root - // + 6 in test_folder1 + test_folder1 itself - assertEquals(30, getNumberOfFilesAndDirectoriesIn(folder.getRoot())); - } - - @Test - public void testDeleteDirectoriesBasedOnAge() throws IOException { - initSubDirectories(); - // Create folder3 which is older than maxAge, inside have a single directory, subSubFolder3, inside it which is - // also older than maxAge inside the sub directory, create some files which are newer than maxAge. - // deleteDirectories() should NOT delete folder3 - File subFolder3 = folder.newFolder("test_folder3"); - File subSubFolder3 = folder.newFolder("test_folder3", "subSubFolder3"); - - for (int j=0; j<11; j++) { - File.createTempFile("test_", ".json", subSubFolder3); - } - - subFolder3.setLastModified(System.currentTimeMillis() - Duration.ofHours(1).toMillis()); - subSubFolder3.setLastModified(System.currentTimeMillis() - Duration.ofHours(3).toMillis()); - - FileHelper.deleteDirectories(folder.getRoot().toPath(), Duration.ofSeconds(50), Optional.of(".*folder.*")); - - //23 files in root - // + 13 in test_folder2 - // + 13 in subSubFolder2 - // + 11 in subSubFolder3 - // + test_folder2 + subSubFolder2 + folder3 + subSubFolder3 itself - assertEquals(64, getNumberOfFilesAndDirectoriesIn(folder.getRoot())); - } - - @Test - public void testRecursivelyDeleteDirectory() throws IOException { - initSubDirectories(); - FileHelper.recursiveDelete(folder.getRoot().toPath()); - assertFalse(folder.getRoot().exists()); - } - - @Test - public void testRecursivelyDeleteRegularFile() throws IOException { - File file = folder.newFile(); - assertTrue(file.exists()); - assertTrue(file.isFile()); - FileHelper.recursiveDelete(file.toPath()); - assertFalse(file.exists()); - } - - @Test - public void testRecursivelyDeleteNonExistingFile() throws IOException { - File file = folder.getRoot().toPath().resolve("non-existing-file.json").toFile(); - assertFalse(file.exists()); - FileHelper.recursiveDelete(file.toPath()); - assertFalse(file.exists()); - } - - @Test - public void testInitSubDirectories() throws IOException { - initSubDirectories(); - assertTrue(folder.getRoot().exists()); - assertTrue(folder.getRoot().isDirectory()); - - Path test_folder1 = folder.getRoot().toPath().resolve("test_folder1"); - assertTrue(test_folder1.toFile().exists()); - assertTrue(test_folder1.toFile().isDirectory()); - - Path test_folder2 = folder.getRoot().toPath().resolve("test_folder2"); - assertTrue(test_folder2.toFile().exists()); - assertTrue(test_folder2.toFile().isDirectory()); - - Path subSubFolder2 = test_folder2.resolve("subSubFolder2"); - assertTrue(subSubFolder2.toFile().exists()); - assertTrue(subSubFolder2.toFile().isDirectory()); - } - - @Test - public void testDoesNotFailOnLastModifiedOnSymLink() throws IOException { - Path symPath = folder.getRoot().toPath().resolve("symlink"); - Path fakePath = Paths.get("/some/not/existant/file"); - - Files.createSymbolicLink(symPath, fakePath); - assertTrue(Files.isSymbolicLink(symPath)); - assertFalse(Files.exists(fakePath)); - - // Not possible to set modified time on symlink in java, so just check that it doesn't crash - FileHelper.getLastModifiedTime(symPath).toInstant(); - } - - private void initSubDirectories() throws IOException { - File subFolder1 = folder.newFolder("test_folder1"); - File subFolder2 = folder.newFolder("test_folder2"); - File subSubFolder2 = folder.newFolder("test_folder2", "subSubFolder2"); - - for (int j=0; j<6; j++) { - File temp = File.createTempFile("test_", ".json", subFolder1); - temp.setLastModified(System.currentTimeMillis() - (j+1)*Duration.ofSeconds(60).toMillis()); - } - - for (int j=0; j<9; j++) { - File.createTempFile("test_", ".json", subFolder2); - } - - for (int j=0; j<4; j++) { - File.createTempFile("abc_", ".txt", subFolder2); - } - - for (int j=0; j<13; j++) { - File temp = File.createTempFile("test_", ".json", subSubFolder2); - temp.setLastModified(System.currentTimeMillis() - (j+1)*Duration.ofSeconds(40).toMillis()); - } - - //Must be after all the files have been created - subFolder1.setLastModified(System.currentTimeMillis() - Duration.ofHours(2).toMillis()); - subFolder2.setLastModified(System.currentTimeMillis() - Duration.ofHours(1).toMillis()); - subSubFolder2.setLastModified(System.currentTimeMillis() - Duration.ofHours(3).toMillis()); - } - - private static int getNumberOfFilesAndDirectoriesIn(File folder) { - int total = 0; - for (File file : folder.listFiles()) { - if (file.isDirectory()) { - total += getNumberOfFilesAndDirectoriesIn(file); - } - total++; - } - - return total; - } - - private static void writeNBytesToFile(File file, int nBytes) throws IOException { - Files.write(file.toPath(), new byte[nBytes]); - } - - - static File[] getContentsOfDirectory(File directory) { - File[] directoryContents = directory.listFiles(); - - return directoryContents == null ? new File[0] : directoryContents; - } -} diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/commons/CommandExecutorTest.java b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/commons/CommandExecutorTest.java deleted file mode 100644 index 5f5aca825c0..00000000000 --- a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/commons/CommandExecutorTest.java +++ /dev/null @@ -1,35 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.verification.commons; - -import org.junit.Before; -import org.junit.Test; - -import java.io.IOException; -import java.util.List; - -import static java.util.Arrays.asList; -import static org.junit.Assert.assertEquals; - -/** - * @author sgrostad - * @author olaaun - */ - -public class CommandExecutorTest { - - private CommandExecutor commandExecutor; - - @Before - public void setup() { - commandExecutor = new CommandExecutor(); - } - - @Test - public void test_if_executeAString_reads_testReadFile_correct() throws IOException { - String command = "cat src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/resources/testReadFile.txt"; - List<String> commandOutput = commandExecutor.executeCommand(command); - List<String> expectedOutput = asList("This test file tests apache commons exec", "Second line"); - assertEquals(expectedOutput, commandOutput); - } - -} diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/commons/noderepo/IPAddressVerifierTest.java b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/commons/noderepo/IPAddressVerifierTest.java deleted file mode 100644 index c30afbf4fe5..00000000000 --- a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/commons/noderepo/IPAddressVerifierTest.java +++ /dev/null @@ -1,130 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.verification.commons.noderepo; - -import org.junit.Before; -import org.junit.Test; - -import java.net.InetAddress; -import java.net.UnknownHostException; -import java.util.Arrays; - -import static org.junit.Assert.assertArrayEquals; -import static org.junit.Assert.assertEquals; -import static org.mockito.Mockito.doReturn; -import static org.mockito.Mockito.doThrow; -import static org.mockito.Mockito.spy; - -/** - * @author sgrostad - * @author olaaun - */ -public class IPAddressVerifierTest { - - private final String ipv4Address = "10.2.4.8"; - private final String ipv6Address = "fdab:0:0:0:0:0:0:1234"; - private final NodeSpec nodeSpec = new NodeSpec(1920, 256, 48, true, 10_000, new String[]{ipv4Address, ipv6Address}); - private final String hostname = "test123.region.domain.tld"; - - private IPAddressVerifier ipAddressVerifier = spy(new IPAddressVerifier(hostname, false, false)); - private String ipv4LookupFormat; - private String ipv6LookupFormat; - - @Before - public void setup() { - ipv4LookupFormat = "8.4.2.10.in-addr.arpa"; - ipv6LookupFormat = "4.3.2.1.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.b.a.d.f.ip6.arpa"; - } - - @Test - public void getFaultyIpAddresses_with_hostname_resolving_to_other_ips() throws Exception { - doReturn(Arrays.asList(InetAddress.getByName("1.2.3.4"), InetAddress.getByName("fd00::1"))).when(ipAddressVerifier).mockableGetAllByName(hostname); - String[] faultyIpAddresses = ipAddressVerifier.getFaultyIpAddresses(nodeSpec); - String[] expectedFaultyIpAddresses = new String[]{ipv4Address, ipv6Address}; - assertArrayEquals(expectedFaultyIpAddresses, faultyIpAddresses); - } - - @Test - public void getFaultyIpAddresses_with_hostname_not_resolving_to_ipv4_address() throws Exception { - doReturn(Arrays.asList(InetAddress.getByName(ipv6Address))).when(ipAddressVerifier).mockableGetAllByName(hostname); - doReturn(hostname).when(ipAddressVerifier).reverseLookUp(ipv6LookupFormat); - String[] faultyIpAddresses = ipAddressVerifier.getFaultyIpAddresses(nodeSpec); - String[] expectedFaultyIpAddresses = new String[]{ipv4Address}; - assertArrayEquals(expectedFaultyIpAddresses, faultyIpAddresses); - } - - @Test - public void getFaultyIpAddresses_with_failing_hostname_resolution() throws Exception { - doThrow(new UnknownHostException("bad hostname")).when(ipAddressVerifier).mockableGetAllByName(hostname); - String[] faultyIpAddresses = ipAddressVerifier.getFaultyIpAddresses(nodeSpec); - String[] expectedFaultyIpAddresses = new String[]{ipv4Address, ipv6Address}; - assertArrayEquals(expectedFaultyIpAddresses, faultyIpAddresses); - } - - @Test - public void getFaultyIpAddresses_with_hostname_not_resolving_to_ipv6_address() throws Exception { - doReturn(Arrays.asList(InetAddress.getByName(ipv4Address))).when(ipAddressVerifier).mockableGetAllByName(hostname); - doReturn(hostname).when(ipAddressVerifier).reverseLookUp(ipv4LookupFormat); - String[] faultyIpAddresses = ipAddressVerifier.getFaultyIpAddresses(nodeSpec); - String[] expectedFaultyIpAddresses = new String[]{ipv6Address}; - assertArrayEquals(expectedFaultyIpAddresses, faultyIpAddresses); - } - - @Test - public void getFaultyIpAddresses_should_return_IP_address_when_different_hostname() throws Exception { - String wrongHostName = "www.yahoo.com"; - doReturn(Arrays.asList(InetAddress.getByName(ipv4Address), InetAddress.getByName(ipv6Address))).when(ipAddressVerifier).mockableGetAllByName(hostname); - doReturn(hostname).when(ipAddressVerifier).reverseLookUp(ipv4LookupFormat); - doReturn(wrongHostName).when(ipAddressVerifier).reverseLookUp(ipv6LookupFormat); - String[] faultyIpAddresses = ipAddressVerifier.getFaultyIpAddresses(nodeSpec); - String[] expectedFaultyIpAddresses = new String[]{ipv6Address}; - assertArrayEquals(expectedFaultyIpAddresses, faultyIpAddresses); - } - - @Test - public void getFaultyIpAddresses_should_return_empty_array_when_all_addresses_point_to_correct_hostname() throws Exception { - doReturn(Arrays.asList(InetAddress.getByName(ipv4Address), InetAddress.getByName(ipv6Address))).when(ipAddressVerifier).mockableGetAllByName(hostname); - doReturn(hostname).when(ipAddressVerifier).reverseLookUp(ipv4LookupFormat); - doReturn(hostname).when(ipAddressVerifier).reverseLookUp(ipv6LookupFormat); - String[] faultyIpAddresses = ipAddressVerifier.getFaultyIpAddresses(nodeSpec); - String[] expectedFaultyIpAddresses = new String[]{}; - assertArrayEquals(expectedFaultyIpAddresses, faultyIpAddresses); - } - - @Test - public void getFaultyIpAddresses_should_return_empty_array_when_lookup_is_skipped() throws Exception { - ipAddressVerifier = spy(new IPAddressVerifier(hostname, true, false)); - doReturn(hostname).when(ipAddressVerifier).reverseLookUp(ipv4LookupFormat); - doReturn(hostname).when(ipAddressVerifier).reverseLookUp(ipv6LookupFormat); - String[] faultyIpAddresses = ipAddressVerifier.getFaultyIpAddresses(nodeSpec); - String[] expectedFaultyIpAddresses = new String[]{}; - assertArrayEquals(expectedFaultyIpAddresses, faultyIpAddresses); - } - - @Test - public void getFaultyIpAddresses_should_return_empty_array_when_reverse_lookup_is_skipped() throws Exception { - ipAddressVerifier = spy(new IPAddressVerifier(hostname, false, true)); - doReturn(Arrays.asList(InetAddress.getByName(ipv4Address), InetAddress.getByName(ipv6Address))).when(ipAddressVerifier).mockableGetAllByName(hostname); - String[] faultyIpAddresses = ipAddressVerifier.getFaultyIpAddresses(nodeSpec); - String[] expectedFaultyIpAddresses = new String[]{}; - assertArrayEquals(expectedFaultyIpAddresses, faultyIpAddresses); - } - - @Test - public void convertIpv6ToLookupFormat_should_return_properly_converted_ipv6_address() { - String actualConvertedAddress = ipAddressVerifier.convertIpv6ToLookupFormat(ipv6Address); - assertEquals(ipv6LookupFormat, actualConvertedAddress); - } - - @Test - public void convertIpv4ToLookupFormat_should_return_properly_converted_ipv6_address() { - String actualConvertedAddress = ipAddressVerifier.convertIpv4ToLookupFormat(ipv4Address); - assertEquals(ipv4LookupFormat, actualConvertedAddress); - } - - @Test - public void getFaultyIpAddresses_should_return_empty_array_when_parameters_are_invalid() { - final NodeSpec nodeWithNoIP = new NodeSpec(1920, 256, 48, true, 10_000, new String[0]); - assertEquals(0, ipAddressVerifier.getFaultyIpAddresses(nodeWithNoIP).length); - } - -} diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/commons/parser/OutputParserTest.java b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/commons/parser/OutputParserTest.java deleted file mode 100644 index 06d34a452eb..00000000000 --- a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/commons/parser/OutputParserTest.java +++ /dev/null @@ -1,87 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.verification.commons.parser; - -import org.junit.Before; -import org.junit.Test; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.List; -import java.util.Optional; - -import static org.junit.Assert.assertEquals; - -/** - * @author sgrostad - * @author olaaun - */ - -public class OutputParserTest { - - private static final String RETURN_VALUE = "#returnValue#"; - private static final String OUTPUT_WITH_MATCH_1 = "This; Should be; a match; when; Parsing ; " + RETURN_VALUE; - private static final String OUTPUT_WITHOUT_MATCH = "This; is; not a; match"; - private static final String OUTPUT_WITH_MATCH_2 = "But; thiS will-also; be; a match:; this ; " + RETURN_VALUE; - private static final String SEARCH_WORD_1 = "Parsing"; - private static final String SEARCH_WORD_2 = "this"; - private static final String REGEX_SEARCH_WORD = ".*S.*"; - private List<String> commandOutput; - private List<String> searchWords; - - @Before - public void setup() { - commandOutput = new ArrayList<>(Arrays.asList(OUTPUT_WITH_MATCH_1, OUTPUT_WITHOUT_MATCH, OUTPUT_WITH_MATCH_2)); - } - - @Test - public void parseOutput_searching_for_two_normal_words() { - searchWords = new ArrayList<>(Arrays.asList(SEARCH_WORD_1, SEARCH_WORD_2)); - ParseInstructions parseInstructions = new ParseInstructions(6, 8, " ", searchWords); - List<ParseResult> parseResults = OutputParser.parseOutput(parseInstructions, commandOutput); - ParseResult expectedParseResult1 = new ParseResult(SEARCH_WORD_1, RETURN_VALUE); - ParseResult expectedParseResult2 = new ParseResult(SEARCH_WORD_2, RETURN_VALUE); - assertEquals(expectedParseResult1, parseResults.get(0)); - assertEquals(expectedParseResult2, parseResults.get(1)); - } - - @Test - public void parseOutput_searching_for_two_normal_words_with_semicolon_as_line_split() { - searchWords = new ArrayList<>(Arrays.asList(SEARCH_WORD_1, SEARCH_WORD_2)); - ParseInstructions parseInstructions = new ParseInstructions(4, 5, ";", searchWords); - List<ParseResult> parseResults = OutputParser.parseOutput(parseInstructions, commandOutput); - ParseResult expectedParseResult1 = new ParseResult(SEARCH_WORD_1, RETURN_VALUE); - ParseResult expectedParseResult2 = new ParseResult(SEARCH_WORD_2, RETURN_VALUE); - assertEquals(expectedParseResult1, parseResults.get(0)); - assertEquals(expectedParseResult2, parseResults.get(1)); - } - - @Test - public void parseOutput_searching_for_word_containing_capital_s() { - searchWords = Collections.singletonList(REGEX_SEARCH_WORD); - ParseInstructions parseInstructions = new ParseInstructions(1, 8, " ", searchWords); - List<ParseResult> parseResults = OutputParser.parseOutput(parseInstructions, commandOutput); - ParseResult expectedParseResult1 = new ParseResult("Should", RETURN_VALUE); - ParseResult expectedParseResult2 = new ParseResult("thiS", RETURN_VALUE); - assertEquals(expectedParseResult1, parseResults.get(0)); - assertEquals(expectedParseResult2, parseResults.get(1)); - } - - @Test - public void parseSingleOutput_should_return_first_match() { - searchWords = Collections.singletonList(SEARCH_WORD_1); - ParseInstructions parseInstructions = new ParseInstructions(6, 8, " ", searchWords); - Optional<ParseResult> parseResult = OutputParser.parseSingleOutput(parseInstructions, commandOutput); - ParseResult expectedParseResult = new ParseResult(SEARCH_WORD_1, RETURN_VALUE); - assertEquals(Optional.of(expectedParseResult), parseResult); - } - - @Test - public void parseSingleOutput_should_return_invalid_parseResult() { - searchWords = Collections.singletonList("No match"); - ParseInstructions parseInstructions = new ParseInstructions(6, 8, " ", searchWords); - Optional<ParseResult> parseResult = OutputParser.parseSingleOutput(parseInstructions, commandOutput); - assertEquals(Optional.empty(), parseResult); - } - -} diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/BenchmarkResultInspectorTest.java b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/BenchmarkResultInspectorTest.java deleted file mode 100644 index 0caf63594be..00000000000 --- a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/BenchmarkResultInspectorTest.java +++ /dev/null @@ -1,87 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.verification.hardware; - -import com.yahoo.vespa.hosted.node.verification.commons.report.BenchmarkReport; -import com.yahoo.vespa.hosted.node.verification.hardware.benchmarks.BenchmarkResults; -import org.junit.Before; -import org.junit.Test; - -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; - -/** - * @author sgrostad - * @author olaaun - */ - -public class BenchmarkResultInspectorTest { - - private BenchmarkResults benchmarkResults; - private static final double VALID_CPU_FREQUENCY = 2.031; - private static final double INVALID_CPU_FREQUENCY = 0.1; - private static final double VALID_DISK_SPEED = 1100.0; - private static final double INVALID_DISK_SPEED = 0.1; - private static final double VALID_MEMORY_WRITE_SPEED = 1.7; - private static final double INVALID_MEMORY_WRITE_SPEED = 0.1; - private static final double VALID_MEMORY_READ_SPEED = 4.3; - private static final double INVALID_MEMORY_READ_SPEED = 0.1; - - @Before - public void setup() { - benchmarkResults = new BenchmarkResults(); - benchmarkResults.setCpuCyclesPerSec(VALID_CPU_FREQUENCY); - benchmarkResults.setDiskSpeedMbs(VALID_DISK_SPEED); - benchmarkResults.setMemoryWriteSpeedGBs(VALID_MEMORY_WRITE_SPEED); - benchmarkResults.setMemoryReadSpeedGBs(VALID_MEMORY_READ_SPEED); - } - - @Test - public void isBenchmarkResultsValid_should_return_BenchmarkReport_with_all_values_null() { - BenchmarkReport benchmarkReport = BenchmarkResultInspector.makeBenchmarkReport(benchmarkResults); - assertNull(benchmarkReport.getCpuCyclesPerSec()); - assertNull(benchmarkReport.getDiskSpeedMbs()); - assertNull(benchmarkReport.getMemoryReadSpeedGBs()); - assertNull(benchmarkReport.getMemoryWriteSpeedGBs()); - } - - @Test - public void isBenchmarkResultsValid_should_only_set_cpu_frequency() { - benchmarkResults.setCpuCyclesPerSec(INVALID_CPU_FREQUENCY); - BenchmarkReport benchmarkReport = BenchmarkResultInspector.makeBenchmarkReport(benchmarkResults); - assertNotNull(benchmarkReport.getCpuCyclesPerSec()); - assertNull(benchmarkReport.getDiskSpeedMbs()); - assertNull(benchmarkReport.getMemoryReadSpeedGBs()); - assertNull(benchmarkReport.getMemoryWriteSpeedGBs()); - } - - @Test - public void isBenchmarkResultsValid_should_only_set_disk_speed() { - benchmarkResults.setDiskSpeedMbs(INVALID_DISK_SPEED); - BenchmarkReport benchmarkReport = BenchmarkResultInspector.makeBenchmarkReport(benchmarkResults); - assertNull(benchmarkReport.getCpuCyclesPerSec()); - assertNotNull(benchmarkReport.getDiskSpeedMbs()); - assertNull(benchmarkReport.getMemoryReadSpeedGBs()); - assertNull(benchmarkReport.getMemoryWriteSpeedGBs()); - } - -// @Test TODO: Temporarily disabled due to Meltdown/Spectre performance impact, see VESPA-11051 - public void isBenchmarkResultsValid_should_only_set_memory_read_speed() { - benchmarkResults.setMemoryReadSpeedGBs(INVALID_MEMORY_READ_SPEED); - BenchmarkReport benchmarkReport = BenchmarkResultInspector.makeBenchmarkReport(benchmarkResults); - assertNull(benchmarkReport.getCpuCyclesPerSec()); - assertNull(benchmarkReport.getDiskSpeedMbs()); - assertNotNull(benchmarkReport.getMemoryReadSpeedGBs()); - assertNull(benchmarkReport.getMemoryWriteSpeedGBs()); - } - -// @Test TODO: Temporarily disabled due to Meltdown/Spectre performance impact, see VESPA-11051 - public void isBenchmarkResultsValid_should_only_set_memory_write_speed() { - benchmarkResults.setMemoryWriteSpeedGBs(INVALID_MEMORY_WRITE_SPEED); - BenchmarkReport benchmarkReport = BenchmarkResultInspector.makeBenchmarkReport(benchmarkResults); - assertNull(benchmarkReport.getCpuCyclesPerSec()); - assertNull(benchmarkReport.getDiskSpeedMbs()); - assertNull(benchmarkReport.getMemoryReadSpeedGBs()); - assertNotNull(benchmarkReport.getMemoryWriteSpeedGBs()); - } - -} diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/HardwareBenchmarkerTest.java b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/HardwareBenchmarkerTest.java deleted file mode 100644 index 13e13f1f83f..00000000000 --- a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/HardwareBenchmarkerTest.java +++ /dev/null @@ -1,77 +0,0 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.verification.hardware; - -import com.yahoo.vespa.hosted.node.verification.Main; -import com.yahoo.vespa.hosted.node.verification.mock.MockCommandExecutor; -import org.junit.Test; - -import static org.junit.Assert.assertEquals; - -public class HardwareBenchmarkerTest { - - private static final String RESOURCE_PATH = "src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/resources/"; - private static final String VALID_DISK_BENCHMARK_PATH = RESOURCE_PATH + "diskBenchmarkValidOutput"; - private static final String VALID_SLOW_DISK_BENCHMARK_PATH = RESOURCE_PATH + "diskBenchmarkValidSlowOutput"; - private static final String VALID_CPU_BENCHMARK_PATH = RESOURCE_PATH + "cpuCyclesWithCommasTimeWithDotTest.txt"; - private static final String VALID_MEMORY_WRITE_BENCHMARK_PATH = RESOURCE_PATH + "validMemoryWriteSpeed"; - private static final String VALID_MEMORY_READ_BENCHMARK_PATH = RESOURCE_PATH + "validMemoryReadSpeed"; - private final MockCommandExecutor commandExecutor = new MockCommandExecutor(); - - @Test - public void benchmark_with_no_failures() throws Exception { - commandExecutor.addCommand("cat " + VALID_DISK_BENCHMARK_PATH); - commandExecutor.addCommand("cat " + VALID_CPU_BENCHMARK_PATH); - commandExecutor.addDummyCommand(); - commandExecutor.addDummyCommand(); - commandExecutor.addCommand("cat " + VALID_MEMORY_WRITE_BENCHMARK_PATH); - commandExecutor.addCommand("cat " + VALID_MEMORY_READ_BENCHMARK_PATH); - commandExecutor.addDummyCommand(); - commandExecutor.addDummyCommand(); - - String result = Main.execute(new String[] { - "benchmark", - }, commandExecutor); - - assertEquals("null", result); - } - - @Test - public void disk_benchmark_failure() throws Exception { - commandExecutor.addCommand("cat " + VALID_SLOW_DISK_BENCHMARK_PATH); - commandExecutor.addCommand("cat " + VALID_CPU_BENCHMARK_PATH); - commandExecutor.addDummyCommand(); - commandExecutor.addDummyCommand(); - commandExecutor.addCommand("cat " + VALID_MEMORY_WRITE_BENCHMARK_PATH); - commandExecutor.addCommand("cat " + VALID_MEMORY_READ_BENCHMARK_PATH); - commandExecutor.addDummyCommand(); - commandExecutor.addDummyCommand(); - - String result = Main.execute(new String[]{ - "benchmark", - }, commandExecutor); - - assertEquals("{\"benchmarkReport\":{\"diskSpeedMbs\":49.0}}", result); - } - - - @Test - public void preserve_previous_spec_verifier_result() throws Exception { - commandExecutor.addCommand("cat " + VALID_DISK_BENCHMARK_PATH); - commandExecutor.addCommand("cat " + VALID_CPU_BENCHMARK_PATH); - commandExecutor.addDummyCommand(); - commandExecutor.addDummyCommand(); - commandExecutor.addCommand("cat " + VALID_MEMORY_WRITE_BENCHMARK_PATH); - commandExecutor.addCommand("cat " + VALID_MEMORY_READ_BENCHMARK_PATH); - commandExecutor.addDummyCommand(); - commandExecutor.addDummyCommand(); - - final String previousResult = "{\"specVerificationReport\":{\"actualMemoryAvailable\":4.042128}}"; - - String result = Main.execute(new String[] { - "benchmark", - "-h", previousResult - }, commandExecutor); - - assertEquals(previousResult, result); - } -} diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/CPUBenchmarkTest.java b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/CPUBenchmarkTest.java deleted file mode 100644 index 70fab6eecb1..00000000000 --- a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/CPUBenchmarkTest.java +++ /dev/null @@ -1,118 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.verification.hardware.benchmarks; - - -import com.yahoo.vespa.hosted.node.verification.commons.parser.ParseResult; -import com.yahoo.vespa.hosted.node.verification.mock.MockCommandExecutor; -import org.junit.Before; -import org.junit.Test; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; - -/** - * @author sgrostad - * @author olaaun - */ - -public class CPUBenchmarkTest { - - private static final String cpuEuropeanDelimiters = "src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/resources/cpuCyclesWithDotsTimeWithCommaTest.txt"; - private static final String cpuAlternativeDelimiters = "src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/resources/cpuCyclesWithCommasTimeWithDotTest.txt"; - private static final String cpuWrongOutput = "src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/resources/cpuWrongOutputTest.txt"; - private BenchmarkResults benchmarkResults; - private MockCommandExecutor commandExecutor; - private CPUBenchmark cpu; - private static final double DELTA = 0.1; - - @Before - public void setup() { - commandExecutor = new MockCommandExecutor(); - benchmarkResults = new BenchmarkResults(); - cpu = new CPUBenchmark(benchmarkResults, commandExecutor); - } - - @Test - public void doBenchmark_find_correct_cpuCyclesPerSec() { - String mockCommand = "cat " + cpuAlternativeDelimiters; - commandExecutor.addCommand(mockCommand); - cpu.doBenchmark(); - double result = benchmarkResults.getCpuCyclesPerSec(); - double expected = 2.1576482291815062; - assertEquals(expected, result, DELTA); - } - - @Test - public void doBenchmark_wrong_output_stores_frequency_of_zero() { - String mockCommand = "cat " + cpuWrongOutput; - commandExecutor.addCommand(mockCommand); - cpu.doBenchmark(); - double result = benchmarkResults.getCpuCyclesPerSec(); - double expected = 0; - assertEquals(expected, result, DELTA); - } - - @Test - public void parseCpuCyclesPerSec_return_correct_ArrayList() throws IOException { - List<String> mockCommandOutput = MockCommandExecutor.readFromFile(cpuEuropeanDelimiters); - List<ParseResult> parseResults = cpu.parseCpuCyclesPerSec(mockCommandOutput); - ParseResult expectedParseCyclesResult = new ParseResult("cycles", "2.066.201.729"); - ParseResult expectedParseSecondsResult = new ParseResult("seconds", "0,957617512"); - assertEquals(expectedParseCyclesResult, parseResults.get(0)); - assertEquals(expectedParseSecondsResult, parseResults.get(1)); - } - - @Test - public void test_if_setCpuCyclesPerSec_reads_output_correctly() { - List<ParseResult> parseResults = new ArrayList<>(); - parseResults.add(new ParseResult("cycles", "2.066.201.729")); - parseResults.add(new ParseResult("seconds", "0,957617512")); - cpu.setCpuCyclesPerSec(parseResults); - double expectedCpuCyclesPerSec = 2.1576482291815062; - assertEquals(expectedCpuCyclesPerSec, benchmarkResults.getCpuCyclesPerSec(), DELTA); - } - - @Test - public void test_if_makeCyclesDouble_converts_European_and_alternative_delimiters_correctly() { - String toBeConvertedEuropean = "2.066.201.729"; - String toBEConvertedAlternative = "2,066,201,729"; - double expectedCycles = 2066201729; - assertEquals(expectedCycles, cpu.makeCyclesDouble(toBeConvertedEuropean), DELTA); - assertEquals(expectedCycles, cpu.makeCyclesDouble(toBEConvertedAlternative), DELTA); - } - - @Test - public void test_if_makeSecondsDouble_converts_European_and_alternative_delimiters_correctly() { - String toBeConvertedEuropean = "0,957617512"; - String toBEConvertedAlternative = "0.957617512"; - double expectedSeconds = 0.957617512; - assertEquals(expectedSeconds, cpu.makeSecondsDouble(toBeConvertedEuropean), DELTA); - assertEquals(expectedSeconds, cpu.makeSecondsDouble(toBEConvertedAlternative), DELTA); - } - - @Test - public void test_if_checkIfNumber_returns_true() { - String number = "125.5"; - assertTrue(cpu.checkIfNumber(number)); - } - - @Test - public void test_if_checkIfNumber_returns_false() { - String notANumber = "125.5a"; - assertFalse(cpu.checkIfNumber(notANumber)); - } - - @Test - public void test_if_convertToGHz_converts_correctly() { - double cycles = 2066201729; - double seconds = 0.957617512; - double expectedGHz = 2.1576482291815062; - assertEquals(expectedGHz, cpu.convertToGHz(cycles, seconds), DELTA); - } - -} diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/DiskBenchmarkTest.java b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/DiskBenchmarkTest.java deleted file mode 100644 index f2dbfbdbb6f..00000000000 --- a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/DiskBenchmarkTest.java +++ /dev/null @@ -1,133 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.verification.hardware.benchmarks; - -import com.yahoo.vespa.hosted.node.verification.commons.parser.ParseResult; -import com.yahoo.vespa.hosted.node.verification.mock.MockCommandExecutor; -import org.junit.Before; -import org.junit.Test; - -import java.util.List; -import java.util.Optional; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; - -/** - * @author sgrostad - * @author olaaun - */ - -public class DiskBenchmarkTest { - - private DiskBenchmark diskBenchmark; - private BenchmarkResults benchmarkResults; - private MockCommandExecutor commandExecutor; - private static final String VALID_OUTPUT_FILE = "src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/resources/diskBenchmarkValidOutput"; - private static final String INVALID_OUTPUT_FILE = "src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/resources/diskBenchmarkInvalidOutput"; - private static final double DELTA = 0.1; - - @Before - public void setup() { - commandExecutor = new MockCommandExecutor(); - benchmarkResults = new BenchmarkResults(); - diskBenchmark = new DiskBenchmark(benchmarkResults, commandExecutor); - } - - @Test - public void doBenchmark_should_store_diskSpeed_when_valid_output() { - String mockCommand = "cat " + VALID_OUTPUT_FILE; - commandExecutor.addCommand(mockCommand); - diskBenchmark.doBenchmark(); - double expectedSpeed = 243; - double actualSpeed = benchmarkResults.getDiskSpeedMbs(); - assertEquals(expectedSpeed, actualSpeed, DELTA); - } - - @Test - public void doBenchmark_should_store_diskSpeed_as_zero_when_invalid_output() { - String mockCommand = "cat " + INVALID_OUTPUT_FILE; - commandExecutor.addCommand(mockCommand); - diskBenchmark.doBenchmark(); - double expectedSpeed = 0; - double actualSpeed = benchmarkResults.getDiskSpeedMbs(); - assertEquals(expectedSpeed, actualSpeed, DELTA); - } - - - @Test - public void parseDiskSpeed_valid_input() throws Exception { - List<String> mockCommandOutput = MockCommandExecutor.readFromFile(VALID_OUTPUT_FILE); - Optional<ParseResult> parseResult = diskBenchmark.parseDiskSpeed(mockCommandOutput); - ParseResult expectedParseResult = new ParseResult("MB/s", "243"); - assertEquals(Optional.of(expectedParseResult), parseResult); - } - - @Test - public void parseDiskSpeed_invalid_input() throws Exception { - List<String> mockCommandOutput = MockCommandExecutor.readFromFile(INVALID_OUTPUT_FILE); - Optional<ParseResult> parseResult = diskBenchmark.parseDiskSpeed(mockCommandOutput); - assertEquals(Optional.empty(), parseResult); - } - - @Test - public void setDiskSpeed_valid_input() { - ParseResult parseResult = new ParseResult("MB/s", "243"); - diskBenchmark.setDiskSpeed(Optional.of(parseResult)); - double expectedDiskSpeed = 243; - assertEquals(expectedDiskSpeed, benchmarkResults.getDiskSpeedMbs(), DELTA); - } - - @Test - public void setDiskSpeed_invalid_input() { - diskBenchmark.setDiskSpeed(Optional.empty()); - double expectedDiskSpeed = 0; - assertEquals(expectedDiskSpeed, benchmarkResults.getDiskSpeedMbs(), DELTA); - } - - @Test - public void getDiskSpeedInMBs_for_KBs_MBs_and_GBs() { - ParseResult KBsParseResult = new ParseResult("kB/s", "243000"); - ParseResult MBsParseResult = new ParseResult("MB/s", "243"); - ParseResult GBsParseResult = new ParseResult("GB/s", "0.243"); - double expectedMBs = 243; - assertEquals(expectedMBs, diskBenchmark.getDiskSpeedInMBs(KBsParseResult), DELTA); - assertEquals(expectedMBs, diskBenchmark.getDiskSpeedInMBs(MBsParseResult), DELTA); - assertEquals(expectedMBs, diskBenchmark.getDiskSpeedInMBs(GBsParseResult), DELTA); - } - - @Test - public void ckeckSpeedValidity_should_return_true_for_valid_format() { - String speed = "123"; - assertTrue(diskBenchmark.checkSpeedValidity(speed)); - speed = "30000"; - assertTrue(diskBenchmark.checkSpeedValidity(speed)); - speed = "6"; - assertTrue(diskBenchmark.checkSpeedValidity(speed)); - } - - @Test - public void ckeckSpeedValidity_should_return_false_for_valid_format() { - String speed = "124 GHz"; - assertFalse(diskBenchmark.checkSpeedValidity(speed)); - speed = null; - assertFalse(diskBenchmark.checkSpeedValidity(speed)); - speed = "This should return false as well"; - assertFalse(diskBenchmark.checkSpeedValidity(speed)); - } - - @Test - public void convertToMbs_should_return_properly_converted_disk_speeds() { - String speed = "1234"; - double factor = 1000; - double expectedSpeed = 1234000; - assertEquals(expectedSpeed, diskBenchmark.convertToMBs(speed, factor), DELTA); - factor = 1 / 1000.0; - expectedSpeed = 1.234; - assertEquals(expectedSpeed, diskBenchmark.convertToMBs(speed, factor), DELTA); - factor = 1; - expectedSpeed = 1234; - assertEquals(expectedSpeed, diskBenchmark.convertToMBs(speed, factor), DELTA); - } - -} diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/MemoryBenchmarkTest.java b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/MemoryBenchmarkTest.java deleted file mode 100644 index e2dc19c4f4c..00000000000 --- a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/benchmarks/MemoryBenchmarkTest.java +++ /dev/null @@ -1,73 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.verification.hardware.benchmarks; - -import com.yahoo.vespa.hosted.node.verification.mock.MockCommandExecutor; -import org.junit.Before; -import org.junit.Test; - -import java.util.List; -import java.util.Optional; - -import static org.junit.Assert.assertEquals; - -/** - * @author sgrostad - * @author olaaun - */ -public class MemoryBenchmarkTest { - - private MemoryBenchmark memoryBenchmark; - private BenchmarkResults benchmarkResults; - private MockCommandExecutor commandExecutor; - private static final double DELTA = 0.1; - - @Before - public void setup() { - benchmarkResults = new BenchmarkResults(); - commandExecutor = new MockCommandExecutor(); - memoryBenchmark = new MemoryBenchmark(benchmarkResults, commandExecutor); - } - - @Test - public void doBenchMark_should_update_read_and_write_memory_speed() { - commandExecutor.addDummyCommand(); - commandExecutor.addDummyCommand(); - commandExecutor.addCommand("cat src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/resources/validMemoryWriteSpeed"); - commandExecutor.addCommand("cat src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/resources/validMemoryReadSpeed"); - commandExecutor.addDummyCommand(); - commandExecutor.addDummyCommand(); - memoryBenchmark.doBenchmark(); - double expectedReadSpeed = 5.9; - double expectedWriteSpeed = 3.4; - assertEquals(expectedReadSpeed, benchmarkResults.getMemoryReadSpeedGBs(), DELTA); - assertEquals(expectedWriteSpeed, benchmarkResults.getMemoryWriteSpeedGBs(), DELTA); - } - - @Test - public void parseMemorySpeed_valid_output() { - Double expectedSpeed = 12.1; - String mockOutput = "This is a test \n the memory speed to be found is " + expectedSpeed + " GB/s"; - List<String> mockCommandOutput = commandExecutor.outputFromString(mockOutput); - Optional<Double> parseResult = memoryBenchmark.parseMemorySpeed(mockCommandOutput); - assertEquals(Optional.of(12.1), parseResult); - } - - @Test - public void parseMemorySpeed_slow_but_valid_output() { - Double expectedSpeed = 960D; - String mockOutput = "This is a test \n the memory speed to be found is " + expectedSpeed + " MB/s"; - List<String> mockCommandOutput = commandExecutor.outputFromString(mockOutput); - Optional<Double> parseResult = memoryBenchmark.parseMemorySpeed(mockCommandOutput); - assertEquals(Optional.of(0.96), parseResult); - } - - @Test - public void parseMemorySpeed_invalid_output() { - List<String> mockCommandOutput = commandExecutor.outputFromString(""); - Optional<Double> parseResult = memoryBenchmark.parseMemorySpeed(mockCommandOutput); - assertEquals(Optional.empty(), parseResult); - mockCommandOutput = commandExecutor.outputFromString("Exit status 1"); - parseResult = memoryBenchmark.parseMemorySpeed(mockCommandOutput); - assertEquals(Optional.empty(), parseResult); - } -} diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/resources/cpuCyclesWithCommasTimeWithDotTest.txt b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/resources/cpuCyclesWithCommasTimeWithDotTest.txt deleted file mode 100644 index 08782ef065f..00000000000 --- a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/resources/cpuCyclesWithCommasTimeWithDotTest.txt +++ /dev/null @@ -1,2 +0,0 @@ - 2,066,201,729 cycles - 0.957617512 seconds time elapsed diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/resources/cpuCyclesWithDotsTimeWithCommaTest.txt b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/resources/cpuCyclesWithDotsTimeWithCommaTest.txt deleted file mode 100644 index fd58bfaf8c7..00000000000 --- a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/resources/cpuCyclesWithDotsTimeWithCommaTest.txt +++ /dev/null @@ -1,2 +0,0 @@ - 2.066.201.729 cycles - 0,957617512 seconds time elapsed diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/resources/cpuWrongOutputTest.txt b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/resources/cpuWrongOutputTest.txt deleted file mode 100644 index 61a4c8fb2a2..00000000000 --- a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/resources/cpuWrongOutputTest.txt +++ /dev/null @@ -1,2 +0,0 @@ - Dummy 2.066.201.729 cycles - 0,957617512 seconds time elapsed diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/resources/diskBenchmarkInvalidOutput b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/resources/diskBenchmarkInvalidOutput deleted file mode 100644 index 4b3d2c6edfb..00000000000 --- a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/resources/diskBenchmarkInvalidOutput +++ /dev/null @@ -1,5 +0,0 @@ -dd: bs: illegal numeric value - -real 0m0.074s -user 0m0.002s -sys 0m0.058s
\ No newline at end of file diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/resources/diskBenchmarkValidOutput b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/resources/diskBenchmarkValidOutput deleted file mode 100644 index 26778d9dd9a..00000000000 --- a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/resources/diskBenchmarkValidOutput +++ /dev/null @@ -1,5 +0,0 @@ -243 MB/s - -real 0m1.448s -user 0m0.000s -sys 0m0.260s
\ No newline at end of file diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/resources/diskBenchmarkValidSlowOutput b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/resources/diskBenchmarkValidSlowOutput deleted file mode 100644 index 773850d8640..00000000000 --- a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/resources/diskBenchmarkValidSlowOutput +++ /dev/null @@ -1,5 +0,0 @@ -49 MB/s - -real 0m1.448s -user 0m0.000s -sys 0m0.260s
\ No newline at end of file diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/resources/testReadFile.txt b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/resources/testReadFile.txt deleted file mode 100644 index 28858c30a70..00000000000 --- a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/resources/testReadFile.txt +++ /dev/null @@ -1,2 +0,0 @@ -This test file tests apache commons exec -Second line
\ No newline at end of file diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/resources/validMemoryReadSpeed b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/resources/validMemoryReadSpeed deleted file mode 100644 index 26657cea9e0..00000000000 --- a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/resources/validMemoryReadSpeed +++ /dev/null @@ -1,3 +0,0 @@ -512+0 records in -512+0 records out -536870912 bytes (537 MB) copied, 0.0904486 s, 5.9 GB/s
\ No newline at end of file diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/resources/validMemoryWriteSpeed b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/resources/validMemoryWriteSpeed deleted file mode 100644 index ab4ccc986f8..00000000000 --- a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/hardware/resources/validMemoryWriteSpeed +++ /dev/null @@ -1,3 +0,0 @@ -512+0 records in -512+0 records out -536870912 bytes (537 MB) copied, 0.158612 s, 3.4 GB/s
\ No newline at end of file diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/mock/MockCommandExecutor.java b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/mock/MockCommandExecutor.java deleted file mode 100644 index 88df18e9d4b..00000000000 --- a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/mock/MockCommandExecutor.java +++ /dev/null @@ -1,52 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.verification.mock; - -import com.yahoo.vespa.hosted.node.verification.commons.CommandExecutor; - -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Paths; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; - -/** - * @author sgrostad - * @author olaaun - */ - -public class MockCommandExecutor extends CommandExecutor { - - private ArrayList<String> mockCommands; - private int counter; - private static final String DUMMY_COMMAND = "DUMMY"; - - public MockCommandExecutor() { - mockCommands = new ArrayList<>(); - counter = 0; - } - - @Override - public List<String> executeCommand(String command) throws IOException { - String mockCommand = mockCommands.get(counter++); - if (mockCommand.equals(DUMMY_COMMAND)) return null; - return super.executeCommand(mockCommand); - } - - public void addCommand(String command) { - mockCommands.add(command); - } - - public void addDummyCommand() { - mockCommands.add(DUMMY_COMMAND); - } - - public static List<String> readFromFile(String filepath) throws IOException { - return new ArrayList<>(Arrays.asList(new String(Files.readAllBytes(Paths.get(filepath))).split("\n"))); - } - - public List<String> outputFromString(String output) { - return new ArrayList<>(Arrays.asList(output.split("\n"))); - } - -} diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/HardwareNodeComparatorTest.java b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/HardwareNodeComparatorTest.java deleted file mode 100644 index 22d1a167a13..00000000000 --- a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/HardwareNodeComparatorTest.java +++ /dev/null @@ -1,63 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.verification.spec; - -import com.fasterxml.jackson.databind.ObjectMapper; -import com.yahoo.vespa.hosted.node.verification.spec.retrievers.HardwareInfo; -import com.yahoo.vespa.hosted.node.verification.spec.retrievers.HardwareInfo.DiskType; -import org.junit.Before; -import org.junit.Test; - -import static org.junit.Assert.assertEquals; - -/** - * @author sgrostad - * @author olaaun - */ - -public class HardwareNodeComparatorTest { - - private HardwareInfo actualHardware; - private HardwareInfo nodeInfo; - - @Before - public void setup() { - actualHardware = new HardwareInfo(); - nodeInfo = new HardwareInfo(); - actualHardware.setMinCpuCores(24); - nodeInfo.setMinCpuCores(24); - actualHardware.setMinMainMemoryAvailableGb(16); - nodeInfo.setMinMainMemoryAvailableGb(16); - nodeInfo.setInterfaceSpeedMbs(10000); - actualHardware.setInterfaceSpeedMbs(10000); - actualHardware.setMinDiskAvailableGb(500); - nodeInfo.setMinDiskAvailableGb(500); - } - - @Test - public void compare_equal_hardware_should_create_emmpty_json() throws Exception { - String actualJson = new ObjectMapper().writeValueAsString(HardwareNodeComparator.compare(nodeInfo, actualHardware)); - String expectedJson = "{}"; - assertEquals(expectedJson, actualJson); - - } - - @Test - public void compare_different_amount_of_cores_should_create_json_with_actual_core_amount() throws Exception { - actualHardware.setMinCpuCores(4); - nodeInfo.setMinCpuCores(1); - String actualJson = new ObjectMapper().writeValueAsString(HardwareNodeComparator.compare(nodeInfo, actualHardware)); - String expectedJson = "{\"actualcpuCores\":4}"; - assertEquals(expectedJson, actualJson); - } - - @Test - public void compare_different_disk_type_should_create_json_with_actual_disk_type() throws Exception { - actualHardware.setDiskType(DiskType.SLOW); - nodeInfo.setDiskType(DiskType.FAST); - String actualJson = new ObjectMapper().writeValueAsString(HardwareNodeComparator.compare(nodeInfo, actualHardware)); - String expectedJson = "{\"actualDiskType\":\"SLOW\"}"; - assertEquals(expectedJson, actualJson); - } - - -} diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/SpecVerifierTest.java b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/SpecVerifierTest.java deleted file mode 100644 index 96af900095f..00000000000 --- a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/SpecVerifierTest.java +++ /dev/null @@ -1,102 +0,0 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.verification.spec; - -import com.yahoo.vespa.hosted.node.verification.Main; -import com.yahoo.vespa.hosted.node.verification.mock.MockCommandExecutor; -import org.junit.Test; - -import static org.junit.Assert.assertEquals; - -/** - * @author freva - */ -public class SpecVerifierTest { - - private static final String RESOURCE_PATH = "src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources"; - private static final String CPU_INFO_PATH = RESOURCE_PATH + "/cpuinfoTest"; - private static final String MEMORY_INFO_PATH = RESOURCE_PATH + "/meminfoTest"; - private static final String FAST_DISK_TYPE_INFO_PATH = RESOURCE_PATH + "/DiskTypeFastDisk"; - private static final String DISK_SIZE_INFO_PATH = RESOURCE_PATH + "/filesize"; - private static final String NET_INTERFACE_INFO_PATH = RESOURCE_PATH + "/ifconfig"; - private static final String NET_INTERFACE_SPEED_INFO_PATH = RESOURCE_PATH + "/eth0"; - private static final String PING_RESPONSE = RESOURCE_PATH + "/validpingresponse"; - private final MockCommandExecutor commandExecutor = new MockCommandExecutor(); - - - @Test - public void spec_verification_with_failures() throws Exception { - commandExecutor.addCommand("cat " + CPU_INFO_PATH); - commandExecutor.addCommand("cat " + MEMORY_INFO_PATH); - commandExecutor.addCommand("cat " + FAST_DISK_TYPE_INFO_PATH); - commandExecutor.addCommand("cat " + DISK_SIZE_INFO_PATH); - commandExecutor.addCommand("cat " + NET_INTERFACE_INFO_PATH); - commandExecutor.addCommand("cat " + NET_INTERFACE_SPEED_INFO_PATH); - commandExecutor.addCommand("cat " + PING_RESPONSE); - - String result = Main.execute(new String[] { - "specification", - "-d", "2500", - "-m", "64", - "-c", "1.5", - "-s", "true", - "-b", "10000.0", - "-i", "10.11.12.13,::1234" - }, commandExecutor); - - assertEquals( - "{\"specVerificationReport\":{\"" + - "actualMemoryAvailable\":4.042128,\"" + - "actualDiskSpaceAvailable\":1760.0,\"" + - "actualInterfaceSpeed\":1000.0,\"" + - "actualcpuCores\":4,\"" + - "faultyIpAddresses\":[\"10.11.12.13\",\"0:0:0:0:0:0:0:1234\"]}}", result); - } - - @Test - public void benchmark_with_no_failures() throws Exception { - commandExecutor.addCommand("cat " + CPU_INFO_PATH); - commandExecutor.addCommand("cat " + MEMORY_INFO_PATH); - commandExecutor.addCommand("cat " + FAST_DISK_TYPE_INFO_PATH); - commandExecutor.addCommand("cat " + DISK_SIZE_INFO_PATH); - commandExecutor.addCommand("cat " + NET_INTERFACE_INFO_PATH); - commandExecutor.addCommand("cat " + NET_INTERFACE_SPEED_INFO_PATH); - commandExecutor.addCommand("cat " + PING_RESPONSE); - - String result = Main.execute(new String[] { - "specification", - "-d", "1760", - "-m", "4", - "-c", "4", - "-s", "true", - "-b", "1000" - }, commandExecutor); - - assertEquals("null", result); - } - - @Test - public void preserve_previous_benchmark_result() throws Exception { - commandExecutor.addCommand("cat " + CPU_INFO_PATH); - commandExecutor.addCommand("cat " + MEMORY_INFO_PATH); - commandExecutor.addCommand("cat " + FAST_DISK_TYPE_INFO_PATH); - commandExecutor.addCommand("cat " + DISK_SIZE_INFO_PATH); - commandExecutor.addCommand("cat " + NET_INTERFACE_INFO_PATH); - commandExecutor.addCommand("cat " + NET_INTERFACE_SPEED_INFO_PATH); - commandExecutor.addCommand("cat " + PING_RESPONSE); - - final String previousResult = "{\"specVerificationReport\":{\"actualMemoryAvailable\":4.042128}," + - "\"benchmarkReport\":{\"diskSpeedMbs\":49.0}}"; - - String result = Main.execute(new String[] { - "specification", - "-d", "1760", - "-m", "4", - "-c", "4", - "-s", "true", - "-b", "1000", - "-h", previousResult - }, commandExecutor); - - assertEquals("{\"benchmarkReport\":{\"diskSpeedMbs\":49.0}}", result); - } -} diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/DiskTypeFastDisk b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/DiskTypeFastDisk deleted file mode 100644 index 3b0eed77569..00000000000 --- a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/DiskTypeFastDisk +++ /dev/null @@ -1,2 +0,0 @@ -Name Rota -sda 0
\ No newline at end of file diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/cpuinfoTest b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/cpuinfoTest deleted file mode 100644 index 8e1b0236a6c..00000000000 --- a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/cpuinfoTest +++ /dev/null @@ -1,104 +0,0 @@ -processor : 0 -vendor_id : GenuineIntel -cpu family : 6 -model : 70 -model name : Intel(R) Core(TM) i7-4870HQ CPU @ 2.50GHz -stepping : 1 -cpu MHz : 2493.821 -cache size : 6144 KB -physical id : 0 -siblings : 1 -core id : 0 -cpu cores : 1 -apicid : 0 -initial apicid : 0 -fpu : yes -fpu_exception : yes -cpuid level : 13 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ss ht pbe syscall nx pdpe1gb lm constant_tsc rep_good nopl xtopology nonstop_tsc eagerfpu pni pclmulqdq dtes64 ds_cpl ssse3 sdbg fma cx16 xtpr pcid sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm abm fsgsbase bmi1 avx2 bmi2 erms xsaveopt arat -bugs : -bogomips : 4987.64 -clflush size : 64 -cache_alignment : 64 -address sizes : 39 bits physical, 48 bits virtual -power management: - -processor : 1 -vendor_id : GenuineIntel -cpu family : 6 -model : 70 -model name : Intel(R) Core(TM) i7-4870HQ CPU @ 2.50GHz -stepping : 1 -cpu MHz : 2493.821 -cache size : 6144 KB -physical id : 1 -siblings : 1 -core id : 0 -cpu cores : 1 -apicid : 1 -initial apicid : 1 -fpu : yes -fpu_exception : yes -cpuid level : 13 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ss ht pbe syscall nx pdpe1gb lm constant_tsc rep_good nopl xtopology nonstop_tsc eagerfpu pni pclmulqdq dtes64 ds_cpl ssse3 sdbg fma cx16 xtpr pcid sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm abm fsgsbase bmi1 avx2 bmi2 erms xsaveopt arat -bugs : -bogomips : 5094.26 -clflush size : 64 -cache_alignment : 64 -address sizes : 39 bits physical, 48 bits virtual -power management: - -processor : 2 -vendor_id : GenuineIntel -cpu family : 6 -model : 70 -model name : Intel(R) Core(TM) i7-4870HQ CPU @ 2.50GHz -stepping : 1 -cpu MHz : 2493.821 -cache size : 6144 KB -physical id : 2 -siblings : 1 -core id : 0 -cpu cores : 1 -apicid : 2 -initial apicid : 2 -fpu : yes -fpu_exception : yes -cpuid level : 13 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ss ht pbe syscall nx pdpe1gb lm constant_tsc rep_good nopl xtopology nonstop_tsc eagerfpu pni pclmulqdq dtes64 ds_cpl ssse3 sdbg fma cx16 xtpr pcid sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm abm fsgsbase bmi1 avx2 bmi2 erms xsaveopt arat -bugs : -bogomips : 4390.19 -clflush size : 64 -cache_alignment : 64 -address sizes : 39 bits physical, 48 bits virtual -power management: - -processor : 3 -vendor_id : GenuineIntel -cpu family : 6 -model : 70 -model name : Intel(R) Core(TM) i7-4870HQ CPU @ 2.50GHz -stepping : 1 -cpu MHz : 2493.821 -cache size : 6144 KB -physical id : 3 -siblings : 1 -core id : 0 -cpu cores : 1 -apicid : 3 -initial apicid : 3 -fpu : yes -fpu_exception : yes -cpuid level : 13 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ss ht pbe syscall nx pdpe1gb lm constant_tsc rep_good nopl xtopology nonstop_tsc eagerfpu pni pclmulqdq dtes64 ds_cpl ssse3 sdbg fma cx16 xtpr pcid sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm abm fsgsbase bmi1 avx2 bmi2 erms xsaveopt arat -bugs : -bogomips : 5448.95 -clflush size : 64 -cache_alignment : 64 -address sizes : 39 bits physical, 48 bits virtual -power management: - diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/eth0 b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/eth0 deleted file mode 100755 index 7b2d15a44c1..00000000000 --- a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/eth0 +++ /dev/null @@ -1,15 +0,0 @@ - Supported ports: [ ] - Supported link modes: Not reported - Supported pause frame use: No - Supports auto-negotiation: No - Advertised link modes: Not reported - Advertised pause frame use: No - Advertised auto-negotiation: No - Speed: 1000Mb/s - Duplex: Full - Port: Twisted Pair - PHYAD: 0 - Transceiver: internal - Auto-negotiation: off - MDI-X: Unknown - Link detected: yes
\ No newline at end of file diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/filesize b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/filesize deleted file mode 100644 index 535a509dcc1..00000000000 --- a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/filesize +++ /dev/null @@ -1,2 +0,0 @@ - PV Size 799.65 GB / not usable 0.00 GB - PV Size 960.35 GB / not usable 0.01 GB
\ No newline at end of file diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/ifconfig b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/ifconfig deleted file mode 100755 index 51cd95e1351..00000000000 --- a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/ifconfig +++ /dev/null @@ -1,9 +0,0 @@ -eth0 Link encap:Ethernet HWaddr 02:42:AC:11:00:03 - inet addr:172.17.0.3 Bcast:0.0.0.0 Mask:255.255.0.0 - inet6 addr: fe80::a00:27ff:fe70:e3f5/64 Scope:Link - UP BROADCAST RUNNING MULTICAST MTU:1500 Metric:1 - RX packets:52908 errors:0 dropped:0 overruns:0 frame:0 - TX packets:29527 errors:0 dropped:0 overruns:0 carrier:0 - collisions:0 txqueuelen:0 - RX bytes:75003406 (71.5 MiB) TX bytes:1839045 (1.7 MiB) - diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/ifconfigNoIpv6 b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/ifconfigNoIpv6 deleted file mode 100755 index 773d9576bde..00000000000 --- a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/ifconfigNoIpv6 +++ /dev/null @@ -1,7 +0,0 @@ -eth0 Link encap:Ethernet HWaddr 02:42:AC:11:00:03 - inet addr:172.17.0.3 Bcast:0.0.0.0 Mask:255.255.0.0 - UP BROADCAST RUNNING MULTICAST MTU:1500 Metric:1 - RX packets:52908 errors:0 dropped:0 overruns:0 frame:0 - TX packets:29527 errors:0 dropped:0 overruns:0 carrier:0 - collisions:0 txqueuelen:0 - RX bytes:75003406 (71.5 MiB) TX bytes:1839045 (1.7 MiB)
\ No newline at end of file diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/invalidpingresponse b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/invalidpingresponse deleted file mode 100644 index d111ac34ce1..00000000000 --- a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/invalidpingresponse +++ /dev/null @@ -1 +0,0 @@ -this is no pingresponse ?!!? 34234
\ No newline at end of file diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/meminfoTest b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/meminfoTest deleted file mode 100644 index df2282d9528..00000000000 --- a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/meminfoTest +++ /dev/null @@ -1,45 +0,0 @@ -MemTotal: 4042128 kB -MemFree: 3584108 kB -MemAvailable: 3562648 kB -Buffers: 9568 kB -Cached: 315836 kB -SwapCached: 0 kB -Active: 176620 kB -Inactive: 202120 kB -Active(anon): 113320 kB -Inactive(anon): 105560 kB -Active(file): 63300 kB -Inactive(file): 96560 kB -Unevictable: 0 kB -Mlocked: 0 kB -SwapTotal: 1048572 kB -SwapFree: 1048572 kB -Dirty: 0 kB -Writeback: 0 kB -AnonPages: 53348 kB -Mapped: 49140 kB -Shmem: 165548 kB -Slab: 54504 kB -SReclaimable: 38372 kB -SUnreclaim: 16132 kB -KernelStack: 3552 kB -PageTables: 1368 kB -NFS_Unstable: 0 kB -Bounce: 0 kB -WritebackTmp: 0 kB -CommitLimit: 3069636 kB -Committed_AS: 810504 kB -VmallocTotal: 34359738367 kB -VmallocUsed: 0 kB -VmallocChunk: 0 kB -AnonHugePages: 0 kB -ShmemHugePages: 0 kB -ShmemPmdMapped: 0 kB -HugePages_Total: 0 -HugePages_Free: 0 -HugePages_Rsvd: 0 -HugePages_Surp: 0 -Hugepagesize: 2048 kB -DirectMap4k: 24576 kB -DirectMap2M: 2072576 kB -DirectMap1G: 4194304 kB diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/validpingresponse b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/validpingresponse deleted file mode 100644 index c227083464f..00000000000 --- a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/validpingresponse +++ /dev/null @@ -1 +0,0 @@ -0
\ No newline at end of file diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/CPURetrieverTest.java b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/CPURetrieverTest.java deleted file mode 100644 index 0cf6bb0907b..00000000000 --- a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/CPURetrieverTest.java +++ /dev/null @@ -1,74 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.verification.spec.retrievers; - -import com.yahoo.vespa.hosted.node.verification.commons.parser.ParseResult; -import com.yahoo.vespa.hosted.node.verification.mock.MockCommandExecutor; -import org.junit.Before; -import org.junit.Test; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; - -import static org.junit.Assert.assertEquals; - -/** - * @author sgrostad - * @author olaaun - */ - -public class CPURetrieverTest { - - private static final String FILENAME = "src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/cpuinfoTest"; - private HardwareInfo hardwareInfo; - private MockCommandExecutor commandExecutor; - private CPURetriever cpu; - private static final double DELTA = 0.1; - - @Before - public void setup() { - hardwareInfo = new HardwareInfo(); - commandExecutor = new MockCommandExecutor(); - cpu = new CPURetriever(hardwareInfo, commandExecutor); - } - - @Test - public void updateInfo_should_write_numOfCpuCores_to_hardware_info() { - commandExecutor.addCommand("cat " + FILENAME); - cpu.updateInfo(); - double expectedAmountOfCores = 4; - assertEquals(expectedAmountOfCores, hardwareInfo.getMinCpuCores(), DELTA); - } - - @Test - public void parseCPUInfoFile_should_return_valid_ArrayList() throws IOException { - List<String> commandOutput = MockCommandExecutor.readFromFile(FILENAME); - List<ParseResult> ParseResults = cpu.parseCPUInfoFile(commandOutput); - String expectedSearchWord = "cpu MHz"; - String expectedValue = "2493.821"; - - assertEquals(expectedSearchWord, ParseResults.get(0).getSearchWord()); - assertEquals(expectedValue, ParseResults.get(0).getValue()); - - assertEquals(expectedSearchWord, ParseResults.get(1).getSearchWord()); - assertEquals(expectedValue, ParseResults.get(1).getValue()); - - assertEquals(expectedSearchWord, ParseResults.get(2).getSearchWord()); - assertEquals(expectedValue, ParseResults.get(2).getValue()); - - assertEquals(expectedSearchWord, ParseResults.get(3).getSearchWord()); - assertEquals(expectedValue, ParseResults.get(3).getValue()); - } - - @Test - public void setCpuCores_counts_cores_correctly() { - List<ParseResult> parseResults = new ArrayList<>(); - parseResults.add(new ParseResult("cpu MHz", "2000")); - parseResults.add(new ParseResult("cpu MHz", "2000")); - parseResults.add(new ParseResult("cpu MHz", "2000")); - cpu.setCpuCores(parseResults); - int expectedCpuCores = 3; - assertEquals(expectedCpuCores, hardwareInfo.getMinCpuCores()); - } - -} diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/DiskRetrieverTest.java b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/DiskRetrieverTest.java deleted file mode 100644 index a72cca1c72a..00000000000 --- a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/DiskRetrieverTest.java +++ /dev/null @@ -1,110 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.verification.spec.retrievers; - -import com.yahoo.vespa.hosted.node.verification.commons.parser.ParseResult; -import com.yahoo.vespa.hosted.node.verification.mock.MockCommandExecutor; -import com.yahoo.vespa.hosted.node.verification.spec.retrievers.HardwareInfo.DiskType; -import org.junit.Before; -import org.junit.Test; - -import java.io.IOException; -import java.util.List; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.fail; - -/** - * @author sgrostad - * @author olaaun - */ - -public class DiskRetrieverTest { - - private MockCommandExecutor commandExecutor; - private HardwareInfo hardwareInfo; - private DiskRetriever diskRetriever; - private static String CAT_RESOURCE_PATH = "cat src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/"; - private static final double DELTA = 0.1; - - @Before - public void setup() { - hardwareInfo = new HardwareInfo(); - commandExecutor = new MockCommandExecutor(); - diskRetriever = new DiskRetriever(hardwareInfo, commandExecutor); - } - - @Test - public void updateInfo_should_store_diskType_and_diskSize_in_hardware_info() { - commandExecutor.addCommand(CAT_RESOURCE_PATH + "DiskTypeFastDisk"); - commandExecutor.addCommand(CAT_RESOURCE_PATH + "filesize"); - diskRetriever.updateInfo(); - assertEquals(DiskType.FAST, hardwareInfo.getDiskType()); - double expectedSize = 1760.0; - assertEquals(expectedSize, hardwareInfo.getMinDiskAvailableGb(), DELTA); - } - - @Test - public void updateDiskType__should_store_diskType_in_hardwareInfo() { - commandExecutor.addCommand(CAT_RESOURCE_PATH + "DiskTypeFastDisk"); - diskRetriever.updateDiskType(); - assertEquals(DiskType.FAST, hardwareInfo.getDiskType()); - } - - @Test - public void updateDiskSize__should_store_diskSize_in_hardwareInfo() { - commandExecutor.addCommand(CAT_RESOURCE_PATH + "filesize"); - diskRetriever.updateDiskSize(); - double expectedSize = 1760.0; - assertEquals(expectedSize, hardwareInfo.getMinDiskAvailableGb(), DELTA); - } - - @Test - public void parseDiskType_should_find_fast_disk() throws Exception { - diskRetriever = new DiskRetriever(hardwareInfo, commandExecutor); - List<String> mockOutput = commandExecutor.outputFromString("Name Rota \nsda 0"); - ParseResult parseResult = diskRetriever.parseDiskType(mockOutput); - ParseResult expectedParseResult = new ParseResult("sda", "0"); - assertEquals(expectedParseResult, parseResult); - } - - @Test - public void parseDiskType_should_not_find_fast_disk() throws Exception { - List<String> mockOutput = commandExecutor.outputFromString("Name Rota \nsda 1"); - ParseResult parseResult = diskRetriever.parseDiskType(mockOutput); - ParseResult expectedParseResult = new ParseResult("sda", "1"); - assertEquals(expectedParseResult, parseResult); - } - - @Test - public void parseDiskType_with_invalid_outputstream_does_not_contain_searchword_should_throw_exception() { - List<String> mockOutput = commandExecutor.outputFromString("Name Rota"); - try { - diskRetriever.parseDiskType(mockOutput); - fail("Should have thrown IOException when outputstream doesn't contain search word"); - } catch (IOException e) { - String expectedExceptionMessage = "Parsing for disk type failed"; - assertEquals(expectedExceptionMessage, e.getMessage()); - } - - } - - @Test - public void parseDiskSize_should_find_size_from_file_and_insert_into_parseResult() throws Exception { - String filepath = "src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/filesize"; - List<String> mockOutput = MockCommandExecutor.readFromFile(filepath); - List<ParseResult> parseResults = diskRetriever.parseDiskSize(mockOutput); - ParseResult expectedParseResult1 = new ParseResult("Size", "799.65"); - assertEquals(expectedParseResult1, parseResults.get(0)); - ParseResult expectedParseResult2 = new ParseResult("Size", "960.35"); - assertEquals(expectedParseResult2, parseResults.get(1)); - } - - @Test - public void setDiskType_invalid_ParseResult_should_set_fastDisk_to_invalid() { - ParseResult parseResult = new ParseResult("Invalid", "Invalid"); - diskRetriever.setDiskType(parseResult); - HardwareInfo.DiskType expectedDiskType = HardwareInfo.DiskType.UNKNOWN; - assertEquals(expectedDiskType, hardwareInfo.getDiskType()); - } - -} diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/HardwareInfoRetrieverTest.java b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/HardwareInfoRetrieverTest.java deleted file mode 100644 index 0ebf1a2b26a..00000000000 --- a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/HardwareInfoRetrieverTest.java +++ /dev/null @@ -1,69 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.verification.spec.retrievers; - - -import com.yahoo.vespa.hosted.node.verification.mock.MockCommandExecutor; -import com.yahoo.vespa.hosted.node.verification.spec.VerifierSettings; -import org.junit.Before; -import org.junit.Test; - -import static org.junit.Assert.assertEquals; -import static org.mockito.Mockito.doReturn; -import static org.mockito.Mockito.spy; - -/** - * @author sgrostad - * @author olaaun - */ - -public class HardwareInfoRetrieverTest { - - private static final String RESOURCE_PATH = "src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/"; - private static final String CPU_INFO_PATH = RESOURCE_PATH + "cpuinfoTest"; - private static final String MEMORY_INFO_PATH = RESOURCE_PATH + "meminfoTest"; - private static final String DISK_TYPE_INFO_PATH = RESOURCE_PATH + "DiskTypeFastDisk"; - private static final String DISK_SIZE_INFO_PATH = RESOURCE_PATH + "filesize"; - private static final String NET_INTERFACE_INFO_PATH = RESOURCE_PATH + "ifconfigNoIpv6"; - private static final String NET_INTERFACE_SPEED_INFO_PATH = RESOURCE_PATH + "eth0"; - private static String PING_RESPONSE = RESOURCE_PATH + "invalidpingresponse"; - private MockCommandExecutor mockCommandExecutor; - private HardwareInfo expectedHardwareInfo; - private VerifierSettings verifierSettings = spy(new VerifierSettings()); - private static final double DELTA = 0.1; - - @Before - public void setup() { - mockCommandExecutor = new MockCommandExecutor(); - mockCommandExecutor.addCommand("cat " + CPU_INFO_PATH); - mockCommandExecutor.addCommand("cat " + MEMORY_INFO_PATH); - mockCommandExecutor.addCommand("cat " + DISK_TYPE_INFO_PATH); - mockCommandExecutor.addCommand("cat " + DISK_SIZE_INFO_PATH); - mockCommandExecutor.addCommand("cat " + NET_INTERFACE_INFO_PATH); - mockCommandExecutor.addCommand("cat " + NET_INTERFACE_SPEED_INFO_PATH); - mockCommandExecutor.addCommand("cat " + PING_RESPONSE); - - expectedHardwareInfo = new HardwareInfo(); - expectedHardwareInfo.setMinCpuCores(4); - expectedHardwareInfo.setMinMainMemoryAvailableGb(4.042128); - expectedHardwareInfo.setInterfaceSpeedMbs(1000); - expectedHardwareInfo.setMinDiskAvailableGb(1760.0); - expectedHardwareInfo.setIpv4Interface(true); - expectedHardwareInfo.setIpv6Interface(false); - expectedHardwareInfo.setIpv6Connection(false); - expectedHardwareInfo.setDiskType(HardwareInfo.DiskType.FAST); - } - - @Test - public void retriever_should_return_valid_HardwareInfo() { - doReturn(true).when(verifierSettings).isCheckIPv6(); - HardwareInfo actualHardwareInfo = HardwareInfoRetriever.retrieve(mockCommandExecutor, verifierSettings); - assertEquals(expectedHardwareInfo.getMinDiskAvailableGb(), actualHardwareInfo.getMinDiskAvailableGb(), DELTA); - assertEquals(expectedHardwareInfo.getMinMainMemoryAvailableGb(), actualHardwareInfo.getMinMainMemoryAvailableGb(), DELTA); - assertEquals(expectedHardwareInfo.getMinCpuCores(), actualHardwareInfo.getMinCpuCores()); - assertEquals(expectedHardwareInfo.getIpv4Interface(), actualHardwareInfo.getIpv4Interface()); - assertEquals(expectedHardwareInfo.getIpv6Interface(), actualHardwareInfo.getIpv6Interface()); - assertEquals(expectedHardwareInfo.getInterfaceSpeedMbs(), actualHardwareInfo.getInterfaceSpeedMbs(), DELTA); - assertEquals(expectedHardwareInfo.getDiskType(), actualHardwareInfo.getDiskType()); - assertEquals(expectedHardwareInfo.isIpv6Connection(), actualHardwareInfo.isIpv6Connection()); - } -} diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/MemoryRetrieverTest.java b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/MemoryRetrieverTest.java deleted file mode 100644 index 1722abdc7b8..00000000000 --- a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/MemoryRetrieverTest.java +++ /dev/null @@ -1,65 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.verification.spec.retrievers; - -import com.yahoo.vespa.hosted.node.verification.commons.parser.ParseResult; -import com.yahoo.vespa.hosted.node.verification.mock.MockCommandExecutor; -import org.junit.Before; -import org.junit.Test; - -import java.io.IOException; -import java.util.List; - -import static org.junit.Assert.assertEquals; - -/** - * @author sgrostad - * @author olaaun - */ - -public class MemoryRetrieverTest { - - private static final String FILENAME = "src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/meminfoTest"; - private HardwareInfo hardwareInfo; - private MockCommandExecutor commandExecutor; - private MemoryRetriever memory; - private final double DELTA = 0.1; - - @Before - public void setup() { - hardwareInfo = new HardwareInfo(); - commandExecutor = new MockCommandExecutor(); - memory = new MemoryRetriever(hardwareInfo, commandExecutor); - } - - @Test - public void updateInfo_should_set_memory_available_in_hardwareInfo() { - commandExecutor.addCommand("cat " + FILENAME); - memory.updateInfo(); - double expectedMemory = 4.042128; - assertEquals(expectedMemory, hardwareInfo.getMinMainMemoryAvailableGb(), DELTA); - } - - @Test - public void parseMemInfoFile_should_return_valid_parseResult() throws IOException { - List<String> commandOutput = MockCommandExecutor.readFromFile(FILENAME); - ParseResult parseResult = memory.parseMemInfoFile(commandOutput); - ParseResult expectedParseResult = new ParseResult("MemTotal", "4042128 kB"); - assertEquals(expectedParseResult, parseResult); - } - - @Test - public void updateMemoryInfo_valid_input() { - ParseResult testParseResult = new ParseResult("MemTotal", "4042128"); - memory.updateMemoryInfo(testParseResult); - double expectedMemory = 4.042128; - assertEquals(expectedMemory, hardwareInfo.getMinMainMemoryAvailableGb(), DELTA); - } - - @Test - public void convertToGB_valid_input() { - String testTotMem = "4042128"; - double expectedTotMem = 4.042128; - assertEquals(expectedTotMem, memory.convertKBToGB(testTotMem), DELTA); - } - -} diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/NetRetrieverTest.java b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/NetRetrieverTest.java deleted file mode 100644 index dad73a9bb90..00000000000 --- a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/verification/spec/retrievers/NetRetrieverTest.java +++ /dev/null @@ -1,161 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.verification.spec.retrievers; - -import com.yahoo.vespa.hosted.node.verification.commons.parser.ParseResult; -import com.yahoo.vespa.hosted.node.verification.mock.MockCommandExecutor; -import com.yahoo.vespa.hosted.node.verification.spec.VerifierSettings; -import org.junit.Before; -import org.junit.Test; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; -import static org.mockito.Mockito.doReturn; -import static org.mockito.Mockito.spy; - -/** - * @author sgrostad - * @author olaaun - */ - -public class NetRetrieverTest { - - private static final String RESOURCE_PATH = "src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/"; - private static final String NET_FIND_INTERFACE = RESOURCE_PATH + "ifconfig"; - private static final String NET_CHECK_INTERFACE_SPEED = RESOURCE_PATH + "eth0"; - private static String VALID_PING_RESPONSE = RESOURCE_PATH + "validpingresponse"; - private static String INVALID_PING_RESPONSE = RESOURCE_PATH + "invalidpingresponse"; - private static String PING_SEARCH_WORD = "\\d+\\.?\\d*"; - private HardwareInfo hardwareInfo; - private MockCommandExecutor commandExecutor; - private NetRetriever net; - private List<ParseResult> parseResults; - private VerifierSettings verifierSettings = spy(new VerifierSettings()); - private static final double DELTA = 0.1; - - @Before - public void setup() { - hardwareInfo = new HardwareInfo(); - commandExecutor = new MockCommandExecutor(); - doReturn(true).when(verifierSettings).isCheckIPv6(); - net = new NetRetriever(hardwareInfo, commandExecutor, verifierSettings); - parseResults = new ArrayList<>(); - } - - @Test - public void updateInfo_should_store_ipv4_ipv6_interface_and_interface_speed() { - commandExecutor.addCommand("cat " + NET_FIND_INTERFACE); - commandExecutor.addCommand("cat " + NET_CHECK_INTERFACE_SPEED); - commandExecutor.addCommand("cat " + VALID_PING_RESPONSE); - net.updateInfo(); - assertTrue(hardwareInfo.getIpv4Interface()); - assertTrue(hardwareInfo.getIpv6Interface()); - assertTrue(hardwareInfo.isIpv6Connection()); - double expectedInterfaceSpeed = 1000; - assertEquals(expectedInterfaceSpeed, hardwareInfo.getInterfaceSpeedMbs(), DELTA); - } - - @Test - public void findInterfaceSpeed_valid_input() { - commandExecutor.addCommand("cat " + NET_FIND_INTERFACE); - commandExecutor.addCommand("cat " + NET_CHECK_INTERFACE_SPEED); - parseResults = net.findInterface(); - net.findInterfaceSpeed(parseResults); - ParseResult expectedParseResults = new ParseResult("Speed", "1000Mb/s"); - assertEquals(expectedParseResults, parseResults.get(2)); - } - - @Test - public void parseNetInterface_get_ipv_from_ifconfig_testFile() throws IOException { - List<String> mockOutput = MockCommandExecutor.readFromFile(NET_FIND_INTERFACE); - parseResults = net.parseNetInterface(mockOutput); - net.updateHardwareInfoWithNet(parseResults); - assertTrue(hardwareInfo.getIpv4Interface()); - assertTrue(hardwareInfo.getIpv6Interface()); - } - - @Test - public void parseNetInterface_get_ipv_from_ifconfigNotIpv6_testFile() throws IOException { - List<String> mockOutput = MockCommandExecutor.readFromFile(NET_FIND_INTERFACE + "NoIpv6"); - parseResults = net.parseNetInterface(mockOutput); - List<ParseResult> expextedParseResults = Collections.singletonList(new ParseResult("inet", "inet")); - assertEquals(expextedParseResults, parseResults); - } - - @Test - public void parseInterfaceSpeed_get_interfaceSpeed_from_eth0_testFile() throws IOException { - List<String> mockOutput = MockCommandExecutor.readFromFile("src/test/java/com/yahoo/vespa/hosted/node/verification/spec/resources/eth0"); - ParseResult parseResult = net.parseInterfaceSpeed(mockOutput); - ParseResult expectedParseResult = new ParseResult("Speed", "1000Mb/s"); - assertEquals(expectedParseResult, parseResult); - } - - - @Test - public void updateHardwareinfoWithNet_valid_input() { - parseResults.add(new ParseResult("inet", "inet")); - parseResults.add(new ParseResult("inet6", "inet6")); - parseResults.add(new ParseResult("Speed", "1000Mb/s")); - net.updateHardwareInfoWithNet(parseResults); - double expectedInterfaceSpeed = 1000; - assertEquals(expectedInterfaceSpeed, hardwareInfo.getInterfaceSpeedMbs(), DELTA); - assertTrue(hardwareInfo.getIpv4Interface()); - assertTrue(hardwareInfo.getIpv6Interface()); - } - - @Test - public void stripInterfaceSpeed_should_return_correct_double() { - String interfaceSpeedToConvert = "1000Mb/s"; - double expectedInterfaceSpeed = 1000; - double actualInterfaceSpeed = net.convertInterfaceSpeed(interfaceSpeedToConvert); - assertEquals(expectedInterfaceSpeed, actualInterfaceSpeed, DELTA); - } - - @Test - public void parsePingResponse_valid_ping_response_should_return_ipv6_connectivity() throws IOException { - List<String> mockCommandOutput = MockCommandExecutor.readFromFile(VALID_PING_RESPONSE); - ParseResult parseResult = net.parsePingResponse(mockCommandOutput); - String expectedPing = "0"; - assertEquals(expectedPing, parseResult.getValue()); - } - - @Test - public void parsePingResponse_invalid_ping_response_should_throw_IOException() throws IOException { - List<String> mockCommandOutput = MockCommandExecutor.readFromFile(INVALID_PING_RESPONSE); - try { - net.parsePingResponse(mockCommandOutput); - fail("Expected an IOException to be thrown"); - } catch (IOException e) { - String expectedExceptionMessage = "Failed to parse ping output."; - assertEquals(expectedExceptionMessage, e.getMessage()); - } - } - - @Test - public void setIpv6Connectivity_valid_ping_response_should_return_ipv6_connectivity() { - ParseResult parseResult = new ParseResult(PING_SEARCH_WORD, "0"); - net.setIpv6Connectivity(parseResult); - assertTrue(hardwareInfo.isIpv6Connection()); - } - - @Test - public void setIpv6Connectivity_invalid_ping_response_should_return_no_ipv6_connectivity_1() { - ParseResult parseResult = new ParseResult(PING_SEARCH_WORD, "100"); - net.setIpv6Connectivity(parseResult); - assertFalse(hardwareInfo.isIpv6Connection()); - } - - @Test - public void setIpv6Connectivity_invalid_ping_response_should_return_no_ipv6_connectivity_2() { - ParseResult parseResult = new ParseResult(PING_SEARCH_WORD, "invalid"); - net.setIpv6Connectivity(parseResult); - assertFalse(hardwareInfo.isIpv6Connection()); - } - -} @@ -100,7 +100,6 @@ <module>model-integration</module> <module>node-repository</module> <module>node-admin</module> - <module>node-maintainer</module> <module>orchestrator-restapi</module> <module>orchestrator</module> <module>parent</module> diff --git a/searchlib/src/vespa/searchlib/diskindex/zcposocciterators.cpp b/searchlib/src/vespa/searchlib/diskindex/zcposocciterators.cpp index 7678b11ba41..bea92e5c009 100644 --- a/searchlib/src/vespa/searchlib/diskindex/zcposocciterators.cpp +++ b/searchlib/src/vespa/searchlib/diskindex/zcposocciterators.cpp @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "zcposocciterators.h" +#include "zc4_posting_params.h" namespace search::diskindex { @@ -64,6 +65,42 @@ ZcPosOccIterator(Position start, uint64_t bitLength, uint32_t docIdLimit, bool d _decodeContext = &_decodeContextReal; } +template <bool bigEndian> +std::unique_ptr<search::queryeval::SearchIterator> +create_zc_posocc_iterator(const PostingListCounts &counts, bitcompression::Position start, uint64_t bit_length, const Zc4PostingParams &posting_params, const bitcompression::PosOccFieldsParams &fields_params, const fef::TermFieldMatchDataArray &match_data) +{ + using EC = bitcompression::EncodeContext64<bigEndian>; + bitcompression::DecodeContext64<bigEndian> d(start.getOccurences(), start.getBitOffset()); + UC64_DECODECONTEXT_CONSTRUCTOR(o, d._); + uint32_t length; + uint64_t val64; + UC64_DECODEEXPGOLOMB_NS(o, K_VALUE_ZCPOSTING_NUMDOCS, EC); + uint32_t num_docs = static_cast<uint32_t>(val64) + 1; + assert((num_docs == counts._numDocs) || ((num_docs == posting_params._min_chunk_docs) && (num_docs < counts._numDocs))); + if (num_docs < posting_params._min_skip_docs) { + if (posting_params._dynamic_k) { + return std::make_unique<ZcRareWordPosOccIterator<bigEndian>>(start, bit_length, posting_params._doc_id_limit, posting_params._encode_cheap_features, &fields_params, match_data); + } else { + return std::make_unique<Zc4RareWordPosOccIterator<bigEndian>>(start, bit_length, posting_params._doc_id_limit, posting_params._encode_cheap_features, &fields_params, match_data); + } + } else { + if (posting_params._dynamic_k) { + return std::make_unique<ZcPosOccIterator<bigEndian>>(start, bit_length, posting_params._doc_id_limit, posting_params._encode_cheap_features, posting_params._min_chunk_docs, counts, &fields_params, match_data); + } else { + return std::make_unique<Zc4PosOccIterator<bigEndian>>(start, bit_length, posting_params._doc_id_limit, posting_params._encode_cheap_features, posting_params._min_chunk_docs, counts, &fields_params, match_data); + } + } +} + +std::unique_ptr<search::queryeval::SearchIterator> +create_zc_posocc_iterator(bool bigEndian, const PostingListCounts &counts, bitcompression::Position start, uint64_t bit_length, const Zc4PostingParams &posting_params, const bitcompression::PosOccFieldsParams &fields_params, const fef::TermFieldMatchDataArray &match_data) +{ + if (bigEndian) { + return create_zc_posocc_iterator<true>(counts, start, bit_length, posting_params, fields_params, match_data); + } else { + return create_zc_posocc_iterator<false>(counts, start, bit_length, posting_params, fields_params, match_data); + } +} template class Zc4RareWordPosOccIterator<true>; template class Zc4RareWordPosOccIterator<false>; diff --git a/searchlib/src/vespa/searchlib/diskindex/zcposocciterators.h b/searchlib/src/vespa/searchlib/diskindex/zcposocciterators.h index 3b58203aab4..76e7b384c11 100644 --- a/searchlib/src/vespa/searchlib/diskindex/zcposocciterators.h +++ b/searchlib/src/vespa/searchlib/diskindex/zcposocciterators.h @@ -7,6 +7,8 @@ namespace search::diskindex { +struct Zc4PostingParams; + template <bool bigEndian> class Zc4RareWordPosOccIterator : public Zc4RareWordPostingIterator<bigEndian> { @@ -73,6 +75,9 @@ public: }; +std::unique_ptr<search::queryeval::SearchIterator> +create_zc_posocc_iterator(bool bigEndian, const index::PostingListCounts &counts, bitcompression::Position start, uint64_t bit_length, const Zc4PostingParams &posting_params, const bitcompression::PosOccFieldsParams &fields_params, const fef::TermFieldMatchDataArray &match_data); + extern template class Zc4RareWordPosOccIterator<true>; extern template class Zc4RareWordPosOccIterator<false>; diff --git a/searchlib/src/vespa/searchlib/diskindex/zcposoccrandread.cpp b/searchlib/src/vespa/searchlib/diskindex/zcposoccrandread.cpp index 9d7df382325..edbd78b9b01 100644 --- a/searchlib/src/vespa/searchlib/diskindex/zcposoccrandread.cpp +++ b/searchlib/src/vespa/searchlib/diskindex/zcposoccrandread.cpp @@ -36,15 +36,11 @@ using vespalib::getLastErrorString; ZcPosOccRandRead::ZcPosOccRandRead() : _file(std::make_unique<FastOS_File>()), _fileSize(0), - _minChunkDocs(1 << 30), - _minSkipDocs(64), - _docIdLimit(10000000), + _posting_params(64, 1 << 30, 10000000, true, true, false), _numWords(0), _fileBitSize(0), _headerBitSize(0), - _fieldsParams(), - _dynamicK(true), - _decode_cheap_features(false) + _fieldsParams() { } @@ -65,8 +61,6 @@ createIterator(const PostingListCounts &counts, { (void) usebitVector; - typedef EGPosOccEncodeContext<true> EC; - assert((handle._bitLength != 0) == (counts._bitLength != 0)); assert((counts._numDocs != 0) == (counts._bitLength != 0)); assert(handle._bitOffsetMem <= handle._bitOffset); @@ -85,22 +79,7 @@ createIterator(const PostingListCounts &counts, handle._bitOffsetMem) & 63; Position start(mem, bitOffset); - - EGPosOccDecodeContext<true> d(mem, bitOffset, &_fieldsParams); - - UC64_DECODECONTEXT_CONSTRUCTOR(o, d._); - uint32_t length; - uint64_t val64; - - UC64BE_DECODEEXPGOLOMB_NS(o, K_VALUE_ZCPOSTING_NUMDOCS, EC); - - uint32_t numDocs = static_cast<uint32_t>(val64) + 1; - - if (numDocs < _minSkipDocs) { - return new ZcRareWordPosOccIterator<true>(start, handle._bitLength, _docIdLimit, _decode_cheap_features, &_fieldsParams, matchData); - } else { - return new ZcPosOccIterator<true>(start, handle._bitLength, _docIdLimit, _decode_cheap_features, _minChunkDocs, counts, &_fieldsParams, matchData); - } + return create_zc_posocc_iterator(true, counts, start, handle._bitLength, _posting_params, _fieldsParams, matchData).release(); } @@ -200,10 +179,11 @@ ZcPosOccRandRead::close() } +template <typename DecodeContext> void -ZcPosOccRandRead::readHeader() +ZcPosOccRandRead::readHeader(const vespalib::string &identifier) { - EGPosOccDecodeContext<true> d(&_fieldsParams); + DecodeContext d(&_fieldsParams); ComprFileReadContext drc(d); drc.setFile(_file.get()); @@ -227,14 +207,14 @@ ZcPosOccRandRead::readHeader() assert(header.hasTag("minSkipDocs")); assert(header.getTag("frozen").asInteger() != 0); _fileBitSize = header.getTag("fileBitSize").asInteger(); - assert(header.getTag("format.0").asString() == myId5); + assert(header.getTag("format.0").asString() == identifier); assert(header.getTag("format.1").asString() == d.getIdentifier()); _numWords = header.getTag("numWords").asInteger(); - _minChunkDocs = header.getTag("minChunkDocs").asInteger(); - _docIdLimit = header.getTag("docIdLimit").asInteger(); - _minSkipDocs = header.getTag("minSkipDocs").asInteger(); + _posting_params._min_chunk_docs = header.getTag("minChunkDocs").asInteger(); + _posting_params._doc_id_limit = header.getTag("docIdLimit").asInteger(); + _posting_params._min_skip_docs = header.getTag("minSkipDocs").asInteger(); if (header.hasTag(cheap_features) && (header.getTag(cheap_features).asInteger() != 0)) { - _decode_cheap_features = true; + _posting_params._encode_cheap_features = true; } // Read feature decoding specific subheader d.readHeader(header, "features."); @@ -245,6 +225,11 @@ ZcPosOccRandRead::readHeader() _headerBitSize = d.getReadOffset(); } +void +ZcPosOccRandRead::readHeader() +{ + readHeader<EGPosOccDecodeContext<true>>(myId5); +} const vespalib::string & ZcPosOccRandRead::getIdentifier() @@ -266,95 +251,14 @@ Zc4PosOccRandRead:: Zc4PosOccRandRead() : ZcPosOccRandRead() { - _dynamicK = false; + _posting_params._dynamic_k = false; } -search::queryeval::SearchIterator * -Zc4PosOccRandRead:: -createIterator(const PostingListCounts &counts, - const PostingListHandle &handle, - const search::fef::TermFieldMatchDataArray &matchData, - bool usebitVector) const -{ - (void) usebitVector; - typedef EGPosOccEncodeContext<true> EC; - - assert((handle._bitLength != 0) == (counts._bitLength != 0)); - assert((counts._numDocs != 0) == (counts._bitLength != 0)); - assert(handle._bitOffsetMem <= handle._bitOffset); - - if (handle._bitLength == 0) { - return new search::queryeval::EmptySearch; - } - - const char *cmem = static_cast<const char *>(handle._mem); - uint64_t memOffset = reinterpret_cast<unsigned long>(cmem) & 7; - const uint64_t *mem = reinterpret_cast<const uint64_t *> - (cmem - memOffset) + - (memOffset * 8 + handle._bitOffset - - handle._bitOffsetMem) / 64; - int bitOffset = (memOffset * 8 + handle._bitOffset - - handle._bitOffsetMem) & 63; - - Position start(mem, bitOffset); - EG2PosOccDecodeContext<true> d(mem, bitOffset, &_fieldsParams); - - UC64_DECODECONTEXT_CONSTRUCTOR(o, d._); - uint32_t length; - uint64_t val64; - - UC64BE_DECODEEXPGOLOMB_NS(o, K_VALUE_ZCPOSTING_NUMDOCS, EC); - - uint32_t numDocs = static_cast<uint32_t>(val64) + 1; - - if (numDocs < _minSkipDocs) { - return new Zc4RareWordPosOccIterator<true>(start, handle._bitLength, _docIdLimit, _decode_cheap_features, &_fieldsParams, matchData); - } else { - return new Zc4PosOccIterator<true>(start, handle._bitLength, _docIdLimit, _decode_cheap_features, _minChunkDocs, counts, &_fieldsParams, matchData); - } -} - void Zc4PosOccRandRead::readHeader() { - EG2PosOccDecodeContext<true> d(&_fieldsParams); - ComprFileReadContext drc(d); - - drc.setFile(_file.get()); - drc.setFileSize(_file->GetSize()); - drc.allocComprBuf(512, 32768u); - d.emptyBuffer(0); - drc.readComprBuffer(); - d.setReadContext(&drc); - - vespalib::FileHeader header; - d.readHeader(header, _file->getSize()); - uint32_t headerLen = header.getSize(); - assert(header.hasTag("frozen")); - assert(header.hasTag("fileBitSize")); - assert(header.hasTag("format.0")); - assert(header.hasTag("format.1")); - assert(!header.hasTag("format.2")); - assert(header.hasTag("numWords")); - assert(header.hasTag("minChunkDocs")); - assert(header.hasTag("docIdLimit")); - assert(header.hasTag("minSkipDocs")); - assert(header.getTag("frozen").asInteger() != 0); - _fileBitSize = header.getTag("fileBitSize").asInteger(); - assert(header.getTag("format.0").asString() == myId4); - assert(header.getTag("format.1").asString() == d.getIdentifier()); - _numWords = header.getTag("numWords").asInteger(); - _minChunkDocs = header.getTag("minChunkDocs").asInteger(); - _docIdLimit = header.getTag("docIdLimit").asInteger(); - _minSkipDocs = header.getTag("minSkipDocs").asInteger(); - // Read feature decoding specific subheader - d.readHeader(header, "features."); - // Align on 64-bit unit - d.smallAlign(64); - headerLen += (-headerLen & 7); - assert(d.getReadOffset() == headerLen * 8); - _headerBitSize = d.getReadOffset(); + readHeader<EG2PosOccDecodeContext<true> >(myId4); } const vespalib::string & diff --git a/searchlib/src/vespa/searchlib/diskindex/zcposoccrandread.h b/searchlib/src/vespa/searchlib/diskindex/zcposoccrandread.h index a78ae6f14f3..dcaca5d1f81 100644 --- a/searchlib/src/vespa/searchlib/diskindex/zcposoccrandread.h +++ b/searchlib/src/vespa/searchlib/diskindex/zcposoccrandread.h @@ -6,6 +6,7 @@ #include <vespa/searchlib/bitcompression/compression.h> #include <vespa/searchlib/bitcompression/posocccompression.h> #include <vespa/searchlib/fef/termfieldmatchdataarray.h> +#include "zc4_posting_params.h" namespace search::diskindex { @@ -14,18 +15,11 @@ class ZcPosOccRandRead : public index::PostingListFileRandRead protected: std::unique_ptr<FastOS_FileInterface> _file; uint64_t _fileSize; - - uint32_t _minChunkDocs; // # of documents needed for chunking - uint32_t _minSkipDocs; // # of documents needed for skipping - uint32_t _docIdLimit; // Limit for document ids (docId < docIdLimit) - + Zc4PostingParams _posting_params; uint64_t _numWords; // Number of words in file uint64_t _fileBitSize; uint64_t _headerBitSize; bitcompression::PosOccFieldsParams _fieldsParams; - bool _dynamicK; - bool _decode_cheap_features; - public: ZcPosOccRandRead(); @@ -50,6 +44,8 @@ public: bool open(const vespalib::string &name, const TuneFileRandRead &tuneFileRead) override; bool close() override; + template <typename DecodeContext> + void readHeader(const vespalib::string &identifier); virtual void readHeader(); static const vespalib::string &getIdentifier(); static const vespalib::string &getSubIdentifier(); @@ -57,17 +53,10 @@ public: class Zc4PosOccRandRead : public ZcPosOccRandRead { + using ZcPosOccRandRead::readHeader; public: Zc4PosOccRandRead(); - /** - * Create iterator for single word. Semantic lifetime of counts and - * handle must exceed lifetime of iterator. - */ - queryeval::SearchIterator * - createIterator(const PostingListCounts &counts, const PostingListHandle &handle, - const fef::TermFieldMatchDataArray &matchData, bool usebitVector) const override; - void readHeader() override; static const vespalib::string &getIdentifier(); diff --git a/searchlib/src/vespa/searchlib/test/fakedata/fakezcfilterocc.cpp b/searchlib/src/vespa/searchlib/test/fakedata/fakezcfilterocc.cpp index c79574a61ff..0f914e2e3b1 100644 --- a/searchlib/src/vespa/searchlib/test/fakedata/fakezcfilterocc.cpp +++ b/searchlib/src/vespa/searchlib/test/fakedata/fakezcfilterocc.cpp @@ -1223,6 +1223,7 @@ FakeZcSkipPosOcc<bigEndian>::FakeZcSkipPosOcc(const FakeWord &fw) { setup(fw); _counts._bitLength = _compressedBits; + _counts._numDocs = _hitDocs; } @@ -1284,6 +1285,7 @@ FakeZc4SkipPosOcc<bigEndian>::FakeZc4SkipPosOcc(const FakeWord &fw, const Zc4Pos { setup(fw); _counts._bitLength = _compressedBits; + _counts._numDocs = _hitDocs; } template <bool bigEndian> @@ -1318,26 +1320,7 @@ SearchIterator * FakeZc4SkipPosOcc<bigEndian>:: createIterator(const TermFieldMatchDataArray &matchData) const { - if (_hitDocs >= _posting_params._min_skip_docs) { - if (_posting_params._dynamic_k) { - return new ZcPosOccIterator<bigEndian>(Position(_compressed.first, 0), _compressedBits, _posting_params._doc_id_limit, _posting_params._encode_cheap_features, - static_cast<uint32_t>(-1), - _counts, - &_fieldsParams, - matchData); - } else { - return new Zc4PosOccIterator<bigEndian>(Position(_compressed.first, 0), _compressedBits, _posting_params._doc_id_limit, _posting_params._encode_cheap_features, - static_cast<uint32_t>(-1), _counts, &_fieldsParams, matchData); - } - } else { - if (_posting_params._dynamic_k) { - return new ZcRareWordPosOccIterator<bigEndian>(Position(_compressed.first, 0), - _compressedBits, _posting_params._doc_id_limit, _posting_params._encode_cheap_features, &_fieldsParams, matchData); - } else { - return new Zc4RareWordPosOccIterator<bigEndian>(Position(_compressed.first, 0), - _compressedBits, _posting_params._doc_id_limit, _posting_params._encode_cheap_features, &_fieldsParams, matchData); - } - } + return create_zc_posocc_iterator(bigEndian, _counts, Position(_compressed.first, 0), _compressedBits, _posting_params, _fieldsParams, matchData).release(); } template <bool bigEndian> |