diff options
12 files changed, 209 insertions, 11 deletions
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java index 5703de4cb18..9ebaa4f0fd6 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java @@ -89,6 +89,7 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat private MbusParams mbusParams; private boolean messageBusEnabled = true; + private int zookeeperSessionTimeoutSeconds = 30; private final int transport_events_before_wakeup; private final int transport_connections_per_target; private final int heapSizePercentageOfTotalNodeMemory; @@ -312,11 +313,11 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat @Override public void getConfig(CuratorConfig.Builder builder) { - if (getParent() instanceof ConfigserverCluster) return; // Produces its own config super.getConfig(builder); + if (getParent() instanceof ConfigserverCluster) return; // Produces its own config - // 12s is 2x the current ZookeeperServerConfig.tickTime() of 6s, and the default minimum the server will accept. - builder.zookeeperSessionTimeoutSeconds(12); // TODO jonmv: make configurable + // Will be bounded by 2x and 20x ZookeeperServerConfig.tickTime(), which is currently 6s. + builder.zookeeperSessionTimeoutSeconds(zookeeperSessionTimeoutSeconds); } public Optional<String> getTlsClientAuthority() { @@ -329,6 +330,10 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat public void setMessageBusEnabled(boolean messageBusEnabled) { this.messageBusEnabled = messageBusEnabled; } + public void setZookeeperSessionTimeoutSeconds(int timeoutSeconds) { + this.zookeeperSessionTimeoutSeconds = timeoutSeconds; + } + protected boolean messageBusEnabled() { return messageBusEnabled; } public void addMbusServer(ComponentId chainId) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java index 6fbe334ba30..0219f8a5275 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java @@ -233,7 +233,9 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { } private void addZooKeeper(ApplicationContainerCluster cluster, Element spec) { - if ( ! hasZooKeeper(spec)) return; + Element zooKeeper = getZooKeeper(spec); + if (zooKeeper == null) return; + Element nodesElement = XML.getChild(spec, "nodes"); boolean isCombined = nodesElement != null && nodesElement.hasAttribute("of"); if (isCombined) { @@ -247,6 +249,17 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { } cluster.addSimpleComponent("com.yahoo.vespa.curator.Curator", null, "zkfacade"); cluster.addSimpleComponent("com.yahoo.vespa.curator.CuratorWrapper", null, "zkfacade"); + String sessionTimeoutSeconds = zooKeeper.getAttribute("session-timeout-seconds"); + if ( ! sessionTimeoutSeconds.isBlank()) { + try { + int timeoutSeconds = Integer.parseInt(sessionTimeoutSeconds); + if (timeoutSeconds <= 0) throw new IllegalArgumentException("must be a positive value"); + cluster.setZookeeperSessionTimeoutSeconds(timeoutSeconds); + } + catch (RuntimeException e) { + throw new IllegalArgumentException("invalid zookeeper session-timeout-seconds '" + sessionTimeoutSeconds + "'", e); + } + } // These need to be setup so that they will use the container's config id, since each container // have different config (id of zookeeper server) @@ -806,7 +819,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { ClusterSpec.Type.container, ClusterSpec.Id.from(cluster.getName()), log, - hasZooKeeper(containerElement)); + getZooKeeper(containerElement) != null); return createNodesFromHosts(hosts, cluster, context.getDeployState()); } @@ -1036,8 +1049,8 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { )); } - private static boolean hasZooKeeper(Element spec) { - return XML.getChild(spec, "zookeeper") != null; + private static Element getZooKeeper(Element spec) { + return XML.getChild(spec, "zookeeper"); } /** Disallow renderers named "XmlRenderer" or "JsonRenderer" */ diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java index e258790cd42..9a2f8a3f0ec 100755 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java @@ -347,7 +347,7 @@ public class ContainerClusterTest { assertEquals(List.of("host-c1", "host-c2"), config.server().stream().map(CuratorConfig.Server::hostname).collect(Collectors.toList())); assertTrue(config.zookeeperLocalhostAffinity()); - assertEquals(12, config.zookeeperSessionTimeoutSeconds()); + assertEquals(30, config.zookeeperSessionTimeoutSeconds()); } @Test diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/configserver/ConfigserverClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/configserver/ConfigserverClusterTest.java index c71f3946937..72b7e7649fd 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/configserver/ConfigserverClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/configserver/ConfigserverClusterTest.java @@ -121,6 +121,7 @@ public class ConfigserverClusterTest { assertEquals(1, config.server().size()); assertEquals("localhost", config.server().get(0).hostname()); assertEquals(2181, config.server().get(0).port()); + assertEquals(120, config.zookeeperSessionTimeoutSeconds()); assertTrue(config.zookeeperLocalhostAffinity()); } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java index 562b1c25b6c..e172e68c0ee 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.container.xml; +import com.yahoo.cloud.config.CuratorConfig; import com.yahoo.cloud.config.ZookeeperServerConfig; import com.yahoo.component.ComponentId; import com.yahoo.config.application.api.ApplicationPackage; @@ -561,7 +562,7 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase { void cluster_with_zookeeper() { Function<Integer, String> servicesXml = (nodeCount) -> "<container version='1.0' id='default'>" + "<nodes count='" + nodeCount + "'/>" + - "<zookeeper/>" + + "<zookeeper session-timeout-seconds='30'/>" + "</container>"; VespaModelTester tester = new VespaModelTester(); tester.addHosts(3); @@ -571,6 +572,7 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase { assertNotNull(cluster); assertComponentConfigured(cluster, "com.yahoo.vespa.curator.Curator"); assertComponentConfigured(cluster, "com.yahoo.vespa.curator.CuratorWrapper"); + assertEquals(30, model.getConfig(CuratorConfig.class, cluster.getConfigId()).zookeeperSessionTimeoutSeconds()); cluster.getContainers().forEach(container -> { assertComponentConfigured(container, "com.yahoo.vespa.zookeeper.ReconfigurableVespaZooKeeperServer"); assertComponentConfigured(container, "com.yahoo.vespa.zookeeper.Reconfigurer"); diff --git a/defaults/src/vespa/.gitignore b/defaults/src/vespa/.gitignore index 75d87529258..cdb1f25b021 100644 --- a/defaults/src/vespa/.gitignore +++ b/defaults/src/vespa/.gitignore @@ -1,3 +1,4 @@ Makefile .depend lib*.so* +config.h diff --git a/defaults/src/vespa/CMakeLists.txt b/defaults/src/vespa/CMakeLists.txt index b442f495c5b..4f1dc1dea6d 100644 --- a/defaults/src/vespa/CMakeLists.txt +++ b/defaults/src/vespa/CMakeLists.txt @@ -5,3 +5,28 @@ vespa_add_library(vespadefaults INSTALL lib64 DEPENDS ) + +function(vespa_configure_config_h) + set(VESPA_USE_ADDRESS_SANITIZER False) + set(VESPA_USE_THREAD_SANITIZER False) + set(VESPA_USE_UNDEFINED_SANITIZER False) + set(VESPA_HAS_IO_URING False) + set(VESPA_HAS_EPOLL False) + if(VESPA_USE_SANITIZER STREQUAL "address") + set(VESPA_USE_ADDRESS_SANITIZER True) + elseif(VESPA_USE_SANITIZER STREQUAL "thread") + set(VESPA_USE_THREAD_SANITIZER True) + elseif(VESPA_USE_SANITIZER STREQUAL "undefined") + set(VESPA_USE_UNDEFINED_SANITIZER True) + endif() + if(NOT APPLE) + set(VESPA_HAS_EPOLL True) + set(VESPA_HAS_IO_URING True) + endif() + set(DO_NOT_EDIT_MESSAGE "This file is generated. Do not edit.") + configure_file(config.h.in config.h @ONLY) +endfunction() + +vespa_configure_config_h() + +install(FILES defaults.h config.h DESTINATION include/vespa) diff --git a/defaults/src/vespa/config.h.in b/defaults/src/vespa/config.h.in new file mode 100644 index 00000000000..40a4d6ce96f --- /dev/null +++ b/defaults/src/vespa/config.h.in @@ -0,0 +1,16 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +/* + * Defines what sanitizers are active. + * Defines if epoll or io_uring is available. + * + * @DO_NOT_EDIT_MESSAGE@ + */ +#cmakedefine VESPA_USE_ADDRESS_SANITIZER +#cmakedefine VESPA_USE_THREAD_SANITIZER +#cmakedefine VESPA_USE_UNDEFINED_SANITIZER +#cmakedefine VESPA_USE_SANITIZER +#cmakedefine VESPA_HAS_EPOLL +#cmakedefine VESPA_HAS_IO_URING diff --git a/maven-plugins/allowed-maven-dependencies.txt b/maven-plugins/allowed-maven-dependencies.txt new file mode 100644 index 00000000000..6a8d1f9b1d6 --- /dev/null +++ b/maven-plugins/allowed-maven-dependencies.txt @@ -0,0 +1,95 @@ +# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +aopalliance:aopalliance:1.0 +com.fasterxml.jackson.core:jackson-annotations:2.13.4 +com.fasterxml.jackson.core:jackson-core:2.13.4 +com.fasterxml.jackson.core:jackson-databind:2.13.4.2 +com.google.errorprone:error_prone_annotations:2.2.0 +com.google.guava:failureaccess:1.0.1 +com.google.guava:guava:27.1-jre +com.google.inject:guice:4.2.3:no_aop +com.google.j2objc:j2objc-annotations:1.1 +commons-io:commons-io:2.5 +commons-io:commons-io:2.6 +commons-io:commons-io:2.7 +javax.annotation:javax.annotation-api:1.2 +javax.annotation:jsr250-api:1.0 +javax.enterprise:cdi-api:1.0 +javax.inject:javax.inject:1 +junit:junit:4.13.2 +net.bytebuddy:byte-buddy:1.11.19 +net.bytebuddy:byte-buddy-agent:1.11.19 +org.apache.commons:commons-compress:1.21 +org.apache.commons:commons-lang3:3.5 +org.apache.commons:commons-lang3:3.8.1 +org.apache.maven:maven-archiver:3.5.2 +org.apache.maven:maven-artifact:3.8.6 +org.apache.maven:maven-builder-support:3.5.0 +org.apache.maven:maven-builder-support:3.8.6 +org.apache.maven:maven-core:3.5.0 +org.apache.maven:maven-core:3.8.6 +org.apache.maven:maven-model:3.8.6 +org.apache.maven:maven-model-builder:3.5.0 +org.apache.maven:maven-model-builder:3.8.6 +org.apache.maven:maven-plugin-api:3.5.0 +org.apache.maven:maven-plugin-api:3.8.5 +org.apache.maven:maven-repository-metadata:3.5.0 +org.apache.maven:maven-repository-metadata:3.8.6 +org.apache.maven:maven-resolver-provider:3.5.0 +org.apache.maven:maven-resolver-provider:3.8.6 +org.apache.maven:maven-settings:3.5.0 +org.apache.maven:maven-settings:3.8.6 +org.apache.maven:maven-settings-builder:3.5.0 +org.apache.maven:maven-settings-builder:3.8.6 +org.apache.maven.enforcer:enforcer-api:3.0.0 +org.apache.maven.plugin-tools:maven-plugin-annotations:3.5 +org.apache.maven.plugin-tools:maven-plugin-annotations:3.6.4 +org.apache.maven.resolver:maven-resolver-api:1.0.3 +org.apache.maven.resolver:maven-resolver-api:1.6.3 +org.apache.maven.resolver:maven-resolver-impl:1.0.3 +org.apache.maven.resolver:maven-resolver-impl:1.6.3 +org.apache.maven.resolver:maven-resolver-spi:1.0.3 +org.apache.maven.resolver:maven-resolver-spi:1.6.3 +org.apache.maven.resolver:maven-resolver-util:1.0.3 +org.apache.maven.resolver:maven-resolver-util:1.6.3 +org.apache.maven.shared:maven-dependency-tree:3.1.1 +org.apache.maven.shared:maven-shared-utils:3.1.0 +org.apache.maven.shared:maven-shared-utils:3.3.3 +org.apache.maven.shared:maven-shared-utils:3.3.4 +org.apiguardian:apiguardian-api:1.1.2 +org.codehaus.plexus:plexus-archiver:4.2.7 +org.codehaus.plexus:plexus-cipher:2.0 +org.codehaus.plexus:plexus-classworlds:2.5.2 +org.codehaus.plexus:plexus-classworlds:2.6.0 +org.codehaus.plexus:plexus-component-annotations:1.7.1 +org.codehaus.plexus:plexus-component-annotations:2.1.0 +org.codehaus.plexus:plexus-interpolation:1.24 +org.codehaus.plexus:plexus-interpolation:1.26 +org.codehaus.plexus:plexus-io:3.2.0 +org.codehaus.plexus:plexus-sec-dispatcher:2.0 +org.codehaus.plexus:plexus-utils:3.0.24 +org.codehaus.plexus:plexus-utils:3.3.0 +org.codehaus.plexus:plexus-utils:3.3.1 +org.eclipse.aether:aether-util:1.1.0 +org.eclipse.sisu:org.eclipse.sisu.inject:0.3.3 +org.eclipse.sisu:org.eclipse.sisu.inject:0.3.5 +org.eclipse.sisu:org.eclipse.sisu.plexus:0.3.3 +org.eclipse.sisu:org.eclipse.sisu.plexus:0.3.5 +org.hamcrest:hamcrest-all:1.3 +org.hamcrest:hamcrest-core:1.3 +org.iq80.snappy:snappy:0.4 +org.junit.jupiter:junit-jupiter:5.8.1 +org.junit.jupiter:junit-jupiter-api:5.8.1 +org.junit.jupiter:junit-jupiter-engine:5.8.1 +org.junit.jupiter:junit-jupiter-params:5.8.1 +org.junit.platform:junit-platform-commons:1.8.1 +org.junit.platform:junit-platform-engine:1.8.1 +org.mockito:mockito-core:4.0.0 +org.objenesis:objenesis:3.2 +org.opentest4j:opentest4j:1.2.0 +org.ow2.asm:asm:9.3 +org.slf4j:slf4j-api:1.7.32 +org.slf4j:slf4j-simple:1.7.22 +org.sonatype.plexus:plexus-cipher:1.4 +org.sonatype.plexus:plexus-sec-dispatcher:1.4 +org.tukaani:xz:1.9 +org.twdata.maven:mojo-executor:2.3.0 diff --git a/maven-plugins/pom.xml b/maven-plugins/pom.xml index 4ad063a39cc..d71e1a0f068 100644 --- a/maven-plugins/pom.xml +++ b/maven-plugins/pom.xml @@ -25,6 +25,44 @@ <module>../vespa-enforcer-extensions</module> </modules> + <build> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-enforcer-plugin</artifactId> + <inherited>false</inherited> + <dependencies> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>vespa-enforcer-extensions</artifactId> + <version>${project.version}</version> + </dependency> + </dependencies> + <executions> + <execution> + <id>default-cli</id> + <goals> + <!-- To allow running 'mvn enforcer:enforce' from the command line --> + <goal>enforce</goal> + </goals> + <configuration> + <rules> + <enforceDependencies implementation="com.yahoo.vespa.maven.plugin.enforcer.EnforceDependenciesAllProjects"> + <specFile>allowed-maven-dependencies.txt</specFile> + <ignored> + <i>com.yahoo.vespa:*:*</i> + <i>ai.vespa:*:*</i> + </ignored> + </enforceDependencies> + </rules> + <fail>true</fail> + </configuration> + </execution> + </executions> + </plugin> + </plugins> + </build> + <properties> <maven.deploy.skip>true</maven.deploy.skip> <maven.javadoc.skip>true</maven.javadoc.skip> diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Nodes.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Nodes.java index 8bf93e838d3..7a3acd18cd6 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Nodes.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Nodes.java @@ -131,7 +131,7 @@ public class Nodes { /** Adds a list of newly created reserved nodes to the node repository */ public List<Node> addReservedNodes(LockedNodeList nodes) { for (Node node : nodes) { - if ( node.flavor().getType() != Flavor.Type.DOCKER_CONTAINER) + if (node.flavor().getType() != Flavor.Type.DOCKER_CONTAINER) illegal("Cannot add " + node + ": This is not a child node"); if (node.allocation().isEmpty()) illegal("Cannot add " + node + ": Child nodes need to be allocated"); diff --git a/vespa-dependencies-enforcer/pom.xml b/vespa-dependencies-enforcer/pom.xml index 1735964ec62..a4f73991a64 100644 --- a/vespa-dependencies-enforcer/pom.xml +++ b/vespa-dependencies-enforcer/pom.xml @@ -41,8 +41,10 @@ <enforceDependencies implementation="com.yahoo.vespa.maven.plugin.enforcer.EnforceDependenciesAllProjects"> <specFile>allowed-maven-dependencies.txt</specFile> <ignored> - <i>com.yahoo.vespa:*:*</i> <i>ai.vespa:*:*</i> + <i>com.yahoo.vespa:*:*</i> + <i>com.yahoo.vespa.bundle-plugin:*:*</i> + <i>com.yahoo.vespa.jdisc_core:*:*</i> </ignored> </enforceDependencies> </rules> |