diff options
13 files changed, 40 insertions, 56 deletions
diff --git a/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java b/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java index 183ab56d45f..323bd73186e 100644 --- a/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java +++ b/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java @@ -89,7 +89,7 @@ public class ModelProvisioningTest { " <component id='injected' />" + " </handler>" + " <nodes count='2' preload='lib/blablamalloc.so'>" + - " <jvm allocated-memory='45%' gc-options='-XX:+UseParNewGC' options='-verbosegc' />" + + " <jvm allocated-memory='45%' gc-options='-XX:+UseParNewGC' options='-Xlog:gc' />" + " </nodes>" + "</container>" + "</services>"; @@ -139,8 +139,8 @@ public class ModelProvisioningTest { assertEquals(getDefaults().underVespaHome("lib64/vespa/malloc/libvespamalloc.so"), mydisc.getContainers().get(2).getPreLoad()); assertEquals(Optional.empty(), mydisc.getMemoryPercentage()); - assertEquals("-verbosegc", mydisc2.getContainers().get(0).getJvmOptions()); - assertEquals("-verbosegc", mydisc2.getContainers().get(1).getJvmOptions()); + assertEquals("-Xlog:gc", mydisc2.getContainers().get(0).getJvmOptions()); + assertEquals("-Xlog:gc", mydisc2.getContainers().get(1).getJvmOptions()); assertEquals("lib/blablamalloc.so", mydisc2.getContainers().get(0).getPreLoad()); assertEquals("lib/blablamalloc.so", mydisc2.getContainers().get(1).getPreLoad()); assertEquals(Optional.of(45), mydisc2.getMemoryPercentage()); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/JvmOptionsTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/JvmOptionsTest.java index 8fc37243483..0cff915266f 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/JvmOptionsTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/JvmOptionsTest.java @@ -254,7 +254,7 @@ public class JvmOptionsTest extends ContainerModelBuilderTestBase { // Valid options, should not log anything verifyLoggingOfJvmOptions(true, "options", "-Xms2G"); - verifyLoggingOfJvmOptions(true, "options", "-verbose:gc"); + verifyLoggingOfJvmOptions(true, "options", "-Xlog:gc"); verifyLoggingOfJvmOptions(true, "options", "-Djava.library.path=/opt/vespa/lib64:/home/y/lib64"); verifyLoggingOfJvmOptions(true, "options", "-XX:-OmitStackTraceInFastThrow"); verifyLoggingOfJvmOptions(false, "options", "-Xms2G"); diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/zone/RoutingMethod.java b/config-provisioning/src/main/java/com/yahoo/config/provision/zone/RoutingMethod.java index 0cb7acf8199..c7cc8a3cd10 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/zone/RoutingMethod.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/zone/RoutingMethod.java @@ -8,9 +8,6 @@ package com.yahoo.config.provision.zone; */ public enum RoutingMethod { - /** Routing happens through shared routing layer */ - shared, - /** Routing happens through a dedicated layer 4 load balancer */ exclusive, @@ -24,7 +21,7 @@ public enum RoutingMethod { /** Returns whether this method routes requests through a shared routing layer */ public boolean isShared() { - return this == shared || this == sharedLayer4; + return this == sharedLayer4; } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileServer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileServer.java index ab007cf6bab..328f903d095 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileServer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileServer.java @@ -1,8 +1,8 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.filedistribution; -import com.yahoo.component.annotation.Inject; import com.yahoo.cloud.config.ConfigserverConfig; +import com.yahoo.component.annotation.Inject; import com.yahoo.concurrent.DaemonThreadFactory; import com.yahoo.config.FileReference; import com.yahoo.config.subscription.ConfigSourceSet; @@ -22,12 +22,12 @@ import com.yahoo.vespa.filedistribution.FileReferenceDownload; import com.yahoo.vespa.filedistribution.LazyFileReferenceData; import com.yahoo.vespa.filedistribution.LazyTemporaryStorageFileReferenceData; import com.yahoo.yolean.Exceptions; - import java.io.File; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; import java.time.Duration; +import java.time.Instant; import java.util.List; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -41,6 +41,9 @@ public class FileServer { private static final Logger log = Logger.getLogger(FileServer.class.getName()); + // Set this low, to make sure we don't wait for a long time trying to download file + private static final Duration timeout = Duration.ofSeconds(10); + private final FileDirectory root; private final ExecutorService executor; private final FileDownloader downloader; @@ -157,11 +160,22 @@ public class FileServer { public void serveFile(String fileReference, boolean downloadFromOtherSourceIfNotFound, Request request, Receiver receiver) { if (executor instanceof ThreadPoolExecutor) log.log(Level.FINE, () -> "Active threads: " + ((ThreadPoolExecutor) executor).getActiveCount()); - executor.execute(() -> serveFileInternal(fileReference, downloadFromOtherSourceIfNotFound, request, receiver)); - } - private void serveFileInternal(String fileReference, boolean downloadFromOtherSourceIfNotFound, Request request, Receiver receiver) { log.log(Level.FINE, () -> "Received request for file reference '" + fileReference + "' from " + request.target()); + Instant deadline = Instant.now().plus(timeout); + executor.execute(() -> serveFileInternal(fileReference, downloadFromOtherSourceIfNotFound, request, receiver, deadline)); + } + + private void serveFileInternal(String fileReference, + boolean downloadFromOtherSourceIfNotFound, + Request request, + Receiver receiver, + Instant deadline) { + if (Instant.now().isAfter(deadline)) { + log.log(Level.INFO, () -> "Deadline exceeded for request for file reference '" + fileReference + "' from " + request.target() + + " , giving up"); + return; + } boolean fileExists; try { @@ -215,11 +229,12 @@ public class FileServer { private static FileDownloader createFileDownloader(List<String> configServers) { Supervisor supervisor = new Supervisor(new Transport("filedistribution-pool")).setDropEmptyBuffers(true); + return new FileDownloader(configServers.isEmpty() ? FileDownloader.emptyConnectionPool() : createConnectionPool(configServers, supervisor), supervisor, - Duration.ofSeconds(10)); // set this low, to make sure we don't wait a for a long time in this thread + timeout); } private static ConnectionPool createConnectionPool(List<String> configServers, Supervisor supervisor) { diff --git a/container-disc/src/main/sh/vespa-start-container-daemon.sh b/container-disc/src/main/sh/vespa-start-container-daemon.sh index 4ee98484438..0f017b8bba2 100755 --- a/container-disc/src/main/sh/vespa-start-container-daemon.sh +++ b/container-disc/src/main/sh/vespa-start-container-daemon.sh @@ -200,7 +200,7 @@ configure_numactl() { configure_gcopts() { consider_fallback jvm_gcopts "-XX:MaxTenuringThreshold=15 -XX:NewRatio=1" if [ "$jvm_verbosegc" = "true" ]; then - jvm_gcopts="${jvm_gcopts} -verbose:gc" + jvm_gcopts="${jvm_gcopts} -Xlog:gc" fi } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index e50f81919df..91b76ac8d05 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -2805,7 +2805,6 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { private static String routingMethodString(RoutingMethod method) { switch (method) { case exclusive: return "exclusive"; - case shared: return "shared"; case sharedLayer4: return "sharedLayer4"; } throw new IllegalArgumentException("Unknown routing method " + method); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiHandler.java index 7b66e60b426..082b11af351 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiHandler.java @@ -369,7 +369,6 @@ public class RoutingApiHandler extends AuditLoggingRequestHandler { private static String asString(RoutingMethod method) { switch (method) { - case shared: return "shared"; case exclusive: return "exclusive"; case sharedLayer4: return "sharedLayer4"; default: return "unknown"; diff --git a/routing-generator/src/main/java/com/yahoo/vespa/hosted/routing/RoutingTable.java b/routing-generator/src/main/java/com/yahoo/vespa/hosted/routing/RoutingTable.java index 90a38da8687..90a246ad424 100644 --- a/routing-generator/src/main/java/com/yahoo/vespa/hosted/routing/RoutingTable.java +++ b/routing-generator/src/main/java/com/yahoo/vespa/hosted/routing/RoutingTable.java @@ -10,7 +10,6 @@ import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.zone.RoutingMethod; import com.yahoo.config.provision.zone.ZoneId; - import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Collections; @@ -24,6 +23,8 @@ import java.util.TreeMap; import java.util.stream.Collectors; import java.util.stream.Stream; +import static com.yahoo.cloud.config.LbServicesConfig.Tenants.Applications.Endpoints.RoutingMethod.Enum.sharedLayer4; + /** * A routing table for a hosted Vespa zone. This holds the details necessary for the routing layer to route traffic to * deployments. @@ -127,10 +128,9 @@ public class RoutingTable { } private static RoutingMethod routingMethodFrom(LbServicesConfig.Tenants.Applications.Endpoints endpoint) { - switch (endpoint.routingMethod()) { - case shared: return RoutingMethod.shared; - case sharedLayer4: return RoutingMethod.sharedLayer4; - } + if (endpoint.routingMethod() == sharedLayer4) + return RoutingMethod.sharedLayer4; + throw new IllegalArgumentException("Unhandled routing method: " + endpoint.routingMethod()); } diff --git a/routing-generator/src/test/java/com/yahoo/vespa/hosted/routing/RoutingTableTest.java b/routing-generator/src/test/java/com/yahoo/vespa/hosted/routing/RoutingTableTest.java index dc4df7d45ad..7fd88d85401 100644 --- a/routing-generator/src/test/java/com/yahoo/vespa/hosted/routing/RoutingTableTest.java +++ b/routing-generator/src/test/java/com/yahoo/vespa/hosted/routing/RoutingTableTest.java @@ -49,18 +49,6 @@ public class RoutingTableTest { List.of(new Real("host1-default", 4443, 1, true), new Real("host2-default", 4443, 1, true))), - new Endpoint("rotation-02.vespa.global.routing", RoutingMethod.shared), - Target.create(ApplicationId.from("vespa", "music", "default"), - ClusterSpec.Id.from("default"), ZoneId.from("prod.us-north-1"), - List.of(new Real("host1-default", 4443, 1, true), - new Real("host2-default", 4443, 1, true))), - - new Endpoint("music--vespa.global.vespa.oath.cloud", RoutingMethod.shared), - Target.create(ApplicationId.from("vespa", "music", "default"), - ClusterSpec.Id.from("default"), ZoneId.from("prod.us-north-1"), - List.of(new Real("host1-default", 4443, 1, true), - new Real("host2-default", 4443, 1, true))), - new Endpoint("use-weighted.music.vespa.us-north-1-r.vespa.oath.cloud", RoutingMethod.sharedLayer4), Target.create("use-weighted.music.vespa.us-north-1-r.vespa.oath.cloud", TenantName.from("vespa"), ApplicationName.from("music"), ClusterSpec.Id.from("default"), ZoneId.from("prod.us-north-1"), diff --git a/routing-generator/src/test/resources/lbservices-config b/routing-generator/src/test/resources/lbservices-config index 9c79a9b063f..d19fc5ee4ae 100644 --- a/routing-generator/src/test/resources/lbservices-config +++ b/routing-generator/src/test/resources/lbservices-config @@ -15,32 +15,18 @@ tenants.vespa.applications.music:prod:us-north-1:default.endpoints[1].routingMet tenants.vespa.applications.music:prod:us-north-1:default.endpoints[1].weight 1 tenants.vespa.applications.music:prod:us-north-1:default.endpoints[2].hosts[0] "host1-default" tenants.vespa.applications.music:prod:us-north-1:default.endpoints[2].hosts[1] "host2-default" -tenants.vespa.applications.music:prod:us-north-1:default.endpoints[2].dnsName "music--vespa.global.vespa.oath.cloud" +tenants.vespa.applications.music:prod:us-north-1:default.endpoints[2].dnsName "rotation-02.vespa.global.routing" tenants.vespa.applications.music:prod:us-north-1:default.endpoints[2].clusterId "default" tenants.vespa.applications.music:prod:us-north-1:default.endpoints[2].scope "global" -tenants.vespa.applications.music:prod:us-north-1:default.endpoints[2].routingMethod "shared" +tenants.vespa.applications.music:prod:us-north-1:default.endpoints[2].routingMethod "sharedLayer4" tenants.vespa.applications.music:prod:us-north-1:default.endpoints[2].weight 1 tenants.vespa.applications.music:prod:us-north-1:default.endpoints[3].hosts[0] "host1-default" tenants.vespa.applications.music:prod:us-north-1:default.endpoints[3].hosts[1] "host2-default" -tenants.vespa.applications.music:prod:us-north-1:default.endpoints[3].dnsName "rotation-02.vespa.global.routing" +tenants.vespa.applications.music:prod:us-north-1:default.endpoints[3].dnsName "use-weighted.music.vespa.us-north-1-r.vespa.oath.cloud" tenants.vespa.applications.music:prod:us-north-1:default.endpoints[3].clusterId "default" -tenants.vespa.applications.music:prod:us-north-1:default.endpoints[3].scope "global" +tenants.vespa.applications.music:prod:us-north-1:default.endpoints[3].scope "application" tenants.vespa.applications.music:prod:us-north-1:default.endpoints[3].routingMethod "sharedLayer4" -tenants.vespa.applications.music:prod:us-north-1:default.endpoints[3].weight 1 -tenants.vespa.applications.music:prod:us-north-1:default.endpoints[4].hosts[0] "host1-default" -tenants.vespa.applications.music:prod:us-north-1:default.endpoints[4].hosts[1] "host2-default" -tenants.vespa.applications.music:prod:us-north-1:default.endpoints[4].dnsName "rotation-02.vespa.global.routing" -tenants.vespa.applications.music:prod:us-north-1:default.endpoints[4].clusterId "default" -tenants.vespa.applications.music:prod:us-north-1:default.endpoints[4].scope "global" -tenants.vespa.applications.music:prod:us-north-1:default.endpoints[4].routingMethod "shared" -tenants.vespa.applications.music:prod:us-north-1:default.endpoints[4].weight 1 -tenants.vespa.applications.music:prod:us-north-1:default.endpoints[5].hosts[0] "host1-default" -tenants.vespa.applications.music:prod:us-north-1:default.endpoints[5].hosts[1] "host2-default" -tenants.vespa.applications.music:prod:us-north-1:default.endpoints[5].dnsName "use-weighted.music.vespa.us-north-1-r.vespa.oath.cloud" -tenants.vespa.applications.music:prod:us-north-1:default.endpoints[5].clusterId "default" -tenants.vespa.applications.music:prod:us-north-1:default.endpoints[5].scope "application" -tenants.vespa.applications.music:prod:us-north-1:default.endpoints[5].routingMethod "sharedLayer4" -tenants.vespa.applications.music:prod:us-north-1:default.endpoints[5].weight 0 +tenants.vespa.applications.music:prod:us-north-1:default.endpoints[3].weight 0 tenants.vespa.applications.music:prod:us-north-1:beta.activeRotation true tenants.vespa.applications.music:prod:us-north-1:beta.endpoints[0].hosts[0] "host3-beta" tenants.vespa.applications.music:prod:us-north-1:beta.endpoints[0].hosts[1] "host4-beta" diff --git a/vespabase/src/common-env.sh b/vespabase/src/common-env.sh index 5f2646ff83c..7268a0a7edc 100755 --- a/vespabase/src/common-env.sh +++ b/vespabase/src/common-env.sh @@ -252,7 +252,7 @@ drop_caches () { fi } -no_transparent_hugepages () { +enable_transparent_hugepages_with_background_compaction () { # Should probably also be done on host. if grep -q "release 6" /etc/redhat-release; then dn=/sys/kernel/mm/redhat_transparent_hugepage diff --git a/vespabase/src/start-vespa-base.sh b/vespabase/src/start-vespa-base.sh index d7e9b6f8f1d..345009e0127 100755 --- a/vespabase/src/start-vespa-base.sh +++ b/vespabase/src/start-vespa-base.sh @@ -79,7 +79,7 @@ cd $VESPA_HOME || { echo "Cannot cd to $VESPA_HOME" 1>&2; exit 1; } fixlimits checkjava runvalidation -no_transparent_hugepages +enable_transparent_hugepages_with_background_compaction disable_vm_zone_reclaim_mode drop_caches diff --git a/vespaclient-java/src/main/sh/vespa-destination.sh b/vespaclient-java/src/main/sh/vespa-destination.sh index 607972ac28c..1e13a37f0dd 100755 --- a/vespaclient-java/src/main/sh/vespa-destination.sh +++ b/vespaclient-java/src/main/sh/vespa-destination.sh @@ -78,7 +78,7 @@ export MALLOC_ARENA_MAX=1 #Does not need fast allocation exec java \ -server -enableassertions \ -XX:ThreadStackSize=512 \ --verbose:gc \ +-Xlog:gc \ -Xms4g -Xmx4g \ -Djava.library.path=${VESPA_HOME}/libexec64/native:${VESPA_HOME}/lib64 \ -XX:MaxDirectMemorySize=32m -Djava.awt.headless=true $(getJavaOptionsIPV46) \ |