diff options
10 files changed, 49 insertions, 52 deletions
diff --git a/default_build_settings.cmake b/default_build_settings.cmake index e29e4c32017..a23a8f1489a 100644 --- a/default_build_settings.cmake +++ b/default_build_settings.cmake @@ -80,11 +80,17 @@ function(setup_vespa_default_build_settings_fedora_32) set(DEFAULT_VESPA_LLVM_VERSION "10" PARENT_SCOPE) endfunction() -function(setup_vespa_default_build_settings_ubuntu_18_10) - message("-- Setting up default build settings for ubuntu 18.10") - set(DEFAULT_EXTRA_LINK_DIRECTORY "${VESPA_DEPS}/lib" "/usr/lib/llvm-6.0/lib" PARENT_SCOPE) - set(DEFAULT_EXTRA_INCLUDE_DIRECTORY "${VESPA_DEPS}/include" "/usr/lib/llvm-6.0/include" PARENT_SCOPE) - set(DEFAULT_VESPA_LLVM_VERSION "6.0" PARENT_SCOPE) +function(setup_vespa_default_build_settings_fedora_33) + message("-- Setting up default build settings for fedora 33") + set(DEFAULT_EXTRA_INCLUDE_DIRECTORY "${VESPA_DEPS}/include" "/usr/include/openblas" PARENT_SCOPE) + set(DEFAULT_VESPA_LLVM_VERSION "10" PARENT_SCOPE) +endfunction() + +function(setup_vespa_default_build_settings_ubuntu_19_10) + message("-- Setting up default build settings for ubuntu 19.10") + set(DEFAULT_EXTRA_LINK_DIRECTORY "${VESPA_DEPS}/lib" "/usr/lib/llvm-9/lib" PARENT_SCOPE) + set(DEFAULT_EXTRA_INCLUDE_DIRECTORY "${VESPA_DEPS}/include" "/usr/lib/llvm-9/include" PARENT_SCOPE) + set(DEFAULT_VESPA_LLVM_VERSION "9" PARENT_SCOPE) endfunction() function(vespa_use_default_vespa_unprivileged) @@ -173,8 +179,10 @@ function(vespa_use_default_build_settings) setup_vespa_default_build_settings_fedora_31() elseif(VESPA_OS_DISTRO_COMBINED STREQUAL "fedora 32") setup_vespa_default_build_settings_fedora_32() - elseif(VESPA_OS_DISTRO_COMBINED STREQUAL "ubuntu 18.10") - setup_vespa_default_build_settings_ubuntu_18_10() + elseif(VESPA_OS_DISTRO_COMBINED STREQUAL "fedora 33") + setup_vespa_default_build_settings_fedora_33() + elseif(VESPA_OS_DISTRO_COMBINED STREQUAL "ubuntu 19.10") + setup_vespa_default_build_settings_ubuntu_19_10() else() message(FATAL_ERROR "-- Unkonwn vespa build platform ${VESPA_OS_DISTRO_COMBINED}") endif() diff --git a/dist/vespa.spec b/dist/vespa.spec index c54e4442167..8ce494fd7cb 100644 --- a/dist/vespa.spec +++ b/dist/vespa.spec @@ -60,27 +60,37 @@ BuildRequires: vespa-protobuf-devel >= 3.7.0-4 %if 0%{?fedora} BuildRequires: cmake >= 3.9.1 BuildRequires: maven -BuildRequires: vespa-protobuf-devel >= 3.7.0-4 BuildRequires: openssl-devel %if 0%{?fc29} +BuildRequires: vespa-protobuf-devel >= 3.7.0-4 BuildRequires: llvm-devel >= 7.0.0 BuildRequires: boost-devel >= 1.66 BuildRequires: gtest-devel BuildRequires: gmock-devel %endif %if 0%{?fc30} +BuildRequires: vespa-protobuf-devel >= 3.7.0-4 BuildRequires: llvm-devel >= 8.0.0 BuildRequires: boost-devel >= 1.69 BuildRequires: gtest-devel BuildRequires: gmock-devel %endif %if 0%{?fc31} +BuildRequires: vespa-protobuf-devel >= 3.7.0-4 BuildRequires: llvm-devel >= 9.0.0 BuildRequires: boost-devel >= 1.69 BuildRequires: gtest-devel BuildRequires: gmock-devel %endif %if 0%{?fc32} +BuildRequires: protobuf-devel +BuildRequires: llvm-devel >= 10.0.0 +BuildRequires: boost-devel >= 1.69 +BuildRequires: gtest-devel +BuildRequires: gmock-devel +%endif +%if 0%{?fc33} +BuildRequires: protobuf-devel BuildRequires: llvm-devel >= 10.0.0 BuildRequires: boost-devel >= 1.69 BuildRequires: gtest-devel @@ -159,21 +169,29 @@ Requires: openssl-libs %define _extra_include_directory %{_vespa_deps_prefix}/include;/usr/include/openblas %endif %if 0%{?fedora} -Requires: vespa-protobuf >= 3.7.0-4 Requires: openssl-libs %if 0%{?fc29} +Requires: vespa-protobuf >= 3.7.0-4 Requires: llvm-libs >= 7.0.0 %define _vespa_llvm_version 7 %endif %if 0%{?fc30} +Requires: vespa-protobuf >= 3.7.0-4 Requires: llvm-libs >= 8.0.0 %define _vespa_llvm_version 8 %endif %if 0%{?fc31} +Requires: vespa-protobuf >= 3.7.0-4 Requires: llvm-libs >= 9.0.0 %define _vespa_llvm_version 9 %endif %if 0%{?fc32} +Requires: protobuf +Requires: llvm-libs >= 10.0.0 +%define _vespa_llvm_version 10 +%endif +%if 0%{?fc33} +Requires: protobuf Requires: llvm-libs >= 10.0.0 %define _vespa_llvm_version 10 %endif diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index 32a235f2989..bb4e281a85f 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -126,14 +126,6 @@ public class Flags { "scheduled evenly distributed in the 1x-2x range (and naturally guaranteed at the 2x boundary).", "Takes effect on next run of NodeRebooter"); - public static final UnboundBooleanFlag ENABLE_LARGE_ORCHESTRATOR_LOCKS = defineFeatureFlag( - "enable-large-orchestrator-locks", true, - "If enabled, the orchestrator will accumulate application locks during probe in batch suspension, " + - "and release them in reverse order only after the non-probe is complete. Can be set depending on " + - "parent hostname.", - "Takes immediate effect for new batch suspensions.", - HOSTNAME); - public static final UnboundBooleanFlag RETIRE_WITH_PERMANENTLY_DOWN = defineFeatureFlag( "retire-with-permanently-down", false, "If enabled, retirement will end with setting the host status to PERMANENTLY_DOWN, " + diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorContext.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorContext.java index eb6a4119f8a..af59eb11e9b 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorContext.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorContext.java @@ -38,9 +38,11 @@ public class OrchestratorContext implements AutoCloseable { private final HashMap<ApplicationInstanceReference, Runnable> locks = new HashMap<>(); /** Create an OrchestratorContext for operations on multiple applications. */ - public static OrchestratorContext createContextForMultiAppOp(Clock clock, boolean largeLocks) { + public static OrchestratorContext createContextForMultiAppOp(Clock clock) { return new OrchestratorContext(null, clock, TimeBudget.fromNow(clock, DEFAULT_TIMEOUT_FOR_BATCH_OP), - false, largeLocks, false); + false, // probe + true, // large locks + false); // use permanently down status } /** Create an OrchestratorContext for an operation on a single application. */ diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorImpl.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorImpl.java index fbe6864274c..f860370faf9 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorImpl.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorImpl.java @@ -64,7 +64,6 @@ public class OrchestratorImpl implements Orchestrator { private final ClusterControllerClientFactory clusterControllerClientFactory; private final Clock clock; private final ApplicationApiFactory applicationApiFactory; - private final BooleanFlag enableLargeOrchestratorLocks; private final BooleanFlag retireWithPermanentlyDownFlag; @Inject @@ -101,7 +100,6 @@ public class OrchestratorImpl implements Orchestrator { this.instanceLookupService = instanceLookupService; this.clock = clock; this.applicationApiFactory = applicationApiFactory; - this.enableLargeOrchestratorLocks = Flags.ENABLE_LARGE_ORCHESTRATOR_LOCKS.bindTo(flagSource); this.retireWithPermanentlyDownFlag = Flags.RETIRE_WITH_PERMANENTLY_DOWN.bindTo(flagSource); } @@ -258,10 +256,7 @@ public class OrchestratorImpl implements Orchestrator { @Override public void suspendAll(HostName parentHostname, List<HostName> hostNames) throws BatchHostStateChangeDeniedException, BatchHostNameNotFoundException, BatchInternalErrorException { - boolean largeLocks = enableLargeOrchestratorLocks - .with(FetchVector.Dimension.HOSTNAME, parentHostname.s()) - .value(); - try (OrchestratorContext context = OrchestratorContext.createContextForMultiAppOp(clock, largeLocks)) { + try (OrchestratorContext context = OrchestratorContext.createContextForMultiAppOp(clock)) { List<NodeGroup> nodeGroupsOrderedByApplication; try { nodeGroupsOrderedByApplication = nodeGroupsOrderedForSuspend(hostNames); diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/HostResource.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/HostResource.java index fc5c5eb5004..c93bb9e1d4a 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/HostResource.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/HostResource.java @@ -157,8 +157,9 @@ public class HostResource implements HostApi { private static WebApplicationException webExceptionFromTimeout(String operationDescription, HostName hostName, UncheckedTimeoutException e) { - return createWebException(operationDescription, hostName, e, HostedVespaPolicy.DEADLINE_CONSTRAINT, e.getMessage(), - Response.Status.GATEWAY_TIMEOUT); + // Return timeouts as 409 Conflict instead of 504 Gateway Timeout to reduce noise in 5xx graphs. + return createWebException(operationDescription, hostName, e, + HostedVespaPolicy.DEADLINE_CONSTRAINT, e.getMessage(), Response.Status.CONFLICT); } private static WebApplicationException webExceptionWithDenialReason( diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/HostSuspensionResource.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/HostSuspensionResource.java index 79e0fc0f3e9..6e857563f9b 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/HostSuspensionResource.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/HostSuspensionResource.java @@ -18,7 +18,6 @@ import javax.ws.rs.WebApplicationException; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; import java.util.List; -import java.util.concurrent.TimeoutException; import java.util.logging.Logger; import java.util.stream.Collectors; @@ -48,7 +47,7 @@ public class HostSuspensionResource implements HostSuspensionApi { throw createWebApplicationException(e.getMessage(), Response.Status.CONFLICT); } catch (UncheckedTimeoutException e) { log.log(LogLevel.DEBUG, "Failed to suspend nodes " + hostnames + " with parent host " + parentHostname, e); - throw createWebApplicationException(e.getMessage(), Response.Status.GATEWAY_TIMEOUT); + throw createWebApplicationException(e.getMessage(), Response.Status.CONFLICT); } catch (BatchHostNameNotFoundException e) { log.log(LogLevel.DEBUG, "Failed to suspend nodes " + hostnames + " with parent host " + parentHostname, e); // Note that we're returning BAD_REQUEST instead of NOT_FOUND because the resource identified diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorContextTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorContextTest.java index 607894ee104..16b66b2804e 100644 --- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorContextTest.java +++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorContextTest.java @@ -23,7 +23,7 @@ public class OrchestratorContextTest { var mutable = new Object() { boolean locked = true; }; Runnable unlock = () -> mutable.locked = false; - try (OrchestratorContext rootContext = OrchestratorContext.createContextForMultiAppOp(new ManualClock(), true)) { + try (OrchestratorContext rootContext = OrchestratorContext.createContextForMultiAppOp(new ManualClock())) { try (OrchestratorContext probeContext = rootContext.createSubcontextForSingleAppOp(true)) { assertFalse(probeContext.hasLock(application)); assertTrue(probeContext.registerLockAcquisition(application, unlock)); @@ -41,19 +41,4 @@ public class OrchestratorContextTest { } assertFalse(mutable.locked); } - - @Test - public void testLargeLocksDisabled() { - var mutable = new Object() { boolean locked = true; }; - Runnable unlock = () -> mutable.locked = false; - - try (OrchestratorContext rootContext = OrchestratorContext.createContextForMultiAppOp(new ManualClock(), false)) { - try (OrchestratorContext probeContext = rootContext.createSubcontextForSingleAppOp(true)) { - assertFalse(probeContext.hasLock(application)); - assertFalse(probeContext.registerLockAcquisition(application, unlock)); - } - } - - assertTrue(mutable.locked); - } }
\ No newline at end of file diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorImplTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorImplTest.java index 77ec824da54..bfe4b523e4a 100644 --- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorImplTest.java +++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorImplTest.java @@ -17,7 +17,6 @@ import com.yahoo.vespa.applicationmodel.ServiceStatus; import com.yahoo.vespa.applicationmodel.ServiceType; import com.yahoo.vespa.applicationmodel.TenantId; import com.yahoo.vespa.curator.mock.MockCurator; -import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.flags.InMemoryFlagSource; import com.yahoo.vespa.orchestrator.controller.ClusterControllerClientFactory; import com.yahoo.vespa.orchestrator.controller.ClusterControllerClientFactoryMock; @@ -325,8 +324,6 @@ public class OrchestratorImplTest { @Test public void testLargeLocks() throws Exception { - flagSource.withBooleanFlag(Flags.ENABLE_LARGE_ORCHESTRATOR_LOCKS.id(), true); - var tenantId = new TenantId("tenant"); var applicationInstanceId = new ApplicationInstanceId("app:dev:us-east-1:default"); var applicationInstanceReference = new ApplicationInstanceReference(tenantId, applicationInstanceId); diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/HostResourceTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/HostResourceTest.java index dc26c1a3770..ff7413cd3bb 100644 --- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/HostResourceTest.java +++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/HostResourceTest.java @@ -368,7 +368,7 @@ public class HostResourceTest { } @Test - public void throws_504_on_timeout() throws HostNameNotFoundException, HostStateChangeDeniedException { + public void throws_409_on_timeout() throws HostNameNotFoundException, HostStateChangeDeniedException { Orchestrator orchestrator = mock(Orchestrator.class); doThrow(new UncheckedTimeoutException("Timeout Message")).when(orchestrator).resume(any(HostName.class)); @@ -377,13 +377,13 @@ public class HostResourceTest { hostResource.resume("hostname"); fail(); } catch (WebApplicationException w) { - assertThat(w.getResponse().getStatus()).isEqualTo(504); + assertThat(w.getResponse().getStatus()).isEqualTo(409); assertEquals("resume failed: Timeout Message [deadline]", w.getMessage()); } } @Test - public void throws_504_on_suspendAll_timeout() throws BatchHostStateChangeDeniedException, BatchHostNameNotFoundException, BatchInternalErrorException { + public void throws_409_on_suspendAll_timeout() throws BatchHostStateChangeDeniedException, BatchHostNameNotFoundException, BatchInternalErrorException { Orchestrator orchestrator = mock(Orchestrator.class); doThrow(new UncheckedTimeoutException("Timeout Message")).when(orchestrator).suspendAll(any(), any()); @@ -392,7 +392,7 @@ public class HostResourceTest { resource.suspendAll("parenthost", Arrays.asList("h1", "h2", "h3")); fail(); } catch (WebApplicationException w) { - assertThat(w.getResponse().getStatus()).isEqualTo(504); + assertThat(w.getResponse().getStatus()).isEqualTo(409); } } } |