diff options
17 files changed, 152 insertions, 130 deletions
diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java index 0cf8b343b6a..638a66db03f 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java @@ -89,7 +89,7 @@ public interface ModelContext { @ModelFeatureFlag(owners = {"baldersheim"}) default int maxMergeQueueSize() { return 100; } @ModelFeatureFlag(owners = {"baldersheim"}) default boolean containerDumpHeapOnShutdownTimeout() { throw new UnsupportedOperationException("TODO specify default value"); } @ModelFeatureFlag(owners = {"baldersheim"}) default double containerShutdownTimeout() { throw new UnsupportedOperationException("TODO specify default value"); } - @ModelFeatureFlag(owners = {"geirst"}) default boolean enableFeedBlockInDistributor() { return true; } + @ModelFeatureFlag(owners = {"geirst"}, removeAfter = "7.541") default boolean enableFeedBlockInDistributor() { return true; } @ModelFeatureFlag(owners = {"bjorncs", "tokle"}) default List<String> allowedAthenzProxyIdentities() { return List.of(); } @ModelFeatureFlag(owners = {"vekterli"}) default int maxActivationInhibitedOutOfSyncGroups() { return 0; } @ModelFeatureFlag(owners = {"hmusum"}) default String jvmOmitStackTraceInFastThrowOption(ClusterSpec.Type type) { return ""; } diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java index f76cb6e43cc..c9ddcdf38eb 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java @@ -51,7 +51,6 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea private Quota quota = Quota.unlimited(); private boolean useAsyncMessageHandlingOnSchedule = false; private double feedConcurrency = 0.5; - private boolean enableFeedBlockInDistributor = true; private int maxActivationInhibitedOutOfSyncGroups = 0; private List<TenantSecretStore> tenantSecretStores = Collections.emptyList(); private String jvmOmitStackTraceInFastThrowOption; @@ -106,7 +105,6 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea @Override public Quota quota() { return quota; } @Override public boolean useAsyncMessageHandlingOnSchedule() { return useAsyncMessageHandlingOnSchedule; } @Override public double feedConcurrency() { return feedConcurrency; } - @Override public boolean enableFeedBlockInDistributor() { return enableFeedBlockInDistributor; } @Override public int maxActivationInhibitedOutOfSyncGroups() { return maxActivationInhibitedOutOfSyncGroups; } @Override public List<TenantSecretStore> tenantSecretStores() { return tenantSecretStores; } @Override public String jvmOmitStackTraceInFastThrowOption(ClusterSpec.Type type) { return jvmOmitStackTraceInFastThrowOption; } @@ -252,11 +250,6 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea return this; } - public TestProperties enableFeedBlockInDistributor(boolean enabled) { - enableFeedBlockInDistributor = enabled; - return this; - } - public TestProperties maxActivationInhibitedOutOfSyncGroups(int nGroups) { maxActivationInhibitedOutOfSyncGroups = nGroups; return this; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/ClusterControllerConfig.java b/config-model/src/main/java/com/yahoo/vespa/model/content/ClusterControllerConfig.java index 25e00dc05dd..f0b32f9140d 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/ClusterControllerConfig.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/ClusterControllerConfig.java @@ -41,7 +41,6 @@ public class ClusterControllerConfig extends AbstractConfigProducer<ClusterContr minNodeRatioPerGroup = clusterTuning.childAsDouble("min-node-ratio-per-group"); bucketSplittingMinimumBits = clusterTuning.childAsInteger("bucket-splitting.minimum-bits"); } - boolean enableClusterFeedBlock = deployState.getProperties().featureFlags().enableFeedBlockInDistributor(); if (tuning != null) { return new ClusterControllerConfig(ancestor, clusterName, @@ -53,14 +52,13 @@ public class ClusterControllerConfig extends AbstractConfigProducer<ClusterContr tuning.childAsDouble("min-storage-up-ratio"), bucketSplittingMinimumBits, minNodeRatioPerGroup, - enableClusterFeedBlock, resourceLimits); } else { return new ClusterControllerConfig(ancestor, clusterName, null, null, null, null, null, null, bucketSplittingMinimumBits, minNodeRatioPerGroup, - enableClusterFeedBlock, resourceLimits); + resourceLimits); } } } @@ -74,7 +72,6 @@ public class ClusterControllerConfig extends AbstractConfigProducer<ClusterContr private final Double minStorageUpRatio; private final Integer minSplitBits; private final Double minNodeRatioPerGroup; - private final boolean enableClusterFeedBlock; private final ResourceLimits resourceLimits; // TODO refactor; too many args @@ -88,7 +85,6 @@ public class ClusterControllerConfig extends AbstractConfigProducer<ClusterContr Double minStorageUpRatio, Integer minSplitBits, Double minNodeRatioPerGroup, - boolean enableClusterFeedBlock, ResourceLimits resourceLimits) { super(parent, "fleetcontroller"); @@ -101,7 +97,6 @@ public class ClusterControllerConfig extends AbstractConfigProducer<ClusterContr this.minStorageUpRatio = minStorageUpRatio; this.minSplitBits = minSplitBits; this.minNodeRatioPerGroup = minNodeRatioPerGroup; - this.enableClusterFeedBlock = enableClusterFeedBlock; this.resourceLimits = resourceLimits; } @@ -144,7 +139,6 @@ public class ClusterControllerConfig extends AbstractConfigProducer<ClusterContr if (minNodeRatioPerGroup != null) { builder.min_node_ratio_per_group(minNodeRatioPerGroup); } - builder.enable_cluster_feed_block(enableClusterFeedBlock); resourceLimits.getConfig(builder); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/ClusterResourceLimits.java b/config-model/src/main/java/com/yahoo/vespa/model/content/ClusterResourceLimits.java index 0a08f81ffca..80dd17213f3 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/ClusterResourceLimits.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/ClusterResourceLimits.java @@ -34,7 +34,6 @@ public class ClusterResourceLimits { public static class Builder { - private final boolean enableFeedBlockInDistributor; private final boolean hostedVespa; private final double resourceLimitDisk; private final double resourceLimitMemory; @@ -42,11 +41,9 @@ public class ClusterResourceLimits { private ResourceLimits.Builder ctrlBuilder = new ResourceLimits.Builder(); private ResourceLimits.Builder nodeBuilder = new ResourceLimits.Builder(); - public Builder(boolean enableFeedBlockInDistributor, - boolean hostedVespa, + public Builder(boolean hostedVespa, double resourceLimitDisk, double resourceLimitMemory) { - this.enableFeedBlockInDistributor = enableFeedBlockInDistributor; this.hostedVespa = hostedVespa; this.resourceLimitDisk = resourceLimitDisk; this.resourceLimitMemory = resourceLimitMemory; @@ -81,17 +78,15 @@ public class ClusterResourceLimits { } private void deriveLimits() { - if (enableFeedBlockInDistributor) { - // This also ensures that content nodes limits are derived according to the formula in calcContentNodeLimit(). - considerSettingDefaultClusterControllerLimit(ctrlBuilder.getDiskLimit(), - nodeBuilder.getDiskLimit(), - ctrlBuilder::setDiskLimit, - resourceLimitDisk); - considerSettingDefaultClusterControllerLimit(ctrlBuilder.getMemoryLimit(), - nodeBuilder.getMemoryLimit(), - ctrlBuilder::setMemoryLimit, - resourceLimitMemory); - } + // This also ensures that content nodes limits are derived according to the formula in calcContentNodeLimit(). + considerSettingDefaultClusterControllerLimit(ctrlBuilder.getDiskLimit(), + nodeBuilder.getDiskLimit(), + ctrlBuilder::setDiskLimit, + resourceLimitDisk); + considerSettingDefaultClusterControllerLimit(ctrlBuilder.getMemoryLimit(), + nodeBuilder.getMemoryLimit(), + ctrlBuilder::setMemoryLimit, + resourceLimitMemory); deriveClusterControllerLimit(ctrlBuilder.getDiskLimit(), nodeBuilder.getDiskLimit(), ctrlBuilder::setDiskLimit); deriveClusterControllerLimit(ctrlBuilder.getMemoryLimit(), nodeBuilder.getMemoryLimit(), ctrlBuilder::setMemoryLimit); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java index eda70ffb2bc..0d8f7148758 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java @@ -120,9 +120,7 @@ public class ContentCluster extends AbstractConfigProducer<AbstractConfigProduce ContentCluster c = new ContentCluster(context.getParentProducer(), getClusterId(contentElement), documentDefinitions, globallyDistributedDocuments, routingSelection, deployState.zone(), deployState.isHosted()); - boolean enableFeedBlockInDistributor = deployState.getProperties().featureFlags().enableFeedBlockInDistributor(); - var resourceLimits = new ClusterResourceLimits.Builder(enableFeedBlockInDistributor, - stateIsHosted(deployState), + var resourceLimits = new ClusterResourceLimits.Builder(stateIsHosted(deployState), deployState.featureFlags().resourceLimitDisk(), deployState.featureFlags().resourceLimitMemory()) .build(contentElement); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/ClusterResourceLimitsTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/ClusterResourceLimitsTest.java index 8cc7805fe3e..86847dae11f 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/ClusterResourceLimitsTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/ClusterResourceLimitsTest.java @@ -21,7 +21,6 @@ import static org.junit.Assert.fail; public class ClusterResourceLimitsTest { private static class Fixture { - private final boolean enableFeedBlockInDistributor; private final boolean hostedVespa; private final ResourceLimits.Builder ctrlBuilder = new ResourceLimits.Builder(); private final ResourceLimits.Builder nodeBuilder = new ResourceLimits.Builder(); @@ -30,12 +29,7 @@ public class ClusterResourceLimitsTest { this(false); } - public Fixture(boolean enableFeedBlockInDistributor) { - this(enableFeedBlockInDistributor, false); - } - - public Fixture(boolean enableFeedBlockInDistributor, boolean hostedVespa) { - this.enableFeedBlockInDistributor = enableFeedBlockInDistributor; + public Fixture(boolean hostedVespa) { this.hostedVespa = hostedVespa; } @@ -57,8 +51,7 @@ public class ClusterResourceLimitsTest { } public ClusterResourceLimits build() { ModelContext.FeatureFlags featureFlags = new TestProperties(); - var builder = new ClusterResourceLimits.Builder(enableFeedBlockInDistributor, - hostedVespa, + var builder = new ClusterResourceLimits.Builder(hostedVespa, featureFlags.resourceLimitDisk(), featureFlags.resourceLimitMemory()); builder.setClusterControllerBuilder(ctrlBuilder); @@ -71,50 +64,32 @@ public class ClusterResourceLimitsTest { public void content_node_limits_are_derived_from_cluster_controller_limits_if_not_set() { assertLimits(0.4, 0.7, 0.7, 0.85, new Fixture().ctrlDisk(0.4).ctrlMemory(0.7)); - assertLimits(0.4, null, 0.7, null, - new Fixture().ctrlDisk(0.4)); - assertLimits(null, 0.7, null, 0.85, - new Fixture().ctrlMemory(0.7)); - - - assertLimits(0.4, 0.7, 0.7, 0.85, - new Fixture(true).ctrlDisk(0.4).ctrlMemory(0.7)); assertLimits(0.4, 0.8, 0.7, 0.9, - new Fixture(true).ctrlDisk(0.4)); + new Fixture().ctrlDisk(0.4)); assertLimits(0.8, 0.7, 0.9, 0.85, - new Fixture(true).ctrlMemory(0.7)); + new Fixture().ctrlMemory(0.7)); } @Test public void content_node_limits_can_be_set_explicit() { assertLimits(0.4, 0.7, 0.9, 0.95, new Fixture().ctrlDisk(0.4).ctrlMemory(0.7).nodeDisk(0.9).nodeMemory(0.95)); - assertLimits(0.4, null, 0.95, null, - new Fixture().ctrlDisk(0.4).nodeDisk(0.95)); - assertLimits(null, 0.7, null, 0.95, - new Fixture().ctrlMemory(0.7).nodeMemory(0.95)); - - assertLimits(0.4, 0.7, 0.9, 0.95, - new Fixture(true).ctrlDisk(0.4).ctrlMemory(0.7).nodeDisk(0.9).nodeMemory(0.95)); assertLimits(0.4, 0.8, 0.95, 0.9, - new Fixture(true).ctrlDisk(0.4).nodeDisk(0.95)); + new Fixture().ctrlDisk(0.4).nodeDisk(0.95)); assertLimits(0.8, 0.7, 0.9, 0.95, - new Fixture(true).ctrlMemory(0.7).nodeMemory(0.95)); + new Fixture().ctrlMemory(0.7).nodeMemory(0.95)); } @Test public void cluster_controller_limits_are_equal_to_content_node_limits_minus_one_percent_if_not_set() { assertLimits(0.89, 0.94, 0.9, 0.95, new Fixture().nodeDisk(0.9).nodeMemory(0.95)); - assertLimits(0.89, null, 0.9, null, + assertLimits(0.89, 0.8, 0.9, 0.9, new Fixture().nodeDisk(0.9)); - assertLimits(null, 0.94, null, 0.95, + assertLimits(0.8, 0.94, 0.9, 0.95, new Fixture().nodeMemory(0.95)); - assertLimits(null, 0.0, null, 0.005, + assertLimits(0.8, 0.0, 0.9, 0.005, new Fixture().nodeMemory(0.005)); - - assertLimits(0.89, 0.94, 0.9, 0.95, - new Fixture(true).nodeDisk(0.9).nodeMemory(0.95)); } @Test @@ -193,7 +168,6 @@ public class ClusterResourceLimitsTest { "</cluster>"); ClusterResourceLimits.Builder builder = new ClusterResourceLimits.Builder(true, - true, featureFlags.resourceLimitDisk(), featureFlags.resourceLimitMemory()); return builder.build(new ModelElement((limitsInXml ? clusterXml : noLimitsXml).getDocumentElement())); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentSchemaClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentSchemaClusterTest.java index 68e722f45d3..fac6eeb3cc3 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentSchemaClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentSchemaClusterTest.java @@ -77,12 +77,6 @@ public class ContentSchemaClusterTest { return new ProtonConfig(builder); } - private static ContentCluster createClusterWithFeatureFlag(String clusterXml, boolean enableFeedBlockInDistributor) throws Exception { - var deployStateBuilder = new DeployState.Builder().properties( - new TestProperties().enableFeedBlockInDistributor(enableFeedBlockInDistributor)); - return createCluster(clusterXml, deployStateBuilder); - } - private static void assertProtonResourceLimits(double expDiskLimit, double expMemoryLimit, String clusterXml) throws Exception { assertProtonResourceLimits(expDiskLimit, expMemoryLimit, createCluster(clusterXml)); } @@ -142,15 +136,8 @@ public class ContentSchemaClusterTest { } @Test - public void default_resource_limits_when_feed_block_is_disabled_in_distributor() throws Exception { - var cluster = createClusterWithFeatureFlag(new ContentClusterBuilder().getXml(), false); - assertProtonResourceLimits(0.8, 0.8, cluster); - assertClusterControllerResourceLimits(0.8, 0.8, cluster); - } - - @Test - public void default_resource_limits_when_feed_block_is_enabled_in_distributor() throws Exception { - var cluster = createClusterWithFeatureFlag(new ContentClusterBuilder().getXml(), true); + public void default_resource_limits_with_feed_block_in_distributor() throws Exception { + var cluster = createCluster(new ContentClusterBuilder().getXml()); assertProtonResourceLimits(0.9, 0.9, cluster); assertClusterControllerResourceLimits(0.8, 0.8, cluster); } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/FleetControllerClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/FleetControllerClusterTest.java index b27a8a192bc..e074d493e68 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/FleetControllerClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/FleetControllerClusterTest.java @@ -19,14 +19,12 @@ public class FleetControllerClusterTest { private ClusterControllerConfig parse(String xml, TestProperties props) { Document doc = XML.getDocument(xml); var deployState = new DeployState.Builder().properties(props).build(); - boolean enableFeedBlockInDistributor = deployState.getProperties().featureFlags().enableFeedBlockInDistributor(); MockRoot root = new MockRoot("", deployState); var clusterElement = new ModelElement(doc.getDocumentElement()); ModelContext.FeatureFlags featureFlags = new TestProperties(); return new ClusterControllerConfig.Builder("storage", clusterElement, - new ClusterResourceLimits.Builder(enableFeedBlockInDistributor, - false, + new ClusterResourceLimits.Builder(false, featureFlags.resourceLimitDisk(), featureFlags.resourceLimitMemory()) .build(clusterElement).getClusterControllerLimits()) @@ -34,7 +32,7 @@ public class FleetControllerClusterTest { } private ClusterControllerConfig parse(String xml) { - return parse(xml, new TestProperties().enableFeedBlockInDistributor(true)); + return parse(xml, new TestProperties()); } @Test @@ -104,7 +102,6 @@ public class FleetControllerClusterTest { assertEquals(0.0, config.min_node_ratio_per_group(), 0.01); } - @Test public void default_cluster_feed_block_limits_are_set() { assertLimits(0.8, 0.8, getConfigForBasicCluster()); @@ -143,17 +140,6 @@ public class FleetControllerClusterTest { } @Test - public void feature_flag_controls_enable_cluster_feed_block() { - verifyThatFeatureFlagControlsEnableClusterFeedBlock(true); - verifyThatFeatureFlagControlsEnableClusterFeedBlock(false); - } - - private void verifyThatFeatureFlagControlsEnableClusterFeedBlock(boolean flag) { - var config = getConfigForBasicCluster(new TestProperties().enableFeedBlockInDistributor(flag)); - assertEquals(flag, config.enable_cluster_feed_block()); - } - - @Test public void feature_flag_controls_min_node_ratio_per_group() { verifyFeatureFlagControlsMinNodeRatioPerGroup(0.0, new TestProperties()); verifyFeatureFlagControlsMinNodeRatioPerGroup(0.3, @@ -175,6 +161,6 @@ public class FleetControllerClusterTest { } private FleetcontrollerConfig getConfigForBasicCluster() { - return getConfigForBasicCluster(new TestProperties().enableFeedBlockInDistributor(true)); + return getConfigForBasicCluster(new TestProperties()); } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java index c50b75b3460..7cc5f5edd14 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java @@ -177,7 +177,6 @@ public class ModelContextImpl implements ModelContext { private final boolean skipMbusReplyThread; private final boolean useAsyncMessageHandlingOnSchedule; private final double feedConcurrency; - private final boolean enableFeedBlockInDistributor; private final List<String> allowedAthenzProxyIdentities; private final int maxActivationInhibitedOutOfSyncGroups; private final ToIntFunction<ClusterSpec.Type> jvmOmitStackTraceInFastThrow; @@ -220,7 +219,6 @@ public class ModelContextImpl implements ModelContext { this.skipMbusReplyThread = flagValue(source, appId, Flags.SKIP_MBUS_REPLY_THREAD); this.useAsyncMessageHandlingOnSchedule = flagValue(source, appId, Flags.USE_ASYNC_MESSAGE_HANDLING_ON_SCHEDULE); this.feedConcurrency = flagValue(source, appId, Flags.FEED_CONCURRENCY); - this.enableFeedBlockInDistributor = flagValue(source, appId, Flags.ENABLE_FEED_BLOCK_IN_DISTRIBUTOR); this.allowedAthenzProxyIdentities = flagValue(source, appId, Flags.ALLOWED_ATHENZ_PROXY_IDENTITIES); this.maxActivationInhibitedOutOfSyncGroups = flagValue(source, appId, Flags.MAX_ACTIVATION_INHIBITED_OUT_OF_SYNC_GROUPS); this.jvmOmitStackTraceInFastThrow = type -> flagValueAsInt(source, appId, type, PermanentFlags.JVM_OMIT_STACK_TRACE_IN_FAST_THROW); @@ -263,7 +261,6 @@ public class ModelContextImpl implements ModelContext { @Override public boolean skipMbusReplyThread() { return skipMbusReplyThread; } @Override public boolean useAsyncMessageHandlingOnSchedule() { return useAsyncMessageHandlingOnSchedule; } @Override public double feedConcurrency() { return feedConcurrency; } - @Override public boolean enableFeedBlockInDistributor() { return enableFeedBlockInDistributor; } @Override public List<String> allowedAthenzProxyIdentities() { return allowedAthenzProxyIdentities; } @Override public int maxActivationInhibitedOutOfSyncGroups() { return maxActivationInhibitedOutOfSyncGroups; } @Override public String jvmOmitStackTraceInFastThrowOption(ClusterSpec.Type type) { diff --git a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/HttpRequestFactory.java b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/HttpRequestFactory.java index c54fa1cf1b9..8edc2eb84d0 100644 --- a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/HttpRequestFactory.java +++ b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/HttpRequestFactory.java @@ -13,6 +13,7 @@ import java.security.cert.X509Certificate; import java.util.Enumeration; import static com.yahoo.jdisc.Response.Status.BAD_REQUEST; +import static com.yahoo.jdisc.Response.Status.METHOD_NOT_ALLOWED; import static com.yahoo.jdisc.http.server.jetty.RequestUtils.getConnection; import static com.yahoo.jdisc.http.server.jetty.RequestUtils.getConnectorLocalPort; @@ -27,7 +28,7 @@ class HttpRequestFactory { HttpRequest httpRequest = HttpRequest.newServerRequest( container, getUri(servletRequest), - HttpRequest.Method.valueOf(servletRequest.getMethod()), + getMethod(servletRequest), HttpRequest.Version.fromString(servletRequest.getProtocol()), new InetSocketAddress(servletRequest.getRemoteAddr(), servletRequest.getRemotePort()), getConnection((Request) servletRequest).getCreatedTimeStamp()); @@ -39,6 +40,15 @@ class HttpRequestFactory { } } + private static HttpRequest.Method getMethod(HttpServletRequest servletRequest) { + String method = servletRequest.getMethod(); + try { + return HttpRequest.Method.valueOf(method); + } catch (IllegalArgumentException e) { + throw new RequestException(METHOD_NOT_ALLOWED, "Invalid method '" + method + "'"); + } + } + // Implementation based on org.eclipse.jetty.server.Request.getRequestURL(), but with the connector's local port instead public static URI getUri(HttpServletRequest servletRequest) { try { diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index 65d4d76f5c3..6d852c65819 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -144,13 +144,6 @@ public class Flags { "Takes effect at redeployment", ZONE_ID, APPLICATION_ID); - public static final UnboundBooleanFlag ENABLE_FEED_BLOCK_IN_DISTRIBUTOR = defineFeatureFlag( - "enable-feed-block-in-distributor", true, - List.of("geirst"), "2021-01-27", "2022-06-01", - "Enables blocking of feed in the distributor if resource usage is above limit on at least one content node", - "Takes effect at redeployment", - ZONE_ID, APPLICATION_ID); - public static final UnboundBooleanFlag CONTAINER_DUMP_HEAP_ON_SHUTDOWN_TIMEOUT = defineFeatureFlag( "container-dump-heap-on-shutdown-timeout", false, List.of("baldersheim"), "2021-09-25", "2022-06-01", diff --git a/vespalib/src/tests/alloc/alloc_test.cpp b/vespalib/src/tests/alloc/alloc_test.cpp index 2b6d3ee4613..da41d75b479 100644 --- a/vespalib/src/tests/alloc/alloc_test.cpp +++ b/vespalib/src/tests/alloc/alloc_test.cpp @@ -139,22 +139,28 @@ TEST("rounding of large mmaped buffer") { EXPECT_EQUAL(MemoryAllocator::HUGEPAGE_SIZE*12ul, buf.size()); } +void verifyExtension(Alloc& buf, size_t currSZ, size_t newSZ) { + bool expectSuccess = (currSZ != newSZ); + void* oldPtr = buf.get(); + EXPECT_EQUAL(currSZ, buf.size()); + EXPECT_EQUAL(expectSuccess, buf.resize_inplace(currSZ + 1)); + EXPECT_EQUAL(oldPtr, buf.get()); + EXPECT_EQUAL(newSZ, buf.size()); +} + TEST("heap alloc can not be extended") { Alloc buf = Alloc::allocHeap(100); - void * oldPtr = buf.get(); - EXPECT_EQUAL(100ul, buf.size()); - EXPECT_FALSE(buf.resize_inplace(101)); - EXPECT_EQUAL(oldPtr, buf.get()); - EXPECT_EQUAL(100ul, buf.size()); + verifyExtension(buf, 100, 100); +} + +TEST("mmap alloc cannot be extended from zero") { + Alloc buf = Alloc::allocMMap(0); + verifyExtension(buf, 0, 0); } TEST("auto alloced heap alloc can not be extended") { Alloc buf = Alloc::alloc(100); - void * oldPtr = buf.get(); - EXPECT_EQUAL(100ul, buf.size()); - EXPECT_FALSE(buf.resize_inplace(101)); - EXPECT_EQUAL(oldPtr, buf.get()); - EXPECT_EQUAL(100ul, buf.size()); + verifyExtension(buf, 100, 100); } TEST("auto alloced heap alloc can not be extended, even if resize will be mmapped") { @@ -166,15 +172,6 @@ TEST("auto alloced heap alloc can not be extended, even if resize will be mmappe EXPECT_EQUAL(100ul, buf.size()); } -void verifyExtension(Alloc & buf, size_t currSZ, size_t newSZ) { - bool expectSuccess = (currSZ != newSZ); - void * oldPtr = buf.get(); - EXPECT_EQUAL(currSZ, buf.size()); - EXPECT_EQUAL(expectSuccess, buf.resize_inplace(currSZ+1)); - EXPECT_EQUAL(oldPtr, buf.get()); - EXPECT_EQUAL(newSZ, buf.size()); -} - void ensureRoomForExtension(const Alloc & buf, Alloc & reserved) { // Normally mmapping starts at the top and grows down in address space. // Then there is no room to extend the last mapping. @@ -253,6 +250,11 @@ TEST("heap alloc can not be shrinked") { EXPECT_EQUAL(101ul, buf.size()); } +TEST("heap alloc cannot be shrunk to zero") { + Alloc buf = Alloc::allocHeap(101); + EXPECT_FALSE(buf.resize_inplace(0)); +} + TEST("mmap alloc can be shrinked") { Alloc buf = Alloc::allocMMap(4097); void * oldPtr = buf.get(); @@ -262,6 +264,11 @@ TEST("mmap alloc can be shrinked") { EXPECT_EQUAL(4_Ki, buf.size()); } +TEST("mmap alloc cannot be shrunk to zero") { + Alloc buf = Alloc::allocMMap(4097); + EXPECT_FALSE(buf.resize_inplace(0)); +} + TEST("auto alloced heap alloc can not be shrinked") { Alloc buf = Alloc::alloc(101); void * oldPtr = buf.get(); @@ -271,6 +278,11 @@ TEST("auto alloced heap alloc can not be shrinked") { EXPECT_EQUAL(101ul, buf.size()); } +TEST("auto alloced heap alloc cannot be shrunk to zero") { + Alloc buf = Alloc::alloc(101); + EXPECT_FALSE(buf.resize_inplace(0)); +} + TEST("auto alloced mmap alloc can be shrinked") { static constexpr size_t SZ = MemoryAllocator::HUGEPAGE_SIZE; Alloc buf = Alloc::alloc(SZ + 1); @@ -281,6 +293,11 @@ TEST("auto alloced mmap alloc can be shrinked") { EXPECT_EQUAL(SZ, buf.size()); } +TEST("auto alloced mmap alloc cannot be shrunk to zero") { + Alloc buf = Alloc::alloc(MemoryAllocator::HUGEPAGE_SIZE + 1); + EXPECT_FALSE(buf.resize_inplace(0)); +} + TEST("auto alloced mmap alloc can not be shrinked below HUGEPAGE_SIZE/2 + 1 ") { static constexpr size_t SZ = MemoryAllocator::HUGEPAGE_SIZE; Alloc buf = Alloc::alloc(SZ + 1); diff --git a/vespalib/src/tests/array/array_test.cpp b/vespalib/src/tests/array/array_test.cpp index 719623c9401..5f80ee893da 100644 --- a/vespalib/src/tests/array/array_test.cpp +++ b/vespalib/src/tests/array/array_test.cpp @@ -2,6 +2,7 @@ #include <vespa/vespalib/stllike/string.h> #include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/test/memory_allocator_observer.h> #include <vespa/vespalib/util/array.hpp> #include <vespa/vespalib/util/size_literals.h> #include <deque> @@ -27,6 +28,11 @@ std::ostream & operator << (std::ostream & os, const Array<T> & a) } +using alloc::Alloc; +using alloc::MemoryAllocator; +using MyMemoryAllocator = vespalib::alloc::test::MemoryAllocatorObserver; +using AllocStats = MyMemoryAllocator::Stats; + class Clever { public: Clever() : _counter(&_global) { (*_counter)++; } @@ -351,4 +357,46 @@ TEST_F("require that try_unreserve() succeedes if mmap can be shrinked", Unreser EXPECT_EQUAL(oldPtr, newPtr); } +struct Fixture { + AllocStats stats; + std::unique_ptr<MemoryAllocator> allocator; + Alloc initial_alloc; + Array<int> arr; + + Fixture(); + ~Fixture(); +}; + +Fixture::Fixture() + : stats(), + allocator(std::make_unique<MyMemoryAllocator>(stats)), + initial_alloc(Alloc::alloc_with_allocator(allocator.get())), + arr(initial_alloc) +{ +} + +Fixture::~Fixture() = default; + +TEST_F("require that memory allocator can be set", Fixture) +{ + f.arr.resize(1); + EXPECT_EQUAL(AllocStats(1, 0), f.stats); +} + +TEST_F("require that memory allocator is preserved across reset", Fixture) +{ + f.arr.resize(1); + f.arr.reset(); + f.arr.resize(1); + EXPECT_EQUAL(AllocStats(2, 1), f.stats); +} + +TEST_F("require that created array uses same memory allocator", Fixture) +{ + auto arr2 = f.arr.create(); + EXPECT_EQUAL(AllocStats(0, 0), f.stats); + arr2.resize(1); + EXPECT_EQUAL(AllocStats(1, 0), f.stats); +} + TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/vespalib/src/vespa/vespalib/util/alloc.cpp b/vespalib/src/vespa/vespalib/util/alloc.cpp index 69f2eedcecb..bf5f0200600 100644 --- a/vespalib/src/vespa/vespalib/util/alloc.cpp +++ b/vespalib/src/vespa/vespalib/util/alloc.cpp @@ -381,6 +381,9 @@ MMapAllocator::sresize_inplace(PtrAndSize current, size_t newSize) { size_t MMapAllocator::extend_inplace(PtrAndSize current, size_t newSize) { + if (current.second == 0u) { + return 0u; + } PtrAndSize got = MMapAllocator::salloc(newSize - current.second, static_cast<char *>(current.first)+current.second); if ((static_cast<const char *>(current.first) + current.second) == static_cast<const char *>(got.first)) { return current.second + got.second; @@ -477,6 +480,9 @@ Alloc::allocHeap(size_t sz) bool Alloc::resize_inplace(size_t newSize) { + if (newSize == 0u) { + return size() == 0u; + } size_t extendedSize = _allocator->resize_inplace(_alloc, newSize); if (extendedSize >= newSize) { _alloc.second = extendedSize; diff --git a/vespalib/src/vespa/vespalib/util/alloc.h b/vespalib/src/vespa/vespalib/util/alloc.h index d25fb6f6a7c..4066894b4e3 100644 --- a/vespalib/src/vespa/vespalib/util/alloc.h +++ b/vespalib/src/vespa/vespalib/util/alloc.h @@ -62,6 +62,13 @@ public: std::swap(_alloc, rhs._alloc); std::swap(_allocator, rhs._allocator); } + void reset() { + if (_alloc.first != nullptr) { + _allocator->free(_alloc); + _alloc.first = nullptr; + _alloc.second = 0u; + } + } Alloc create(size_t sz) const noexcept { return (sz == 0) ? Alloc(_allocator) : Alloc(_allocator, sz); } diff --git a/vespalib/src/vespa/vespalib/util/array.h b/vespalib/src/vespa/vespalib/util/array.h index 30d87cd98f6..cb5d5c7cc63 100644 --- a/vespalib/src/vespa/vespalib/util/array.h +++ b/vespalib/src/vespa/vespalib/util/array.h @@ -135,6 +135,7 @@ public: std::destroy(array(0), array(_sz)); _sz = 0; } + void reset(); bool empty() const { return _sz == 0; } T & operator [] (size_t i) { return *array(i); } const T & operator [] (size_t i) const { return *array(i); } @@ -145,6 +146,7 @@ public: rhs._sz = 0; return std::move(rhs._array); } + Array<T> create() const; private: T * array(size_t i) { return static_cast<T *>(_array.get()) + i; } const T * array(size_t i) const { return static_cast<const T *>(_array.get()) + i; } diff --git a/vespalib/src/vespa/vespalib/util/array.hpp b/vespalib/src/vespa/vespalib/util/array.hpp index e9070f5759c..72178f0391b 100644 --- a/vespalib/src/vespa/vespalib/util/array.hpp +++ b/vespalib/src/vespa/vespalib/util/array.hpp @@ -205,5 +205,20 @@ void Array<T>::cleanup() Alloc().swap(_array); } +template <typename T> +void Array<T>::reset() +{ + std::destroy(array(0), array(_sz)); + _sz = 0; + _array.reset(); +} + +template <typename T> +Array<T> +Array<T>::create() const +{ + return Array<T>(_array); // Use same memory allocator +} + } |