diff options
33 files changed, 320 insertions, 101 deletions
diff --git a/build_settings.cmake b/build_settings.cmake index 10f4c7ff926..8dcef41caff 100644 --- a/build_settings.cmake +++ b/build_settings.cmake @@ -55,6 +55,9 @@ if ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang" OR "${CMAKE_CXX_COMPILER_ID}" ST else() set(CXX_SPECIFIC_WARN_OPTS "-Wnoexcept -Wsuggest-override -Wnon-virtual-dtor -Wformat-security") if(VESPA_OS_DISTRO_COMBINED STREQUAL "centos 8" OR + (VESPA_OS_DISTRO STREQUAL "rocky" AND + VESPA_OS_DISTRO_VERSION VERSION_GREATER_EQUAL "8" AND + VESPA_OS_DISTRO_VERSION VERSION_LESS "9") OR (VESPA_OS_DISTRO STREQUAL "rhel" AND VESPA_OS_DISTRO_VERSION VERSION_GREATER_EQUAL "8" AND VESPA_OS_DISTRO_VERSION VERSION_LESS "9")) diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java index 869be883661..4b0e30e04da 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java @@ -99,6 +99,8 @@ public interface ModelContext { @ModelFeatureFlag(owners = {"geirst", "vekterli"}) default int numDistributorStripes() { return 0; } @ModelFeatureFlag(owners = {"arnej"}) default boolean requireConnectivityCheck() { return true; } @ModelFeatureFlag(owners = {"hmusum"}) default boolean throwIfResourceLimitsSpecified() { return false; } + @ModelFeatureFlag(owners = {"hmusum"}) default double resourceLimitDisk() { return 0.8; } + @ModelFeatureFlag(owners = {"hmusum"}) default double resourceLimitMemory() { return 0.8; } } /** Warning: As elsewhere in this package, do not make backwards incompatible changes that will break old config models! */ diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java index 308dcf25d2a..62c192e2c99 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java @@ -64,6 +64,8 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea private boolean allowDisableMtls = true; private boolean dryRunOnnxOnSetup = false; private List<X509Certificate> operatorCertificates = Collections.emptyList(); + private double resourceLimitDisk = 0.8; + private double resourceLimitMemory = 0.8; @Override public ModelContext.FeatureFlags featureFlags() { return this; } @Override public boolean multitenant() { return multitenant; } @@ -107,6 +109,8 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea @Override public int maxConcurrentMergesPerNode() { return maxConcurrentMergesPerNode; } @Override public int maxMergeQueueSize() { return maxMergeQueueSize; } @Override public boolean dryRunOnnxOnSetup() { return dryRunOnnxOnSetup; } + @Override public double resourceLimitDisk() { return resourceLimitDisk; } + @Override public double resourceLimitMemory() { return resourceLimitMemory; } public TestProperties setDryRunOnnxOnSetup(boolean value) { dryRunOnnxOnSetup = value; @@ -260,6 +264,16 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea return this; } + public TestProperties setResourceLimitDisk(double value) { + this.resourceLimitDisk = value; + return this; + } + + public TestProperties setResourceLimitMemory(double value) { + this.resourceLimitMemory = value; + return this; + } + public static class Spec implements ConfigServerSpec { private final String hostName; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/ClusterResourceLimits.java b/config-model/src/main/java/com/yahoo/vespa/model/content/ClusterResourceLimits.java index 638864d85bb..dc5e6c9baee 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/ClusterResourceLimits.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/ClusterResourceLimits.java @@ -39,15 +39,21 @@ public class ClusterResourceLimits { private final boolean hostedVespa; private final boolean throwIfSpecified; private final DeployLogger deployLogger; + private final double resourceLimitDisk; + private final double resourceLimitMemory; private ResourceLimits.Builder ctrlBuilder = new ResourceLimits.Builder(); private ResourceLimits.Builder nodeBuilder = new ResourceLimits.Builder(); - public Builder(boolean enableFeedBlockInDistributor, boolean hostedVespa, boolean throwIfSpecified, DeployLogger deployLogger) { + public Builder(boolean enableFeedBlockInDistributor, boolean hostedVespa, boolean throwIfSpecified, + DeployLogger deployLogger, double resourceLimitDisk, double resourceLimitMemory) { this.enableFeedBlockInDistributor = enableFeedBlockInDistributor; this.hostedVespa = hostedVespa; this.throwIfSpecified = throwIfSpecified; this.deployLogger = deployLogger; + this.resourceLimitDisk = resourceLimitDisk; + this.resourceLimitMemory = resourceLimitMemory; + verifyLimits(resourceLimitDisk, resourceLimitMemory); } public ClusterResourceLimits build(ModelElement clusterElem) { @@ -80,8 +86,14 @@ public class ClusterResourceLimits { private void deriveLimits() { if (enableFeedBlockInDistributor) { // This also ensures that content nodes limits are derived according to the formula in calcContentNodeLimit(). - considerSettingDefaultClusterControllerLimit(ctrlBuilder.getDiskLimit(), nodeBuilder.getDiskLimit(), ctrlBuilder::setDiskLimit); - considerSettingDefaultClusterControllerLimit(ctrlBuilder.getMemoryLimit(), nodeBuilder.getMemoryLimit(), ctrlBuilder::setMemoryLimit); + considerSettingDefaultClusterControllerLimit(ctrlBuilder.getDiskLimit(), + nodeBuilder.getDiskLimit(), + ctrlBuilder::setDiskLimit, + resourceLimitDisk); + considerSettingDefaultClusterControllerLimit(ctrlBuilder.getMemoryLimit(), + nodeBuilder.getMemoryLimit(), + ctrlBuilder::setMemoryLimit, + resourceLimitMemory); } deriveClusterControllerLimit(ctrlBuilder.getDiskLimit(), nodeBuilder.getDiskLimit(), ctrlBuilder::setDiskLimit); @@ -93,10 +105,11 @@ public class ClusterResourceLimits { private void considerSettingDefaultClusterControllerLimit(Optional<Double> clusterControllerLimit, Optional<Double> contentNodeLimit, - Consumer<Double> setter) { + Consumer<Double> setter, + double resourceLimit) { // TODO: remove this when feed block in distributor is default enabled. if (clusterControllerLimit.isEmpty() && contentNodeLimit.isEmpty()) { - setter.accept(0.8); + setter.accept(resourceLimit); } } @@ -120,10 +133,25 @@ public class ClusterResourceLimits { } private double calcContentNodeLimit(double clusterControllerLimit) { - // Note that validation in the range [0.0-1.0] is handled by the rnc schema. return clusterControllerLimit + ((1.0 - clusterControllerLimit) / 2); } + + private void verifyLimits(double resourceLimitDisk, double resourceLimitMemory) { + verifyLimitInRange(resourceLimitDisk, "disk"); + verifyLimitInRange(resourceLimitMemory, "memory"); + } + + private void verifyLimitInRange(double limit, String type) { + String message = "Resource limit for " + type + " is set to illegal value " + limit + + ", but must be in the range [0.0, 1.0]"; + if (limit < 0.0) + throw new IllegalArgumentException(message); + + if (limit > 1.0) + throw new IllegalArgumentException(message); + } + } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java index c298b7f5f5a..fabcbbcc9ec 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java @@ -124,7 +124,9 @@ public class ContentCluster extends AbstractConfigProducer<AbstractConfigProduce var resourceLimits = new ClusterResourceLimits.Builder(enableFeedBlockInDistributor, stateIsHosted(deployState), deployState.featureFlags().throwIfResourceLimitsSpecified(), - deployState.getDeployLogger()) + deployState.getDeployLogger(), + deployState.featureFlags().resourceLimitDisk(), + deployState.featureFlags().resourceLimitMemory()) .build(contentElement); c.clusterControllerConfig = new ClusterControllerConfig.Builder(getClusterId(contentElement), contentElement, diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/DomResourceLimitsBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/DomResourceLimitsBuilder.java index 37adb73bc15..32b0f5b6477 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/DomResourceLimitsBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/DomResourceLimitsBuilder.java @@ -24,10 +24,12 @@ public class DomResourceLimitsBuilder { if (hostedVespa) { String message = "Element '" + resourceLimits + "' is not allowed to be set"; - if (throwIfSpecified) - throw new IllegalArgumentException(message); - else - deployLogger.logApplicationPackage(Level.WARNING, message); + if (throwIfSpecified) throw new IllegalArgumentException(message); + + + deployLogger.logApplicationPackage(Level.WARNING, message); + // TODO: return (default values will then be used). Cannot be done now as an app needs current behavior + //return builder; } if (resourceLimits.child("disk") != null) { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/ClusterResourceLimitsTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/ClusterResourceLimitsTest.java index 4324f257922..c0b1a64bace 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/ClusterResourceLimitsTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/ClusterResourceLimitsTest.java @@ -2,10 +2,13 @@ package com.yahoo.vespa.model.content; import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.config.model.api.ModelContext; import com.yahoo.config.model.application.provider.BaseDeployLogger; +import com.yahoo.config.model.deploy.TestProperties; import com.yahoo.searchdefinition.derived.TestableDeployLogger; import com.yahoo.text.XML; import com.yahoo.vespa.model.builder.xml.dom.ModelElement; +import org.junit.Ignore; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; @@ -24,15 +27,23 @@ public class ClusterResourceLimitsTest { private static class Fixture { private final boolean enableFeedBlockInDistributor; + private final boolean hostedVespa; + private final boolean throwIfSpecified; private final ResourceLimits.Builder ctrlBuilder = new ResourceLimits.Builder(); private final ResourceLimits.Builder nodeBuilder = new ResourceLimits.Builder(); public Fixture() { - this.enableFeedBlockInDistributor = false; + this(false); } public Fixture(boolean enableFeedBlockInDistributor) { + this(enableFeedBlockInDistributor, false, false); + } + + public Fixture(boolean enableFeedBlockInDistributor, boolean hostedVespa, boolean throwIfSpecified) { this.enableFeedBlockInDistributor = enableFeedBlockInDistributor; + this.hostedVespa = hostedVespa; + this.throwIfSpecified = throwIfSpecified; } public Fixture ctrlDisk(double limit) { @@ -52,10 +63,13 @@ public class ClusterResourceLimitsTest { return this; } public ClusterResourceLimits build() { + ModelContext.FeatureFlags featureFlags = new TestProperties(); var builder = new ClusterResourceLimits.Builder(enableFeedBlockInDistributor, - false, - false, - new BaseDeployLogger()); + hostedVespa, + throwIfSpecified, + new BaseDeployLogger(), + featureFlags.resourceLimitDisk(), + featureFlags.resourceLimitMemory()); builder.setClusterControllerBuilder(ctrlBuilder); builder.setContentNodeBuilder(nodeBuilder); return builder.build(); @@ -130,27 +144,86 @@ public class ClusterResourceLimitsTest { } @Test - public void exception_is_thrown_when_resource_limits_are_specified() { + @Ignore // TODO: Remove hosted_limits_are_used_if_app_is_allowed_to_set_limits and enable this when code is fixed to do so + public void hosted_log_when_resource_limits_are_specified() { TestableDeployLogger logger = new TestableDeployLogger(); - buildClusterResourceLimitsAndLogIfSpecified(logger); + var limits = hostedBuildAndLogIfSpecified(logger); assertEquals(1, logger.warnings.size()); assertEquals("Element 'resource-limits' is not allowed to be set", logger.warnings.get(0)); + // Verify that default limits are used + assertLimits(0.8, 0.8, limits.getClusterControllerLimits()); + assertLimits(0.9, 0.9, limits.getContentNodeLimits()); + } + + @Test + public void hosted_exception_is_thrown_when_resource_limits_are_specified() { expectedException.expect(IllegalArgumentException.class); expectedException.expectMessage(containsString("Element 'resource-limits' is not allowed to be set")); - buildClusterResourceLimitsAndThrowIfSpecified(logger); + hostedBuildAndThrowIfSpecified(); + } + + @Test + // TODO: Remove this and enable hosted_log_when_resource_limits_are_specified when code is fixed to do so + public void hosted_limits_are_used_if_app_is_allowed_to_set_limits() { + TestableDeployLogger logger = new TestableDeployLogger(); + + var limits = hostedBuildAndLogIfSpecified(logger); + assertEquals(1, logger.warnings.size()); + assertEquals("Element 'resource-limits' is not allowed to be set", logger.warnings.get(0)); + + // Verify that limits in XML are used + assertLimits(0.8, 0.92, limits.getClusterControllerLimits()); + assertLimits(0.9, 0.96, limits.getContentNodeLimits()); + } + + @Test + public void hosted_limits_from_feature_flag_are_used() { + TestableDeployLogger logger = new TestableDeployLogger(); + + TestProperties featureFlags = new TestProperties(); + featureFlags.setResourceLimitDisk(0.85); + featureFlags.setResourceLimitMemory(0.90); + var limits = hostedBuild(false, logger, featureFlags, false); + + // Verify that limits from feature flags are used + assertLimits(0.85, 0.90, limits.getClusterControllerLimits()); + assertLimits(0.925, 0.95, limits.getContentNodeLimits()); } - private void buildClusterResourceLimitsAndThrowIfSpecified(DeployLogger deployLogger) { - buildClusterResourceLimits(true, deployLogger); + @Test + public void exception_is_thrown_when_resource_limits_are_out_of_range() { + TestableDeployLogger logger = new TestableDeployLogger(); + + TestProperties featureFlags = new TestProperties(); + featureFlags.setResourceLimitDisk(1.1); + expectedException.expect(IllegalArgumentException.class); + expectedException.expectMessage(containsString("Resource limit for disk is set to illegal value 1.1, but must be in the range [0.0, 1.0]")); + hostedBuild(false, logger, featureFlags, false); + + featureFlags = new TestProperties(); + featureFlags.setResourceLimitDisk(-0.1); + expectedException.expectMessage(containsString("Resource limit for disk is set to illegal value -0.1, but must be in the range [0.0, 1.0]")); + hostedBuild(false, logger, featureFlags, false); + } + + private void hostedBuildAndThrowIfSpecified() { + hostedBuild(true, new TestableDeployLogger(), new TestProperties(), true); } - private void buildClusterResourceLimitsAndLogIfSpecified(DeployLogger deployLogger) { - buildClusterResourceLimits(false, deployLogger); + private ClusterResourceLimits hostedBuildAndLogIfSpecified(DeployLogger deployLogger) { + return hostedBuild(false, deployLogger); } - private void buildClusterResourceLimits(boolean throwIfSpecified, DeployLogger deployLogger) { + private ClusterResourceLimits hostedBuild(boolean throwIfSpecified, DeployLogger deployLogger) { + return hostedBuild(throwIfSpecified, deployLogger, new TestProperties(), true); + } + + private ClusterResourceLimits hostedBuild(boolean throwIfSpecified, + DeployLogger deployLogger, + ModelContext.FeatureFlags featureFlags, + boolean limitsInXml) { Document clusterXml = XML.getDocument("<cluster id=\"test\">" + " <tuning>\n" + " <resource-limits>\n" + @@ -159,11 +232,16 @@ public class ClusterResourceLimitsTest { " </tuning>\n" + "</cluster>"); + Document noLimitsXml = XML.getDocument("<cluster id=\"test\">" + + "</cluster>"); + ClusterResourceLimits.Builder builder = new ClusterResourceLimits.Builder(true, true, throwIfSpecified, - deployLogger); - builder.build(new ModelElement(clusterXml.getDocumentElement())); + deployLogger, + featureFlags.resourceLimitDisk(), + featureFlags.resourceLimitMemory()); + return builder.build(new ModelElement((limitsInXml ? clusterXml : noLimitsXml).getDocumentElement())); } private void assertLimits(Double expCtrlDisk, Double expCtrlMemory, Double expNodeDisk, Double expNodeMemory, Fixture f) { @@ -173,15 +251,15 @@ public class ClusterResourceLimitsTest { } private void assertLimits(Double expDisk, Double expMemory, ResourceLimits limits) { - assertLimit(expDisk, limits.getDiskLimit()); - assertLimit(expMemory, limits.getMemoryLimit()); + assertLimit(expDisk, limits.getDiskLimit(), "disk"); + assertLimit(expMemory, limits.getMemoryLimit(), "memory"); } - private void assertLimit(Double expLimit, Optional<Double> actLimit) { + private void assertLimit(Double expLimit, Optional<Double> actLimit, String limitType) { if (expLimit == null) { assertFalse(actLimit.isPresent()); } else { - assertEquals(expLimit, actLimit.get(), 0.00001); + assertEquals(limitType + " limit not as expected", expLimit, actLimit.get(), 0.00001); } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/FleetControllerClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/FleetControllerClusterTest.java index 10bb00168bb..dedf344c546 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/FleetControllerClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/FleetControllerClusterTest.java @@ -1,6 +1,7 @@ // Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.content; +import com.yahoo.config.model.api.ModelContext; import com.yahoo.config.model.application.provider.BaseDeployLogger; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.deploy.TestProperties; @@ -22,12 +23,15 @@ public class FleetControllerClusterTest { new TestProperties().enableFeedBlockInDistributor(enableFeedBlockInDistributor)).build(); MockRoot root = new MockRoot("", deployState); var clusterElement = new ModelElement(doc.getDocumentElement()); + ModelContext.FeatureFlags featureFlags = new TestProperties(); return new ClusterControllerConfig.Builder("storage", clusterElement, new ClusterResourceLimits.Builder(enableFeedBlockInDistributor, false, false, - new BaseDeployLogger()) + new BaseDeployLogger(), + featureFlags.resourceLimitDisk(), + featureFlags.resourceLimitMemory()) .build(clusterElement).getClusterControllerLimits()) .build(root.getDeployState(), root, clusterElement.getXml()); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java index 5662fb15890..0149a1a7be8 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java @@ -183,6 +183,8 @@ public class ModelContextImpl implements ModelContext { private final int largeRankExpressionLimit; private final boolean throwIfResourceLimitsSpecified; private final boolean dryRunOnnxOnSetup; + private final double resourceLimitDisk; + private final double resourceLimitMemory; public FeatureFlags(FlagSource source, ApplicationId appId) { this.dedicatedClusterControllerFlavor = parseDedicatedClusterControllerFlavor(flagValue(source, appId, Flags.DEDICATED_CLUSTER_CONTROLLER_FLAVOR)); @@ -211,6 +213,8 @@ public class ModelContextImpl implements ModelContext { this.maxMergeQueueSize = flagValue(source, appId, Flags.MAX_MERGE_QUEUE_SIZE); this.throwIfResourceLimitsSpecified = flagValue(source, appId, Flags.THROW_EXCEPTION_IF_RESOURCE_LIMITS_SPECIFIED); this.dryRunOnnxOnSetup = flagValue(source, appId, Flags.DRY_RUN_ONNX_ON_SETUP); + this.resourceLimitDisk = flagValue(source, appId, PermanentFlags.RESOURCE_LIMIT_DISK); + this.resourceLimitMemory = flagValue(source, appId, PermanentFlags.RESOURCE_LIMIT_MEMORY); } @Override public Optional<NodeResources> dedicatedClusterControllerFlavor() { return Optional.ofNullable(dedicatedClusterControllerFlavor); } @@ -241,6 +245,8 @@ public class ModelContextImpl implements ModelContext { @Override public int maxMergeQueueSize() { return maxMergeQueueSize; } @Override public boolean throwIfResourceLimitsSpecified() { return throwIfResourceLimitsSpecified; } @Override public boolean dryRunOnnxOnSetup() { return dryRunOnnxOnSetup; } + @Override public double resourceLimitDisk() { return resourceLimitDisk; } + @Override public double resourceLimitMemory() { return resourceLimitMemory; } private static <V> V flagValue(FlagSource source, ApplicationId appId, UnboundFlag<? extends V, ?, ?> flag) { return flag.bindTo(source) diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClient.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClient.java index dfe4f12195d..26e9e45f1e7 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClient.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClient.java @@ -94,7 +94,8 @@ public class SessionZooKeeperClient { String data = configCurator.getData(sessionStatusPath.getAbsolute()); return Session.Status.parse(data); } catch (Exception e) { - log.log(Level.INFO, "Unable to read session status, assuming it was deleted"); + log.log(Level.INFO, "Failed to read session status at " + sessionStatusPath.getAbsolute() + + ", will assume session has been removed: " + e.getMessage()); return Session.Status.NONE; } } diff --git a/default_build_settings.cmake b/default_build_settings.cmake index a61410ebf31..f034c9260e4 100644 --- a/default_build_settings.cmake +++ b/default_build_settings.cmake @@ -34,6 +34,12 @@ function(setup_vespa_default_build_settings_centos_8) set(DEFAULT_VESPA_LLVM_VERSION "11" PARENT_SCOPE) endfunction() +function(setup_vespa_default_build_settings_rocky_8_4) + message("-- Setting up default build settings for rocky 8.4") + set(DEFAULT_EXTRA_INCLUDE_DIRECTORY "${VESPA_DEPS}/include" "/usr/include/openblas" PARENT_SCOPE) + set(DEFAULT_VESPA_LLVM_VERSION "11" PARENT_SCOPE) +endfunction() + function(setup_vespa_default_build_settings_darwin) message("-- Setting up default build settings for darwin") if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang" OR "${CMAKE_CXX_COMPILER_ID}" STREQUAL "AppleClang") @@ -187,6 +193,8 @@ function(vespa_use_default_build_settings) setup_vespa_default_build_settings_centos_7() elseif(VESPA_OS_DISTRO_COMBINED STREQUAL "centos 8") setup_vespa_default_build_settings_centos_8() + elseif(VESPA_OS_DISTRO_COMBINED STREQUAL "rocky 8.4") + setup_vespa_default_build_settings_rocky_8_4() elseif(VESPA_OS_DISTRO STREQUAL "darwin") setup_vespa_default_build_settings_darwin() elseif(VESPA_OS_DISTRO_COMBINED STREQUAL "fedora 32") diff --git a/dist/vespa.spec b/dist/vespa.spec index 13ea9a733e1..7bee7e2a8d0 100644 --- a/dist/vespa.spec +++ b/dist/vespa.spec @@ -27,8 +27,10 @@ License: Commercial URL: http://vespa.ai Source0: vespa-%{version}.tar.gz -%if 0%{?centos} +%if 0%{?centos} || 0%{?rocky} BuildRequires: epel-release +%endif +%if 0%{?centos} %if 0%{?el7} && ! 0%{?amzn2} BuildRequires: centos-release-scl %endif @@ -62,6 +64,7 @@ BuildRequires: maven BuildRequires: pybind11-devel BuildRequires: python3-pytest BuildRequires: python36-devel +BuildRequires: glibc-langpack-en %endif %if 0%{?fedora} BuildRequires: gcc-c++ @@ -69,6 +72,7 @@ BuildRequires: libatomic BuildRequires: pybind11-devel BuildRequires: python3-pytest BuildRequires: python3-devel +BuildRequires: glibc-langpack-en %endif %if 0%{?el7} BuildRequires: cmake3 @@ -92,9 +96,11 @@ BuildRequires: vespa-libzstd-devel >= 1.4.5-2 %endif %if 0%{?el8} BuildRequires: cmake >= 3.11.4-3 +%if 0%{?centos} || 0%{?rocky} %if 0%{?centos} # Current cmake on CentOS 8 is broken and manually requires libarchive install BuildRequires: libarchive +%endif %define _command_cmake cmake BuildRequires: (llvm-devel >= 11.0.0 and llvm-devel < 12) %else @@ -219,7 +225,7 @@ Requires: vespa-valgrind >= 3.17.0-1 %endif %endif %if 0%{?el8} -%if 0%{?centos} +%if 0%{?centos} || 0%{?rocky} %define _vespa_llvm_version 11 %else %define _vespa_llvm_version 10 @@ -290,7 +296,7 @@ Vespa - The open big data serving engine - base Summary: Vespa - The open big data serving engine - base C++ libraries -%if 0%{?centos} +%if 0%{?centos} || 0%{?rocky} Requires: epel-release %endif %if 0%{?amzn2} @@ -307,6 +313,7 @@ Requires: vespa-lz4 >= 1.9.2-2 Requires: vespa-libzstd >= 1.4.5-2 %if 0%{?el8} Requires: openblas +Requires: glibc-langpack-en %else %if 0%{?amzn2} Requires: vespa-openblas = 0.3.15 @@ -319,6 +326,9 @@ Requires: vespa-re2 = 20190801 %else Requires: re2 %endif +%if 0%{?fedora} +Requires: glibc-langpack-en +%endif %description base-libs @@ -347,7 +357,7 @@ Requires: libicu Requires: openssl-libs %endif %if 0%{?el8} -%if 0%{?centos} +%if 0%{?centos} || 0%{?rocky} Requires: (llvm-libs >= 11.0.0 and llvm-libs < 12) %else Requires: (llvm-libs >= 10.0.1 and llvm-libs < 11) diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index 862e75b088c..9f78056d15b 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -206,35 +206,35 @@ public class Flags { public static final UnboundIntFlag MAX_CONCURRENT_MERGES_PER_NODE = defineIntFlag( "max-concurrent-merges-per-node", 16, - List.of("balder", "vekterli"), "2021-06-06", "2021-08-01", + List.of("balder", "vekterli"), "2021-06-06", "2021-09-01", "Specifies max concurrent merges per content node.", "Takes effect at redeploy", ZONE_ID, APPLICATION_ID); public static final UnboundIntFlag MAX_MERGE_QUEUE_SIZE = defineIntFlag( "max-merge-queue-size", 1024, - List.of("balder", "vekterli"), "2021-06-06", "2021-08-01", + List.of("balder", "vekterli"), "2021-06-06", "2021-09-01", "Specifies max size of merge queue.", "Takes effect at redeploy", ZONE_ID, APPLICATION_ID); public static final UnboundBooleanFlag USE_EXTERNAL_RANK_EXPRESSION = defineFeatureFlag( "use-external-rank-expression", false, - List.of("baldersheim"), "2021-05-24", "2021-07-01", + List.of("baldersheim"), "2021-05-24", "2021-09-01", "Whether to use distributed external rank expression or inline in rankproperties", "Takes effect on next internal redeployment", APPLICATION_ID); public static final UnboundBooleanFlag DISTRIBUTE_EXTERNAL_RANK_EXPRESSION = defineFeatureFlag( "distribute-external-rank-expression", false, - List.of("baldersheim"), "2021-05-27", "2021-07-01", + List.of("baldersheim"), "2021-05-27", "2021-09-01", "Whether to use distributed external rank expression files by filedistribution", "Takes effect on next internal redeployment", APPLICATION_ID); public static final UnboundIntFlag LARGE_RANK_EXPRESSION_LIMIT = defineIntFlag( "large-rank-expression-limit", 0x10000, - List.of("baldersheim"), "2021-06-09", "2021-07-01", + List.of("baldersheim"), "2021-06-09", "2021-09-01", "Limit for size of rank expressions distributed by filedistribution", "Takes effect on next internal redeployment", APPLICATION_ID); @@ -281,7 +281,7 @@ public class Flags { public static final UnboundBooleanFlag DRY_RUN_ONNX_ON_SETUP = defineFeatureFlag( "dry-run-onnx-on-setup", false, - List.of("baldersheim"), "2021-06-23", "2021-08-01", + List.of("baldersheim"), "2021-06-23", "2021-09-01", "Whether to dry run onnx models on setup for better error checking", "Takes effect on next internal redeployment", APPLICATION_ID); diff --git a/flags/src/main/java/com/yahoo/vespa/flags/PermanentFlags.java b/flags/src/main/java/com/yahoo/vespa/flags/PermanentFlags.java index d497b9de6ba..cccd75fe7ae 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/PermanentFlags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/PermanentFlags.java @@ -169,6 +169,20 @@ public class PermanentFlags { APPLICATION_ID ); + public static final UnboundDoubleFlag RESOURCE_LIMIT_DISK = defineDoubleFlag( + "resource-limit-disk", 0.8, + "Resource limit (between 0.0 and 1.0) for disk used by cluster controller for when to block feed", + "Takes effect on next deployment", + APPLICATION_ID + ); + + public static final UnboundDoubleFlag RESOURCE_LIMIT_MEMORY = defineDoubleFlag( + "resource-limit-memory", 0.8, + "Resource limit (between 0.0 and 1.0) for memory used by cluster controller for when to block feed", + "Takes effect on next deployment", + APPLICATION_ID + ); + private PermanentFlags() {} private static UnboundBooleanFlag defineFeatureFlag( diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/QuestMetricsDb.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/QuestMetricsDb.java index 8d97e14fc7c..b57b3070d15 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/QuestMetricsDb.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/QuestMetricsDb.java @@ -9,6 +9,7 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.io.IOUtils; import com.yahoo.vespa.defaults.Defaults; +import com.yahoo.yolean.concurrent.ConcurrentResourcePool; import io.questdb.cairo.CairoEngine; import io.questdb.cairo.CairoException; import io.questdb.cairo.DefaultCairoConfiguration; @@ -35,6 +36,7 @@ import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.logging.Level; import java.util.logging.Logger; import java.util.stream.Collectors; @@ -56,9 +58,9 @@ public class QuestMetricsDb extends AbstractComponent implements MetricsDb { private final Clock clock; private final String dataDir; private final CairoEngine engine; - private final ThreadLocal<SqlCompiler> sqlCompiler; + private final ConcurrentResourcePool<SqlCompiler> sqlCompilerPool; + private final AtomicBoolean closed = new AtomicBoolean(false); - private volatile int nullRecords = 0; @Inject public QuestMetricsDb() { @@ -80,12 +82,18 @@ public class QuestMetricsDb extends AbstractComponent implements MetricsDb { this.dataDir = dataDir; engine = new CairoEngine(new DefaultCairoConfiguration(dataDir)); - sqlCompiler = ThreadLocal.withInitial(() -> new SqlCompiler(engine)); + sqlCompilerPool = new ConcurrentResourcePool<>(() -> new SqlCompiler(engine())); nodeTable = new Table(dataDir, "metrics", clock); clusterTable = new Table(dataDir, "clusterMetrics", clock); ensureTablesExist(); } + private CairoEngine engine() { + if (closed.get()) + throw new IllegalStateException("Attempted to access QuestDb after calling close"); + return engine; + } + @Override public Clock clock() { return clock; } @@ -182,11 +190,8 @@ public class QuestMetricsDb extends AbstractComponent implements MetricsDb { } } - public int getNullRecordsCount() { return nullRecords; } - @Override public void gc() { - nullRecords = 0; nodeTable.gc(); clusterTable.gc(); } @@ -196,8 +201,14 @@ public class QuestMetricsDb extends AbstractComponent implements MetricsDb { @Override public void close() { - if (engine != null) - engine.close(); + if (closed.getAndSet(true)) return; + synchronized (nodeTable.writeLock) { + synchronized (clusterTable.writeLock) { + for (SqlCompiler sqlCompiler : sqlCompilerPool) + sqlCompiler.close(); + engine.close(); + } + } } private void ensureTablesExist() { @@ -222,7 +233,7 @@ public class QuestMetricsDb extends AbstractComponent implements MetricsDb { private void ensureClusterTableIsUpdated() { try { - if (0 == engine.getStatus(newContext().getCairoSecurityContext(), new Path(), clusterTable.name)) { + if (0 == engine().getStatus(newContext().getCairoSecurityContext(), new Path(), clusterTable.name)) { // Example: clusterTable.ensureColumnExists("write_rate", "float"); } } catch (Exception e) { @@ -277,10 +288,6 @@ public class QuestMetricsDb extends AbstractComponent implements MetricsDb { try (RecordCursor cursor = factory.getCursor(context)) { Record record = cursor.getRecord(); while (cursor.hasNext()) { - if (record == null || record.getStr(0) == null) { // Observed to happen. QuestDb bug? - nullRecords++; - continue; - } String hostname = record.getStr(0).toString(); if (hostnames.isEmpty() || hostnames.contains(hostname)) { snapshots.put(hostname, @@ -323,11 +330,16 @@ public class QuestMetricsDb extends AbstractComponent implements MetricsDb { /** Issues an SQL statement against the QuestDb engine */ private CompiledQuery issue(String sql, SqlExecutionContext context) throws SqlException { - return sqlCompiler.get().compile(sql, context); + SqlCompiler sqlCompiler = sqlCompilerPool.alloc(); + try { + return sqlCompiler.compile(sql, context); + } finally { + sqlCompilerPool.free(sqlCompiler); + } } private SqlExecutionContext newContext() { - return new SqlExecutionContextImpl(engine, 1); + return new SqlExecutionContextImpl(engine(), 1); } /** A questDb table */ @@ -349,11 +361,11 @@ public class QuestMetricsDb extends AbstractComponent implements MetricsDb { } boolean exists() { - return 0 == engine.getStatus(newContext().getCairoSecurityContext(), new Path(), name); + return 0 == engine().getStatus(newContext().getCairoSecurityContext(), new Path(), name); } TableWriter getWriter() { - return engine.getWriter(newContext().getCairoSecurityContext(), name); + return engine().getWriter(newContext().getCairoSecurityContext(), name); } void gc() { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeMetricsDbMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeMetricsDbMaintainer.java index d671900d08c..4d16c90c002 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeMetricsDbMaintainer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeMetricsDbMaintainer.java @@ -53,9 +53,6 @@ public class NodeMetricsDbMaintainer extends NodeRepositoryMaintainer { Thread.sleep(pauseMs); } - if (nodeRepository().metricsDb().getNullRecordsCount() > 0) - log.warning(nodeRepository().metricsDb().getNullRecordsCount() + " records returned null"); - nodeRepository().metricsDb().gc(); return asSuccessFactor(attempts, failures.get()); diff --git a/slobrok/src/vespa/slobrok/server/cmd.cpp b/slobrok/src/vespa/slobrok/server/cmd.cpp index 99e86814852..60060b9868f 100644 --- a/slobrok/src/vespa/slobrok/server/cmd.cpp +++ b/slobrok/src/vespa/slobrok/server/cmd.cpp @@ -8,7 +8,7 @@ #include "sbenv.h" #include <vespa/log/log.h> -LOG_SETUP(".cmd"); +LOG_SETUP(".slobrok.server.cmd"); namespace slobrok { diff --git a/slobrok/src/vespa/slobrok/server/exchange_manager.cpp b/slobrok/src/vespa/slobrok/server/exchange_manager.cpp index c4a4acd1ced..c7e3c3b32a6 100644 --- a/slobrok/src/vespa/slobrok/server/exchange_manager.cpp +++ b/slobrok/src/vespa/slobrok/server/exchange_manager.cpp @@ -6,7 +6,7 @@ #include <vespa/fnet/frt/supervisor.h> #include <vespa/log/log.h> -LOG_SETUP(".rpcserver"); +LOG_SETUP(".slobrok.server.exchange_manager"); namespace slobrok { diff --git a/slobrok/src/vespa/slobrok/server/managed_rpc_server.cpp b/slobrok/src/vespa/slobrok/server/managed_rpc_server.cpp index 3220a9e5d69..930cdbc962a 100644 --- a/slobrok/src/vespa/slobrok/server/managed_rpc_server.cpp +++ b/slobrok/src/vespa/slobrok/server/managed_rpc_server.cpp @@ -7,7 +7,7 @@ #include <vespa/fnet/frt/target.h> #include <vespa/log/log.h> -LOG_SETUP(".rpcserver"); +LOG_SETUP(".slobrok.server.managed_rpc_server"); namespace slobrok { diff --git a/slobrok/src/vespa/slobrok/server/named_service.cpp b/slobrok/src/vespa/slobrok/server/named_service.cpp index c1aeb83a8a5..9747bbbce52 100644 --- a/slobrok/src/vespa/slobrok/server/named_service.cpp +++ b/slobrok/src/vespa/slobrok/server/named_service.cpp @@ -3,7 +3,7 @@ #include "named_service.h" #include <vespa/log/log.h> -LOG_SETUP(".rpcserver"); +LOG_SETUP(".slobrok.server.named_service"); namespace slobrok { diff --git a/slobrok/src/vespa/slobrok/server/reconfigurable_stateserver.cpp b/slobrok/src/vespa/slobrok/server/reconfigurable_stateserver.cpp index 3bb9841d791..5107b5844c8 100644 --- a/slobrok/src/vespa/slobrok/server/reconfigurable_stateserver.cpp +++ b/slobrok/src/vespa/slobrok/server/reconfigurable_stateserver.cpp @@ -8,7 +8,7 @@ #include <vespa/log/log.h> #include <vespa/config/common/exceptions.h> -LOG_SETUP(".reconfigurable_stateserver"); +LOG_SETUP(".slobrok.server.reconfigurable_stateserver"); using namespace std::chrono_literals; diff --git a/slobrok/src/vespa/slobrok/server/remote_check.cpp b/slobrok/src/vespa/slobrok/server/remote_check.cpp index f08a1646b88..157b959dbfe 100644 --- a/slobrok/src/vespa/slobrok/server/remote_check.cpp +++ b/slobrok/src/vespa/slobrok/server/remote_check.cpp @@ -9,7 +9,7 @@ #include "exchange_manager.h" #include <vespa/log/log.h> -LOG_SETUP(".remcheck"); +LOG_SETUP(".slobrok.server.remote_check"); namespace slobrok { diff --git a/slobrok/src/vespa/slobrok/server/remote_slobrok.cpp b/slobrok/src/vespa/slobrok/server/remote_slobrok.cpp index d50aee21a29..539a901fd9d 100644 --- a/slobrok/src/vespa/slobrok/server/remote_slobrok.cpp +++ b/slobrok/src/vespa/slobrok/server/remote_slobrok.cpp @@ -9,7 +9,7 @@ #include <vespa/fnet/frt/target.h> #include <vespa/log/log.h> -LOG_SETUP(".rpcserver"); +LOG_SETUP(".slobrok.server.remote_slobrok"); namespace slobrok { diff --git a/slobrok/src/vespa/slobrok/server/rpc_server_manager.cpp b/slobrok/src/vespa/slobrok/server/rpc_server_manager.cpp index ead062e9783..dfcdc0ef9d1 100644 --- a/slobrok/src/vespa/slobrok/server/rpc_server_manager.cpp +++ b/slobrok/src/vespa/slobrok/server/rpc_server_manager.cpp @@ -9,7 +9,7 @@ #include <sstream> #include <vespa/log/log.h> -LOG_SETUP(".rpcserver"); +LOG_SETUP(".slobrok.server.rpc_server_manager"); using vespalib::make_string_short::fmt; diff --git a/slobrok/src/vespa/slobrok/server/rpc_server_map.cpp b/slobrok/src/vespa/slobrok/server/rpc_server_map.cpp index 0f0fae61d4c..5f26608c294 100644 --- a/slobrok/src/vespa/slobrok/server/rpc_server_map.cpp +++ b/slobrok/src/vespa/slobrok/server/rpc_server_map.cpp @@ -6,7 +6,7 @@ #include "sbenv.h" #include <vespa/log/log.h> -LOG_SETUP(".rpcsrvmap"); +LOG_SETUP(".slobrok.server.rpc_server_map"); namespace slobrok { diff --git a/slobrok/src/vespa/slobrok/server/rpchooks.cpp b/slobrok/src/vespa/slobrok/server/rpchooks.cpp index 7388e7f7e22..51cd3d65857 100644 --- a/slobrok/src/vespa/slobrok/server/rpchooks.cpp +++ b/slobrok/src/vespa/slobrok/server/rpchooks.cpp @@ -12,7 +12,7 @@ #include <vespa/vespalib/component/vtag.h> #include <vespa/log/log.h> -LOG_SETUP(".rpchooks"); +LOG_SETUP(".slobrok.server.rpchooks"); namespace slobrok { diff --git a/slobrok/src/vespa/slobrok/server/rpcmirror.cpp b/slobrok/src/vespa/slobrok/server/rpcmirror.cpp index 7139dd47838..be533d9b30f 100644 --- a/slobrok/src/vespa/slobrok/server/rpcmirror.cpp +++ b/slobrok/src/vespa/slobrok/server/rpcmirror.cpp @@ -5,7 +5,7 @@ #include <vespa/fnet/frt/rpcrequest.h> #include <vespa/log/log.h> -LOG_SETUP(".rpcmirror"); +LOG_SETUP(".slobrok.server.rpcmirror"); namespace slobrok { diff --git a/slobrok/src/vespa/slobrok/server/sbenv.cpp b/slobrok/src/vespa/slobrok/server/sbenv.cpp index 525ba0c0aa7..2bf8e57cb26 100644 --- a/slobrok/src/vespa/slobrok/server/sbenv.cpp +++ b/slobrok/src/vespa/slobrok/server/sbenv.cpp @@ -13,7 +13,7 @@ #include <sstream> #include <vespa/log/log.h> -LOG_SETUP(".sbenv"); +LOG_SETUP(".slobrok.server.sbenv"); using namespace std::chrono_literals; diff --git a/slobrok/src/vespa/slobrok/server/service_map_history.cpp b/slobrok/src/vespa/slobrok/server/service_map_history.cpp index f9de736c093..3882a41bde5 100644 --- a/slobrok/src/vespa/slobrok/server/service_map_history.cpp +++ b/slobrok/src/vespa/slobrok/server/service_map_history.cpp @@ -3,7 +3,7 @@ #include "service_map_history.h" #include <vespa/log/log.h> -LOG_SETUP(".slobrok.publisher"); +LOG_SETUP(".slobrok.server.service_map_history"); namespace slobrok { diff --git a/vespalib/src/vespa/vespalib/data/slime/binary_format.cpp b/vespalib/src/vespa/vespalib/data/slime/binary_format.cpp index 1956dde689c..2eb6173b1e5 100644 --- a/vespalib/src/vespa/vespalib/data/slime/binary_format.cpp +++ b/vespalib/src/vespa/vespalib/data/slime/binary_format.cpp @@ -8,6 +8,25 @@ LOG_SETUP(".vespalib.data.slime.binary_format"); namespace vespalib::slime::binary_format { +namespace { + +void +write_cmpr_ulong_impl(OutputWriter &out, uint64_t value) { + out.commit(encode_cmpr_ulong(out.reserve(10), value)); +} + +void +write_type_and_size_impl(OutputWriter &out, uint32_t type, uint64_t size) { + char *start = out.reserve(11); // max size + char *pos = start; + if (size <= 30) { + *pos++ = encode_type_and_meta(type, size + 1); + } else { + *pos++ = encode_type_and_meta(type, 0); + pos += encode_cmpr_ulong(pos, size); + } + out.commit(pos - start); +} struct BinaryEncoder : public ArrayTraverser, public ObjectSymbolTraverser @@ -27,21 +46,21 @@ struct BinaryEncoder : public ArrayTraverser, write_type_and_bytes<true>(out, DOUBLE::ID, encode_double(value)); } void encodeString(const Memory &memory) { - write_type_and_size(out, STRING::ID, memory.size); + write_type_and_size_impl(out, STRING::ID, memory.size); out.write(memory.data, memory.size); } void encodeData(const Memory &memory) { - write_type_and_size(out, DATA::ID, memory.size); + write_type_and_size_impl(out, DATA::ID, memory.size); out.write(memory.data, memory.size); } void encodeArray(const Inspector &inspector) { ArrayTraverser &array_traverser = *this; - write_type_and_size(out, ARRAY::ID, inspector.children()); + write_type_and_size_impl(out, ARRAY::ID, inspector.children()); inspector.traverse(array_traverser); } void encodeObject(const Inspector &inspector) { ObjectSymbolTraverser &object_traverser = *this; - write_type_and_size(out, OBJECT::ID, inspector.children()); + write_type_and_size_impl(out, OBJECT::ID, inspector.children()); inspector.traverse(object_traverser); } void encodeValue(const Inspector &inspector) { @@ -59,10 +78,10 @@ struct BinaryEncoder : public ArrayTraverser, } void encodeSymbolTable(const Slime &slime) { size_t numSymbols = slime.symbols(); - write_cmpr_ulong(out, numSymbols); + write_cmpr_ulong_impl(out, numSymbols); for (size_t i = 0; i < numSymbols; ++i) { Memory image = slime.inspect(Symbol(i)); - write_cmpr_ulong(out, image.size); + write_cmpr_ulong_impl(out, image.size); out.write(image.data, image.size); } } @@ -79,7 +98,7 @@ BinaryEncoder::entry(size_t, const Inspector &inspector) void BinaryEncoder::field(const Symbol &symbol, const Inspector &inspector) { - write_cmpr_ulong(out, symbol.getValue()); + write_cmpr_ulong_impl(out, symbol.getValue()); encodeValue(inspector); } @@ -108,8 +127,8 @@ struct MappedSymbols { } Symbol map_symbol(Symbol symbol) const { return (symbol.getValue() < symbol_mapping.size()) - ? symbol_mapping[symbol.getValue()] - : symbol; + ? symbol_mapping[symbol.getValue()] + : symbol; } }; @@ -175,9 +194,7 @@ struct BinaryDecoder : SymbolHandler<remap_symbols>::type { void decodeValue(const Inserter &inserter) { char byte = in.read(); - Cursor &cursor = decodeValue(inserter, - decode_type(byte), - decode_meta(byte)); + Cursor &cursor = decodeValue(inserter, decode_type(byte), decode_meta(byte)); if (!cursor.valid()) { in.fail("failed to decode value"); } @@ -239,22 +256,16 @@ size_t decode(const Memory &memory, Slime &slime, const Inserter &inserter) { return input.failed() ? 0 : input.get_offset(); } +} + void write_cmpr_ulong(OutputWriter &out, uint64_t value) { - out.commit(encode_cmpr_ulong(out.reserve(10), value)); + write_cmpr_ulong_impl(out, value); } void write_type_and_size(OutputWriter &out, uint32_t type, uint64_t size) { - char *start = out.reserve(11); // max size - char *pos = start; - if (size <= 30) { - *pos++ = encode_type_and_meta(type, size + 1); - } else { - *pos++ = encode_type_and_meta(type, 0); - pos += encode_cmpr_ulong(pos, size); - } - out.commit(pos - start); + write_type_and_size_impl(out, type, size); } } diff --git a/yolean/abi-spec.json b/yolean/abi-spec.json index 82bf59ebf87..85aaaf5f64e 100644 --- a/yolean/abi-spec.json +++ b/yolean/abi-spec.json @@ -169,6 +169,7 @@ ], "methods": [ "public void <init>(com.yahoo.yolean.concurrent.ResourceFactory)", + "public void <init>(java.util.function.Supplier)", "public final java.lang.Object alloc()", "public final void free(java.lang.Object)", "public java.util.Iterator iterator()" @@ -209,7 +210,8 @@ ], "methods": [ "public void <init>()", - "public abstract java.lang.Object create()" + "public abstract java.lang.Object create()", + "public final java.util.function.Supplier asSupplier()" ], "fields": [] }, diff --git a/yolean/src/main/java/com/yahoo/yolean/concurrent/ConcurrentResourcePool.java b/yolean/src/main/java/com/yahoo/yolean/concurrent/ConcurrentResourcePool.java index 24d4cfe4318..a00de4866d0 100644 --- a/yolean/src/main/java/com/yahoo/yolean/concurrent/ConcurrentResourcePool.java +++ b/yolean/src/main/java/com/yahoo/yolean/concurrent/ConcurrentResourcePool.java @@ -4,24 +4,41 @@ package com.yahoo.yolean.concurrent; import java.util.Iterator; import java.util.Queue; import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.function.Supplier; /** + * A pool of a resource. This create new instances of the resource on request until enough are created + * to deliver a unique one to all threads needing one concurrently and then reuse those instances + * in subsequent requests. + * * @author baldersheim */ public class ConcurrentResourcePool<T> implements Iterable<T> { private final Queue<T> pool = new ConcurrentLinkedQueue<>(); - private final ResourceFactory<T> factory; + private final Supplier<T> factory; + // TODO: Deprecate public ConcurrentResourcePool(ResourceFactory<T> factory) { + this.factory = factory.asSupplier(); + } + + public ConcurrentResourcePool(Supplier<T> factory) { this.factory = factory; } + /** + * Allocates an instance of the resource to the requestor. + * The resource will be allocated exclusively to the requestor until it calls free(instance). + * + * @return a reused or newly created instance of the resource + */ public final T alloc() { - final T e = pool.poll(); - return e != null ? e : factory.create(); + T e = pool.poll(); + return e != null ? e : factory.get(); } + /** Frees an instance previously acquired bty alloc */ public final void free(T e) { pool.offer(e); } @@ -30,4 +47,5 @@ public class ConcurrentResourcePool<T> implements Iterable<T> { public Iterator<T> iterator() { return pool.iterator(); } + } diff --git a/yolean/src/main/java/com/yahoo/yolean/concurrent/ResourceFactory.java b/yolean/src/main/java/com/yahoo/yolean/concurrent/ResourceFactory.java index f926283a47f..3a99b189ed8 100644 --- a/yolean/src/main/java/com/yahoo/yolean/concurrent/ResourceFactory.java +++ b/yolean/src/main/java/com/yahoo/yolean/concurrent/ResourceFactory.java @@ -1,11 +1,18 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.yolean.concurrent; +import java.util.function.Supplier; + /** * @author baldersheim - * @since 5.2 */ +// TODO: Deprecate public abstract class ResourceFactory<T> { public abstract T create(); + + public final Supplier<T> asSupplier() { + return () -> create(); + } + } |