diff options
23 files changed, 80 insertions, 100 deletions
diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/rpc/SlobrokClient.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/rpc/SlobrokClient.java index e180016f286..b4e9a760d8e 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/rpc/SlobrokClient.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/rpc/SlobrokClient.java @@ -25,7 +25,7 @@ import java.util.logging.Logger; public class SlobrokClient implements NodeLookup { - public static Logger log = Logger.getLogger(SlobrokClient.class.getName()); + public static final Logger log = Logger.getLogger(SlobrokClient.class.getName()); private final Timer timer; private String[] connectionSpecs; diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java index abca424a838..42ef763fc62 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java @@ -74,20 +74,16 @@ public interface ModelContext { @ModelFeatureFlag(owners = {"baldersheim"}, comment = "Select sequencer type use while feeding") default String feedSequencerType() { throw new UnsupportedOperationException("TODO specify default value"); } @ModelFeatureFlag(owners = {"baldersheim"}) default String responseSequencerType() { throw new UnsupportedOperationException("TODO specify default value"); } @ModelFeatureFlag(owners = {"baldersheim"}) default int defaultNumResponseThreads() { return 2; } - @ModelFeatureFlag(owners = {"baldersheim"}) default int maxPendingMoveOps() { return 100; } @ModelFeatureFlag(owners = {"baldersheim"}) default boolean skipCommunicationManagerThread() { throw new UnsupportedOperationException("TODO specify default value"); } @ModelFeatureFlag(owners = {"baldersheim"}) default boolean skipMbusRequestThread() { throw new UnsupportedOperationException("TODO specify default value"); } @ModelFeatureFlag(owners = {"baldersheim"}) default boolean skipMbusReplyThread() { throw new UnsupportedOperationException("TODO specify default value"); } @ModelFeatureFlag(owners = {"tokle"}) default boolean useAccessControlTlsHandshakeClientAuth() { return true; } @ModelFeatureFlag(owners = {"baldersheim"}) default boolean useAsyncMessageHandlingOnSchedule() { throw new UnsupportedOperationException("TODO specify default value"); } @ModelFeatureFlag(owners = {"baldersheim"}) default double feedConcurrency() { throw new UnsupportedOperationException("TODO specify default value"); } - @ModelFeatureFlag(owners = {"baldersheim"}) default boolean useBucketExecutorForLidSpaceCompact() { return true; } - @ModelFeatureFlag(owners = {"baldersheim"}) default boolean useBucketExecutorForBucketMove() { return true; } - @ModelFeatureFlag(owners = {"baldersheim"}) default boolean useBucketExecutorForPruneRemoved() { throw new UnsupportedOperationException("TODO specify default value"); } + @ModelFeatureFlag(owners = {"baldersheim"}) default boolean useBucketExecutorForPruneRemoved() { return true; } @ModelFeatureFlag(owners = {"baldersheim"}) default boolean useExternalRankExpressions() { return false; } @ModelFeatureFlag(owners = {"baldersheim"}) default boolean distributeExternalRankExpressions() { return false; } @ModelFeatureFlag(owners = {"geirst"}) default boolean enableFeedBlockInDistributor() { return true; } - @ModelFeatureFlag(owners = {"baldersheim", "geirst", "toregge"}) default double maxDeadBytesRatio() { return 0.05; } @ModelFeatureFlag(owners = {"hmusum"}, removeAfter = "7.406") default int clusterControllerMaxHeapSizeInMb() { return 128; } @ModelFeatureFlag(owners = {"hmusum"}) default int metricsProxyMaxHeapSizeInMb(ClusterSpec.Type type) { return 256; } @ModelFeatureFlag(owners = {"bjorncs", "tokle"}) default List<String> allowedAthenzProxyIdentities() { return List.of(); } diff --git a/config-model/src/main/java/com/yahoo/config/model/ApplicationConfigProducerRoot.java b/config-model/src/main/java/com/yahoo/config/model/ApplicationConfigProducerRoot.java index 061ad42e028..364dd1742ae 100644 --- a/config-model/src/main/java/com/yahoo/config/model/ApplicationConfigProducerRoot.java +++ b/config-model/src/main/java/com/yahoo/config/model/ApplicationConfigProducerRoot.java @@ -65,8 +65,8 @@ public class ApplicationConfigProducerRoot extends AbstractConfigProducer<Abstra * Creates and initializes a new Vespa from the service config file * in the given application directory. * - * @param parent The parent, usually VespaModel - * @param name The name, used as configId + * @param parent the parent, usually VespaModel + * @param name the name, used as configId * @param documentModel DocumentModel to serve global document config from. */ public ApplicationConfigProducerRoot(AbstractConfigProducer parent, String name, DocumentModel documentModel, Version vespaVersion, ApplicationId applicationId) { diff --git a/config-model/src/main/java/com/yahoo/config/model/ConfigModelRepo.java b/config-model/src/main/java/com/yahoo/config/model/ConfigModelRepo.java index 557df4bd106..100b5ec5a24 100644 --- a/config-model/src/main/java/com/yahoo/config/model/ConfigModelRepo.java +++ b/config-model/src/main/java/com/yahoo/config/model/ConfigModelRepo.java @@ -11,6 +11,16 @@ import com.yahoo.config.model.builder.xml.XmlHelper; import com.yahoo.config.model.graph.ModelGraphBuilder; import com.yahoo.config.model.graph.ModelNode; import com.yahoo.config.model.provision.HostsXmlProvisioner; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.TreeMap; import java.util.logging.Level; import com.yahoo.path.Path; import com.yahoo.text.XML; @@ -29,7 +39,6 @@ import java.io.IOException; import java.io.Reader; import java.io.Serializable; import java.io.StringReader; -import java.util.*; import java.util.logging.Logger; /** diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java index 5dd6ffe7247..304855e545d 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java @@ -51,7 +51,6 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea private boolean useAccessControlTlsHandshakeClientAuth; private boolean useAsyncMessageHandlingOnSchedule = false; private double feedConcurrency = 0.5; - private boolean useBucketExecutorForPruneRemoved; private boolean enableFeedBlockInDistributor = true; private boolean useExternalRankExpression = false; private int clusterControllerMaxHeapSizeInMb = 128; @@ -92,7 +91,6 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea @Override public boolean useAccessControlTlsHandshakeClientAuth() { return useAccessControlTlsHandshakeClientAuth; } @Override public boolean useAsyncMessageHandlingOnSchedule() { return useAsyncMessageHandlingOnSchedule; } @Override public double feedConcurrency() { return feedConcurrency; } - @Override public boolean useBucketExecutorForPruneRemoved() { return useBucketExecutorForPruneRemoved; } @Override public boolean enableFeedBlockInDistributor() { return enableFeedBlockInDistributor; } @Override public int clusterControllerMaxHeapSizeInMb() { return clusterControllerMaxHeapSizeInMb; } @Override public int metricsProxyMaxHeapSizeInMb(ClusterSpec.Type type) { return metricsProxyMaxHeapSizeInMb; } @@ -201,11 +199,6 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea return this; } - public TestProperties useBucketExecutorForPruneRemoved(boolean enabled) { - useBucketExecutorForPruneRemoved = enabled; - return this; - } - public TestProperties enableFeedBlockInDistributor(boolean enabled) { enableFeedBlockInDistributor = enabled; return this; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java index af40dc947c8..7e5b52f6af1 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java @@ -807,7 +807,6 @@ public class RankProfile implements Cloneable { // Function compiling second pass: compile all functions and insert previously compiled inline functions // TODO This merges all functions from inherited profiles too and erases inheritance information. Not good. functions = compileFunctions(this::getFunctions, queryProfiles, featureTypes, importedModels, inlineFunctions, expressionTransforms); - } private void checkNameCollisions(Map<String, RankingExpressionFunction> functions, Map<String, Value> constants) { @@ -843,8 +842,8 @@ public class RankProfile implements Cloneable { return compiledFunctions; } - private Map.Entry<String, RankingExpressionFunction> findUncompiledFunction(Map<String, RankingExpressionFunction> functions, - Set<String> compiledFunctionNames) { + private static Map.Entry<String, RankingExpressionFunction> findUncompiledFunction(Map<String, RankingExpressionFunction> functions, + Set<String> compiledFunctionNames) { for (Map.Entry<String, RankingExpressionFunction> entry : functions.entrySet()) { if ( ! compiledFunctionNames.contains(entry.getKey())) return entry; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java index 18580249ddc..51949e78838 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java @@ -65,7 +65,6 @@ public class ContentSearchCluster extends AbstractConfigProducer<SearchCluster> private Optional<ResourceLimits> resourceLimits = Optional.empty(); private final ProtonConfig.Indexing.Optimize.Enum feedSequencerType; private final double defaultFeedConcurrency; - private final boolean useBucketExecutorForPruneRemoved; /** Whether the nodes of this cluster also hosts a container cluster in a hosted system */ private final boolean combined; @@ -210,7 +209,6 @@ public class ContentSearchCluster extends AbstractConfigProducer<SearchCluster> this.combined = combined; feedSequencerType = convertFeedSequencerType(featureFlags.feedSequencerType()); defaultFeedConcurrency = featureFlags.feedConcurrency(); - useBucketExecutorForPruneRemoved = featureFlags.useBucketExecutorForPruneRemoved(); } public void setVisibilityDelay(double delay) { @@ -427,7 +425,6 @@ public class ContentSearchCluster extends AbstractConfigProducer<SearchCluster> } else { builder.indexing.optimize(feedSequencerType); } - builder.pruneremoveddocuments.usebucketexecutor(useBucketExecutorForPruneRemoved); } private boolean isGloballyDistributed(NewDocumentType docType) { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java index ad4603e5c6b..7bff4890b7e 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java @@ -824,24 +824,6 @@ public class ContentBuilderTest extends DomBuilderTest { verifyThatFeatureFlagControlsVisibilityDelayDefault(0.6, 0.6); } - private void verifyThatFeatureFlagControlsUseBucketExecutorForPruneRemoved(boolean flag) { - DeployState.Builder deployStateBuilder = new DeployState.Builder().properties(new TestProperties().useBucketExecutorForPruneRemoved(flag)); - VespaModel model = new VespaModelCreatorWithMockPkg(new MockApplicationPackage.Builder() - .withServices(singleNodeContentXml()) - .withSearchDefinition(MockApplicationPackage.MUSIC_SEARCHDEFINITION) - .build()) - .create(deployStateBuilder); - ProtonConfig config = getProtonConfig(model.getContentClusters().values().iterator().next()); - assertEquals(flag, config.pruneremoveddocuments().usebucketexecutor()); - } - - - @Test - public void verifyUseBucketExecutorForPruneRemoved() { - verifyThatFeatureFlagControlsUseBucketExecutorForPruneRemoved(true); - verifyThatFeatureFlagControlsUseBucketExecutorForPruneRemoved(false); - } - @Test public void failWhenNoDocumentsElementSpecified() { expectedException.expect(IllegalArgumentException.class); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java index 1a635d5236a..d2fb5fd6f4b 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java @@ -168,7 +168,6 @@ public class ModelContextImpl implements ModelContext { private final boolean skipMbusReplyThread; private final boolean useAsyncMessageHandlingOnSchedule; private final double feedConcurrency; - private final boolean useBucketExecutorForPruneRemoved; private final boolean enableFeedBlockInDistributor; private final ToIntFunction<ClusterSpec.Type> metricsProxyMaxHeapSizeInMb; private final List<String> allowedAthenzProxyIdentities; @@ -192,7 +191,6 @@ public class ModelContextImpl implements ModelContext { this.skipMbusReplyThread = flagValue(source, appId, Flags.SKIP_MBUS_REPLY_THREAD); this.useAsyncMessageHandlingOnSchedule = flagValue(source, appId, Flags.USE_ASYNC_MESSAGE_HANDLING_ON_SCHEDULE); this.feedConcurrency = flagValue(source, appId, Flags.FEED_CONCURRENCY); - this.useBucketExecutorForPruneRemoved = flagValue(source, appId, Flags.USE_BUCKET_EXECUTOR_FOR_PRUNE_REMOVED); this.enableFeedBlockInDistributor = flagValue(source, appId, Flags.ENABLE_FEED_BLOCK_IN_DISTRIBUTOR); this.metricsProxyMaxHeapSizeInMb = type -> Flags.METRICS_PROXY_MAX_HEAP_SIZE_IN_MB.bindTo(source).with(CLUSTER_TYPE, type.name()).value(); this.allowedAthenzProxyIdentities = flagValue(source, appId, Flags.ALLOWED_ATHENZ_PROXY_IDENTITIES); @@ -216,7 +214,6 @@ public class ModelContextImpl implements ModelContext { @Override public boolean skipMbusReplyThread() { return skipMbusReplyThread; } @Override public boolean useAsyncMessageHandlingOnSchedule() { return useAsyncMessageHandlingOnSchedule; } @Override public double feedConcurrency() { return feedConcurrency; } - @Override public boolean useBucketExecutorForPruneRemoved() { return useBucketExecutorForPruneRemoved; } @Override public boolean enableFeedBlockInDistributor() { return enableFeedBlockInDistributor; } @Override public int metricsProxyMaxHeapSizeInMb(ClusterSpec.Type type) { return metricsProxyMaxHeapSizeInMb.applyAsInt(type); } @Override public List<String> allowedAthenzProxyIdentities() { return allowedAthenzProxyIdentities; } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java index 5fe1c70028b..c7270b6c426 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java @@ -182,7 +182,7 @@ public class InternalStepRunner implements StepRunner { private Optional<RunStatus> deployReal(RunId id, DualLogger logger) { Versions versions = controller.jobController().run(id).get().versions(); logger.log("Deploying platform version " + versions.targetPlatform() + - " and application version " + versions.targetApplication().id() + " ..."); + " and application version " + versions.targetApplication().id() + " ..."); return deployReal(id, false, logger); } @@ -239,7 +239,7 @@ public class InternalStepRunner implements StepRunner { case ACTIVATION_CONFLICT: case APPLICATION_LOCK_FAILURE: logger.log("Deployment failed with possibly transient error " + e.code() + - ", will retry: " + e.getMessage()); + ", will retry: " + e.getMessage()); return result; case LOAD_BALANCER_NOT_READY: case PARENT_HOST_NOT_READY: diff --git a/dist/vespa.spec b/dist/vespa.spec index de92077fdbd..34afc46c80c 100644 --- a/dist/vespa.spec +++ b/dist/vespa.spec @@ -724,9 +724,12 @@ fi %defattr(-,%{_vespa_user},%{_vespa_group},-) %endif %dir %{_prefix} +%dir %{_prefix}/conf +%dir %{_prefix}/conf/vespa-feed-client %dir %{_prefix}/lib %dir %{_prefix}/lib/jars %{_prefix}/bin/vespa-feed-client +%{_prefix}/conf/vespa-feed-client/logging.properties %{_prefix}/lib/jars/vespa-http-client-jar-with-dependencies.jar %{_prefix}/lib/jars/vespa-feed-client-cli.jar diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index aeec579183d..03e2b2fcb1f 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -143,13 +143,6 @@ public class Flags { "Takes effect at redeployment", ZONE_ID, APPLICATION_ID); - public static final UnboundBooleanFlag USE_BUCKET_EXECUTOR_FOR_PRUNE_REMOVED = defineFeatureFlag( - "use-bucket-executor-for-prune-removed", true, - List.of("baldersheim"), "2021-05-04", "2021-06-01", - "Wheter to use content-level bucket executor or legacy frozen buckets for prune removed", - "Takes effect on next internal redeployment", - APPLICATION_ID); - public static final UnboundBooleanFlag GROUP_SUSPENSION = defineFeatureFlag( "group-suspension", true, List.of("hakon"), "2021-01-22", "2021-06-22", diff --git a/jrt/src/com/yahoo/jrt/slobrok/api/Mirror.java b/jrt/src/com/yahoo/jrt/slobrok/api/Mirror.java index 996459dc5db..058a1380480 100644 --- a/jrt/src/com/yahoo/jrt/slobrok/api/Mirror.java +++ b/jrt/src/com/yahoo/jrt/slobrok/api/Mirror.java @@ -31,16 +31,16 @@ import java.util.logging.Level; */ public class Mirror implements IMirror { - private static Logger log = Logger.getLogger(Mirror.class.getName()); + private static final Logger log = Logger.getLogger(Mirror.class.getName()); private final Supervisor orb; private final SlobrokList slobroks; - private String currSlobrok; - private final BackOffPolicy backOff; - private volatile int updates = 0; + private String currSlobrok; + private final BackOffPolicy backOff; + private volatile int updates = 0; private boolean requestDone = false; private boolean logOnSuccess = true; - private AtomicReference<Entry[]> specs = new AtomicReference<>(new Entry[0]); + private final AtomicReference<Entry[]> specs = new AtomicReference<>(new Entry[0]); private int specsGeneration = 0; private final TransportThread transportThread; private final Task updateTask; @@ -55,7 +55,7 @@ public class Mirror implements IMirror { * @param orb the Supervisor to use * @param slobroks slobrok connect spec list * @param bop custom backoff policy, mostly useful for testing - **/ + */ public Mirror(Supervisor orb, SlobrokList slobroks, BackOffPolicy bop) { this.orb = orb; this.slobroks = slobroks; diff --git a/jrt/src/com/yahoo/jrt/slobrok/api/SlobrokList.java b/jrt/src/com/yahoo/jrt/slobrok/api/SlobrokList.java index 10d8923d9f5..654ccd7e350 100644 --- a/jrt/src/com/yahoo/jrt/slobrok/api/SlobrokList.java +++ b/jrt/src/com/yahoo/jrt/slobrok/api/SlobrokList.java @@ -27,7 +27,6 @@ public class SlobrokList { } } - public String nextSlobrokSpec() { checkUpdate(); if (idx < slobroks.length) { @@ -90,4 +89,5 @@ public class SlobrokList { return Arrays.toString(slobroks); } } + } diff --git a/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/SlobrokConfigSubscriber.java b/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/SlobrokConfigSubscriber.java index 7f4c27a45f9..d576ec50af7 100755 --- a/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/SlobrokConfigSubscriber.java +++ b/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/SlobrokConfigSubscriber.java @@ -12,13 +12,13 @@ import com.yahoo.cloud.config.SlobroksConfig; */ public class SlobrokConfigSubscriber implements ConfigSubscriber.SingleSubscriber<SlobroksConfig>{ - private SlobrokList slobroks = new SlobrokList(); + private final SlobrokList slobroks = new SlobrokList(); private ConfigSubscriber subscriber; /** * Constructs a new config subscriber for a given config id. * - * @param configId The id of the config to subscribe to. + * @param configId the id of the config to subscribe to */ public SlobrokConfigSubscriber(String configId) { subscriber = new ConfigSubscriber(); @@ -55,4 +55,5 @@ public class SlobrokConfigSubscriber implements ConfigSubscriber.SingleSubscribe subscriber.close(); } } + } diff --git a/storage/src/tests/distributor/mergeoperationtest.cpp b/storage/src/tests/distributor/mergeoperationtest.cpp index 7a90299bfb3..1026ea2855e 100644 --- a/storage/src/tests/distributor/mergeoperationtest.cpp +++ b/storage/src/tests/distributor/mergeoperationtest.cpp @@ -315,7 +315,7 @@ TEST_F(MergeOperationTest, allow_deleting_active_source_only_replica) { _sender.getLastCommand(true)); } -TEST_F(MergeOperationTest, MarkRedundantTrustedCopiesAsSourceOnly) { +TEST_F(MergeOperationTest, mark_redundant_trusted_copies_as_source_only) { // This test uses the same distribution as testGenerateNodeList(), i.e. // an ideal state sequence of [3, 5, 7, 6, 8, 0, 9, 2, 1, 4] @@ -415,6 +415,21 @@ TEST_F(MergeOperationTest, merge_operation_is_blocked_by_any_busy_target_node) { EXPECT_TRUE(op.isBlocked(*_pendingTracker, _operation_sequencer)); } + +TEST_F(MergeOperationTest, global_bucket_merges_are_not_blocked_by_busy_nodes) { + getClock().setAbsoluteTimeInSeconds(10); + document::BucketId bucket_id(16, 1); + addNodesToBucketDB(bucket_id, "0=10/1/1/t,1=20/1/1,2=10/1/1/t"); + enableDistributorClusterState("distributor:1 storage:3"); + document::Bucket global_bucket(document::FixedBucketSpaces::global_space(), bucket_id); + MergeOperation op(BucketAndNodes(global_bucket, toVector<uint16_t>(0, 1, 2))); + op.setIdealStateManager(&getIdealStateManager()); + + // Node 1 is included in operation node set but should not cause a block of global bucket merge + _pendingTracker->getNodeInfo().setBusy(0, std::chrono::seconds(10)); + EXPECT_FALSE(op.isBlocked(*_pendingTracker, _operation_sequencer)); +} + TEST_F(MergeOperationTest, merge_operation_is_blocked_by_locked_bucket) { getClock().setAbsoluteTimeInSeconds(10); addNodesToBucketDB(document::BucketId(16, 1), "0=10/1/1/t,1=20/1/1,2=10/1/1/t"); diff --git a/storage/src/vespa/storage/distributor/operations/idealstate/idealstateoperation.cpp b/storage/src/vespa/storage/distributor/operations/idealstate/idealstateoperation.cpp index 142ff72bc79..1a48df0fd7c 100644 --- a/storage/src/vespa/storage/distributor/operations/idealstate/idealstateoperation.cpp +++ b/storage/src/vespa/storage/distributor/operations/idealstate/idealstateoperation.cpp @@ -9,8 +9,8 @@ #include <vespa/log/log.h> LOG_SETUP(".distributor.operation"); -using namespace storage; -using namespace storage::distributor; +namespace storage::distributor { + using document::BucketSpace; const uint32_t IdealStateOperation::MAINTENANCE_MESSAGE_TYPES[] = @@ -85,7 +85,7 @@ IdealStateOperation::setIdealStateManager(IdealStateManager* manager) { void IdealStateOperation::done() { - if (_manager != NULL) { + if (_manager) { if (ok()) { _manager->getMetrics().operations[getType()]->ok.inc(1); } else { @@ -107,35 +107,6 @@ IdealStateOperation::setCommandMeta(api::MaintenanceCommand& cmd) const cmd.setReason(_detailedReason); } -std::string -IdealStateOperation::toXML(framework::Clock& clock) const -{ - std::ostringstream ost; - - ost << "<operation bucketid=\"" << getBucketId() - << "\" reason=\"" << _detailedReason << "\" operations=\""; - - ost << getName() << "["; - for (uint32_t j = 0; j < getNodes().size(); j++) { - if (j != 0) { - ost << ","; - } - ost << getNodes()[j]; - } - ost << "]"; - - if (getStartTime().isSet()) { - uint64_t timeSpent( - (clock.getTimeInMillis() - getStartTime()).getTime()); - ost << "\" runtime_secs=\"" << timeSpent << "\""; - } else { - ost << "\""; - } - - ost << "/>"; - return ost.str(); -} - namespace { class IdealStateOpChecker : public PendingMessageTracker::Checker @@ -277,3 +248,5 @@ IdealStateOperation::shouldBlockThisOperation(uint32_t messageType, return false; } + +} diff --git a/storage/src/vespa/storage/distributor/operations/idealstate/idealstateoperation.h b/storage/src/vespa/storage/distributor/operations/idealstate/idealstateoperation.h index 7906150d0cb..0e45d7f3b3a 100644 --- a/storage/src/vespa/storage/distributor/operations/idealstate/idealstateoperation.h +++ b/storage/src/vespa/storage/distributor/operations/idealstate/idealstateoperation.h @@ -208,8 +208,6 @@ public: */ void setCommandMeta(api::MaintenanceCommand& cmd) const; - std::string toXML(framework::Clock& clock) const; - std::string toString() const override; /** diff --git a/storage/src/vespa/storage/distributor/operations/idealstate/mergeoperation.cpp b/storage/src/vespa/storage/distributor/operations/idealstate/mergeoperation.cpp index 481506096eb..27e203a9060 100644 --- a/storage/src/vespa/storage/distributor/operations/idealstate/mergeoperation.cpp +++ b/storage/src/vespa/storage/distributor/operations/idealstate/mergeoperation.cpp @@ -1,5 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "mergeoperation.h" +#include <vespa/document/bucket/fixed_bucket_spaces.h> #include <vespa/storage/distributor/idealstatemanager.h> #include <vespa/storage/distributor/distributor_bucket_space.h> #include <vespa/storage/distributor/pendingmessagetracker.h> @@ -325,13 +326,30 @@ bool MergeOperation::shouldBlockThisOperation(uint32_t messageType, uint8_t pri) bool MergeOperation::isBlocked(const PendingMessageTracker& pending_tracker, const OperationSequencer& op_seq) const { - const auto& node_info = pending_tracker.getNodeInfo(); - for (auto node : getNodes()) { - if (node_info.isBusy(node)) { - return true; + // To avoid starvation of high priority global bucket merges, we do not consider + // these for blocking due to a node being "busy" (usually caused by a full merge + // throttler queue). + // + // This is for two reasons: + // 1. When an ideal state op is blocked, it is still removed from the internal + // maintenance priority queue. This means a blocked high pri operation will + // not be retried until the next DB pass (at which point the node is likely + // to still be marked as busy when there's heavy merge traffic). + // 2. Global bucket merges have high priority and will most likely be allowed + // to enter the merge throttler queues, displacing lower priority merges. + if (!is_global_bucket_merge()) { + const auto& node_info = pending_tracker.getNodeInfo(); + for (auto node : getNodes()) { + if (node_info.isBusy(node)) { + return true; + } } } return IdealStateOperation::isBlocked(pending_tracker, op_seq); } +bool MergeOperation::is_global_bucket_merge() const noexcept { + return getBucket().getBucketSpace() == document::FixedBucketSpaces::global_space(); +} + } diff --git a/storage/src/vespa/storage/distributor/operations/idealstate/mergeoperation.h b/storage/src/vespa/storage/distributor/operations/idealstate/mergeoperation.h index 5df9421e815..11b5494fd9b 100644 --- a/storage/src/vespa/storage/distributor/operations/idealstate/mergeoperation.h +++ b/storage/src/vespa/storage/distributor/operations/idealstate/mergeoperation.h @@ -61,6 +61,7 @@ private: void deleteSourceOnlyNodes(const BucketDatabase::Entry& currentState, DistributorStripeMessageSender& sender); + bool is_global_bucket_merge() const noexcept; }; } diff --git a/vespa-feed-client-cli/CMakeLists.txt b/vespa-feed-client-cli/CMakeLists.txt index a918981dcd3..3967c135d1c 100644 --- a/vespa-feed-client-cli/CMakeLists.txt +++ b/vespa-feed-client-cli/CMakeLists.txt @@ -2,3 +2,4 @@ install_java_artifact(vespa-feed-client-cli) vespa_install_script(src/main/sh/vespa-feed-client.sh vespa-feed-client bin) +install(FILES src/main/resources/logging.properties DESTINATION conf/vespa-feed-client) diff --git a/vespa-feed-client-cli/src/main/resources/logging.properties b/vespa-feed-client-cli/src/main/resources/logging.properties new file mode 100644 index 00000000000..3f0e8b24e78 --- /dev/null +++ b/vespa-feed-client-cli/src/main/resources/logging.properties @@ -0,0 +1,2 @@ +# Disable verbose info logging from org.apache.hc.client5.http.impl.async.AsyncHttpRequestRetryExec +org.apache.hc.client5.http.impl.async.level = WARNING diff --git a/vespa-feed-client-cli/src/main/sh/vespa-feed-client.sh b/vespa-feed-client-cli/src/main/sh/vespa-feed-client.sh index 2a166dd40bb..ab43fca2f67 100755 --- a/vespa-feed-client-cli/src/main/sh/vespa-feed-client.sh +++ b/vespa-feed-client-cli/src/main/sh/vespa-feed-client.sh @@ -79,4 +79,6 @@ exec java \ -Djava.library.path=${VESPA_HOME}/libexec64/native:${VESPA_HOME}/lib64 \ -Djava.awt.headless=true \ -Xms128m -Xmx2048m $(getJavaOptionsIPV46) \ +--add-opens=java.base/sun.security.ssl=ALL-UNNAMED \ +-Djava.util.logging.config.file=${VESPA_HOME}/conf/vespa-feed-client/logging.properties \ -cp ${VESPA_HOME}/lib/jars/vespa-feed-client-cli.jar ai.vespa.feed.client.CliClient "$@" |