From 9a4b0b11a9b168698f94e45a3023215dcdcd975f Mon Sep 17 00:00:00 2001 From: Henning Baldersheim Date: Fri, 14 Jan 2022 21:50:30 +0100 Subject: All non application clusters only needs a few threads. 2 should be sufficient. This is controlled by the featureflag so far only used for the metrics-proxy. This will bring the clustercontroller and logserver in line with the metrics-proxy. --- .../src/main/java/com/yahoo/config/model/api/ModelContext.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'config-model-api/src') diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java index 6108c39f9d3..6b46e0957ff 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java @@ -82,7 +82,8 @@ public interface ModelContext { @ModelFeatureFlag(owners = {"baldersheim"}) default boolean skipMbusReplyThread() { throw new UnsupportedOperationException("TODO specify default value"); } @ModelFeatureFlag(owners = {"baldersheim"}) default boolean useAsyncMessageHandlingOnSchedule() { throw new UnsupportedOperationException("TODO specify default value"); } @ModelFeatureFlag(owners = {"baldersheim"}) default double feedConcurrency() { throw new UnsupportedOperationException("TODO specify default value"); } - @ModelFeatureFlag(owners = {"baldersheim"}) default int metricsproxyNumThreads() { throw new UnsupportedOperationException("TODO specify default value"); } + @ModelFeatureFlag(owners = {"baldersheim"}, removeAfter = "7.527") default int metricsproxyNumThreads() { return defaultPoolNumThreads(); } + @ModelFeatureFlag(owners = {"baldersheim"}) default int defaultPoolNumThreads() { return 2; } @ModelFeatureFlag(owners = {"baldersheim"}, removeAfter = "7.527") default int largeRankExpressionLimit() { return 8192; } @ModelFeatureFlag(owners = {"baldersheim"}) default int maxUnCommittedMemory() { return 130000; } @ModelFeatureFlag(owners = {"baldersheim"}) default int maxConcurrentMergesPerNode() { return 16; } -- cgit v1.2.3 From c5bbaa70a27554eec7db349e40bdca19c8313759 Mon Sep 17 00:00:00 2001 From: Geir Storli Date: Mon, 17 Jan 2022 13:46:08 +0000 Subject: Remove merge performance flags that are no longer needed. The functions in ModelContext.java can be removed when applications are on 7.528.3. --- .../com/yahoo/config/model/api/ModelContext.java | 8 ++--- .../yahoo/config/model/deploy/TestProperties.java | 28 ------------------ .../vespa/model/content/DistributorCluster.java | 14 +-------- .../content/storagecluster/FileStorProducer.java | 3 -- .../content/storagecluster/StorServerProducer.java | 5 ---- .../vespa/model/content/ContentClusterTest.java | 34 ---------------------- .../vespa/model/content/StorageClusterTest.java | 18 ------------ .../config/server/deploy/ModelContextImpl.java | 12 -------- .../src/main/java/com/yahoo/vespa/flags/Flags.java | 30 ------------------- 9 files changed, 5 insertions(+), 147 deletions(-) (limited to 'config-model-api/src') diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java index 6b46e0957ff..f4cffd5a724 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java @@ -88,7 +88,7 @@ public interface ModelContext { @ModelFeatureFlag(owners = {"baldersheim"}) default int maxUnCommittedMemory() { return 130000; } @ModelFeatureFlag(owners = {"baldersheim"}) default int maxConcurrentMergesPerNode() { return 16; } @ModelFeatureFlag(owners = {"baldersheim"}) default int maxMergeQueueSize() { return 100; } - @ModelFeatureFlag(owners = {"vekterli", "geirst"}) default boolean ignoreMergeQueueLimit() { return true; } + @ModelFeatureFlag(owners = {"vekterli", "geirst"}, removeAfter = "7.528.3") default boolean ignoreMergeQueueLimit() { return true; } @ModelFeatureFlag(owners = {"baldersheim"}) default boolean containerDumpHeapOnShutdownTimeout() { throw new UnsupportedOperationException("TODO specify default value"); } @ModelFeatureFlag(owners = {"baldersheim"}) default double containerShutdownTimeout() { throw new UnsupportedOperationException("TODO specify default value"); } @ModelFeatureFlag(owners = {"baldersheim"}, removeAfter = "7.527") default double diskBloatFactor() { return 0.25; } @@ -100,10 +100,10 @@ public interface ModelContext { @ModelFeatureFlag(owners = {"hmusum"}) default double resourceLimitDisk() { return 0.8; } @ModelFeatureFlag(owners = {"hmusum"}) default double resourceLimitMemory() { return 0.8; } @ModelFeatureFlag(owners = {"geirst", "vekterli"}) default double minNodeRatioPerGroup() { return 0.0; } - @ModelFeatureFlag(owners = {"geirst", "vekterli"}) default int distributorMergeBusyWait() { return 1; } - @ModelFeatureFlag(owners = {"vekterli", "geirst"}) default boolean distributorEnhancedMaintenanceScheduling() { return true; } + @ModelFeatureFlag(owners = {"geirst", "vekterli"}, removeAfter = "7.528.3") default int distributorMergeBusyWait() { return 1; } + @ModelFeatureFlag(owners = {"vekterli", "geirst"}, removeAfter = "7.528.3") default boolean distributorEnhancedMaintenanceScheduling() { return true; } @ModelFeatureFlag(owners = {"arnej"}) default boolean forwardIssuesAsErrors() { return true; } - @ModelFeatureFlag(owners = {"geirst", "vekterli"}) default boolean asyncApplyBucketDiff() { return true; } + @ModelFeatureFlag(owners = {"geirst", "vekterli"}, removeAfter = "7.528.3") default boolean asyncApplyBucketDiff() { return true; } @ModelFeatureFlag(owners = {"arnej"}) default boolean ignoreThreadStackSizes() { return false; } @ModelFeatureFlag(owners = {"vekterli", "geirst"}) default boolean unorderedMergeChaining() { return true; } @ModelFeatureFlag(owners = {"arnej"}) default boolean useV8GeoPositions() { return false; } diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java index 40b88372348..1478854aecf 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java @@ -58,7 +58,6 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea private String jvmOmitStackTraceInFastThrowOption; private int maxConcurrentMergesPerNode = 16; private int maxMergeQueueSize = 100; - private boolean ignoreMergeQueueLimit = true; private boolean allowDisableMtls = true; private List operatorCertificates = Collections.emptyList(); private double resourceLimitDisk = 0.8; @@ -66,10 +65,7 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea private double minNodeRatioPerGroup = 0.0; private boolean containerDumpHeapOnShutdownTimeout = false; private double containerShutdownTimeout = 50.0; - private int distributorMergeBusyWait = 1; private int maxUnCommittedMemory = 123456; - private boolean distributorEnhancedMaintenanceScheduling = true; - private boolean asyncApplyBucketDiff = true; private boolean unorderedMergeChaining = true; private List zoneDnsSuffixes = List.of(); private int maxCompactBuffers = 1; @@ -114,16 +110,12 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea @Override public List operatorCertificates() { return operatorCertificates; } @Override public int maxConcurrentMergesPerNode() { return maxConcurrentMergesPerNode; } @Override public int maxMergeQueueSize() { return maxMergeQueueSize; } - @Override public boolean ignoreMergeQueueLimit() { return ignoreMergeQueueLimit; } @Override public double resourceLimitDisk() { return resourceLimitDisk; } @Override public double resourceLimitMemory() { return resourceLimitMemory; } @Override public double minNodeRatioPerGroup() { return minNodeRatioPerGroup; } @Override public double containerShutdownTimeout() { return containerShutdownTimeout; } @Override public boolean containerDumpHeapOnShutdownTimeout() { return containerDumpHeapOnShutdownTimeout; } - @Override public int distributorMergeBusyWait() { return distributorMergeBusyWait; } - @Override public boolean distributorEnhancedMaintenanceScheduling() { return distributorEnhancedMaintenanceScheduling; } @Override public int maxUnCommittedMemory() { return maxUnCommittedMemory; } - @Override public boolean asyncApplyBucketDiff() { return asyncApplyBucketDiff; } @Override public boolean unorderedMergeChaining() { return unorderedMergeChaining; } @Override public List zoneDnsSuffixes() { return zoneDnsSuffixes; } @Override public int maxCompactBuffers() { return maxCompactBuffers; } @@ -196,11 +188,6 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea return this; } - public TestProperties setIgnoreMergeQueueLimit(boolean ignoreMergeQueueLimit) { - this.ignoreMergeQueueLimit = ignoreMergeQueueLimit; - return this; - } - public TestProperties setDefaultTermwiseLimit(double limit) { defaultTermwiseLimit = limit; return this; @@ -301,21 +288,6 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea return this; } - public TestProperties setDistributorMergeBusyWait(int value) { - distributorMergeBusyWait = value; - return this; - } - - public TestProperties distributorEnhancedMaintenanceScheduling(boolean enhancedScheduling) { - distributorEnhancedMaintenanceScheduling = enhancedScheduling; - return this; - } - - public TestProperties setAsyncApplyBucketDiff(boolean value) { - asyncApplyBucketDiff = value; - return this; - } - public TestProperties setUnorderedMergeChaining(boolean unordered) { unorderedMergeChaining = unordered; return this; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/DistributorCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/DistributorCluster.java index 7a45689901f..b8d2a4f91fe 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/DistributorCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/DistributorCluster.java @@ -42,8 +42,6 @@ public class DistributorCluster extends AbstractConfigProducer impl private final boolean hasIndexedDocumentType; private final boolean useThreePhaseUpdates; private final int maxActivationInhibitedOutOfSyncGroups; - private final int mergeBusyWait; - private final boolean enhancedMaintenanceScheduling; private final boolean unorderedMergeChaining; public static class Builder extends VespaDomBuilder.DomConfigProducerBuilder { @@ -107,16 +105,12 @@ public class DistributorCluster extends AbstractConfigProducer impl final boolean hasIndexedDocumentType = clusterContainsIndexedDocumentType(documentsNode); boolean useThreePhaseUpdates = deployState.getProperties().featureFlags().useThreePhaseUpdates(); int maxInhibitedGroups = deployState.getProperties().featureFlags().maxActivationInhibitedOutOfSyncGroups(); - int mergeBusyWait = deployState.getProperties().featureFlags().distributorMergeBusyWait(); - boolean useEnhancedMaintenanceScheduling = deployState.getProperties().featureFlags().distributorEnhancedMaintenanceScheduling(); boolean unorderedMergeChaining = deployState.getProperties().featureFlags().unorderedMergeChaining(); return new DistributorCluster(parent, new BucketSplitting.Builder().build(new ModelElement(producerSpec)), gc, hasIndexedDocumentType, useThreePhaseUpdates, - maxInhibitedGroups, mergeBusyWait, - useEnhancedMaintenanceScheduling, - unorderedMergeChaining); + maxInhibitedGroups, unorderedMergeChaining); } } @@ -124,8 +118,6 @@ public class DistributorCluster extends AbstractConfigProducer impl GcOptions gc, boolean hasIndexedDocumentType, boolean useThreePhaseUpdates, int maxActivationInhibitedOutOfSyncGroups, - int mergeBusyWait, - boolean enhancedMaintenanceScheduling, boolean unorderedMergeChaining) { super(parent, "distributor"); @@ -135,8 +127,6 @@ public class DistributorCluster extends AbstractConfigProducer impl this.hasIndexedDocumentType = hasIndexedDocumentType; this.useThreePhaseUpdates = useThreePhaseUpdates; this.maxActivationInhibitedOutOfSyncGroups = maxActivationInhibitedOutOfSyncGroups; - this.mergeBusyWait = mergeBusyWait; - this.enhancedMaintenanceScheduling = enhancedMaintenanceScheduling; this.unorderedMergeChaining = unorderedMergeChaining; } @@ -151,8 +141,6 @@ public class DistributorCluster extends AbstractConfigProducer impl builder.disable_bucket_activation(hasIndexedDocumentType == false); builder.enable_metadata_only_fetch_phase_for_inconsistent_updates(useThreePhaseUpdates); builder.max_activation_inhibited_out_of_sync_groups(maxActivationInhibitedOutOfSyncGroups); - builder.inhibit_merge_sending_on_busy_node_duration_sec(mergeBusyWait); - builder.implicitly_clear_bucket_priority_on_schedule(enhancedMaintenanceScheduling); builder.use_unordered_merge_chaining(unorderedMergeChaining); bucketSplitting.getConfig(builder); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/FileStorProducer.java b/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/FileStorProducer.java index 1f3a76b766e..5be06306f89 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/FileStorProducer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/FileStorProducer.java @@ -48,7 +48,6 @@ public class FileStorProducer implements StorFilestorConfig.Producer { private final StorFilestorConfig.Response_sequencer_type.Enum responseSequencerType; private final StorFilestorConfig.Async_operation_throttler_type.Enum asyncOperationThrottlerType; private final boolean useAsyncMessageHandlingOnSchedule; - private final boolean asyncApplyBucketDiff; private static StorFilestorConfig.Response_sequencer_type.Enum convertResponseSequencerType(String sequencerType) { try { @@ -73,7 +72,6 @@ public class FileStorProducer implements StorFilestorConfig.Producer { this.responseSequencerType = convertResponseSequencerType(featureFlags.responseSequencerType()); this.asyncOperationThrottlerType = toAsyncOperationThrottlerType(featureFlags.persistenceAsyncThrottling()); useAsyncMessageHandlingOnSchedule = featureFlags.useAsyncMessageHandlingOnSchedule(); - asyncApplyBucketDiff = featureFlags.asyncApplyBucketDiff(); } @Override @@ -85,7 +83,6 @@ public class FileStorProducer implements StorFilestorConfig.Producer { builder.num_response_threads(reponseNumThreads); builder.response_sequencer_type(responseSequencerType); builder.use_async_message_handling_on_schedule(useAsyncMessageHandlingOnSchedule); - builder.async_apply_bucket_diff(asyncApplyBucketDiff); builder.async_operation_throttler_type(asyncOperationThrottlerType); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/StorServerProducer.java b/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/StorServerProducer.java index c7d581fca35..6f9166f4493 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/StorServerProducer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/StorServerProducer.java @@ -32,7 +32,6 @@ public class StorServerProducer implements StorServerConfig.Producer { private Integer maxMergesPerNode; private Integer queueSize; private Integer bucketDBStripeBits; - private Boolean ignoreMergeQueueLimit; private StorServerProducer setMaxMergesPerNode(Integer value) { if (value != null) { @@ -55,7 +54,6 @@ public class StorServerProducer implements StorServerConfig.Producer { this.clusterName = clusterName; maxMergesPerNode = featureFlags.maxConcurrentMergesPerNode(); queueSize = featureFlags.maxMergeQueueSize(); - ignoreMergeQueueLimit = featureFlags.ignoreMergeQueueLimit(); } @Override @@ -75,8 +73,5 @@ public class StorServerProducer implements StorServerConfig.Producer { if (bucketDBStripeBits != null) { builder.content_node_bucket_db_stripe_bits(bucketDBStripeBits); } - if (ignoreMergeQueueLimit != null) { - builder.disable_queue_limits_for_chained_merges(ignoreMergeQueueLimit); - } } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java index 87a962339e9..10a2feaba5b 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java @@ -1119,40 +1119,6 @@ public class ContentClusterTest extends ContentBaseTest { assertEquals(4, resolveTunedNumDistributorStripesConfig(65)); } - @Test - public void distributor_merge_busy_wait_controlled_by_properties() throws Exception { - assertEquals(1, resolveDistributorMergeBusyWaitConfig(Optional.empty())); - assertEquals(5, resolveDistributorMergeBusyWaitConfig(Optional.of(5))); - } - - private int resolveDistributorMergeBusyWaitConfig(Optional mergeBusyWait) throws Exception { - var props = new TestProperties(); - if (mergeBusyWait.isPresent()) { - props.setDistributorMergeBusyWait(mergeBusyWait.get()); - } - var cluster = createOneNodeCluster(props); - var builder = new StorDistributormanagerConfig.Builder(); - cluster.getDistributorNodes().getConfig(builder); - return (new StorDistributormanagerConfig(builder)).inhibit_merge_sending_on_busy_node_duration_sec(); - } - - @Test - public void distributor_enhanced_maintenance_scheduling_controlled_by_properties() throws Exception { - assertFalse(resolveDistributorEnhancedSchedulingConfig(Optional.of(false))); - assertTrue(resolveDistributorEnhancedSchedulingConfig(Optional.empty())); - } - - private boolean resolveDistributorEnhancedSchedulingConfig(Optional enhancedScheduling) throws Exception { - var props = new TestProperties(); - if (enhancedScheduling.isPresent()) { - props.distributorEnhancedMaintenanceScheduling(enhancedScheduling.get()); - } - var cluster = createOneNodeCluster(props); - var builder = new StorDistributormanagerConfig.Builder(); - cluster.getDistributorNodes().getConfig(builder); - return (new StorDistributormanagerConfig(builder)).implicitly_clear_bucket_priority_on_schedule(); - } - @Test public void unordered_merge_chaining_config_controlled_by_properties() throws Exception { assertFalse(resolveUnorderedMergeChainingConfig(Optional.of(false))); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java index cf877d3bf88..3a67157f135 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java @@ -159,24 +159,6 @@ public class StorageClusterTest { assertEquals(1919, config.max_merge_queue_size()); } - @Test - public void ignore_merge_queue_limit_can_be_controlled_by_feature_flag() { - var config = configFromProperties(new TestProperties().setIgnoreMergeQueueLimit(true)); - assertTrue(config.disable_queue_limits_for_chained_merges()); - - config = configFromProperties(new TestProperties().setIgnoreMergeQueueLimit(false)); - assertFalse(config.disable_queue_limits_for_chained_merges()); - } - - @Test - public void async_apply_bucket_diff_can_be_controlled_by_feature_flag() { - var config = filestorConfigFromProperties(new TestProperties()); - assertTrue(config.async_apply_bucket_diff()); - - config = filestorConfigFromProperties(new TestProperties().setAsyncApplyBucketDiff(false)); - assertFalse(config.async_apply_bucket_diff()); - } - @Test public void testVisitors() { StorVisitorConfig.Builder builder = new StorVisitorConfig.Builder(); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java index db9869fa5f2..03ba8fd5efd 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java @@ -183,18 +183,14 @@ public class ModelContextImpl implements ModelContext { private final ToIntFunction jvmOmitStackTraceInFastThrow; private final int maxConcurrentMergesPerContentNode; private final int maxMergeQueueSize; - private final boolean ignoreMergeQueueLimit; private final double resourceLimitDisk; private final double resourceLimitMemory; private final double minNodeRatioPerGroup; private final int metricsproxyNumThreads; private final boolean containerDumpHeapOnShutdownTimeout; private final double containerShutdownTimeout; - private final int distributorMergeBusyWait; - private final boolean distributorEnhancedMaintenanceScheduling; private final int maxUnCommittedMemory; private final boolean forwardIssuesAsErrors; - private final boolean asyncApplyBucketDiff; private final boolean ignoreThreadStackSizes; private final boolean unorderedMergeChaining; private final boolean useV8GeoPositions; @@ -225,18 +221,14 @@ public class ModelContextImpl implements ModelContext { this.jvmOmitStackTraceInFastThrow = type -> flagValueAsInt(source, appId, type, PermanentFlags.JVM_OMIT_STACK_TRACE_IN_FAST_THROW); this.maxConcurrentMergesPerContentNode = flagValue(source, appId, Flags.MAX_CONCURRENT_MERGES_PER_NODE); this.maxMergeQueueSize = flagValue(source, appId, Flags.MAX_MERGE_QUEUE_SIZE); - this.ignoreMergeQueueLimit = flagValue(source, appId, Flags.IGNORE_MERGE_QUEUE_LIMIT); this.resourceLimitDisk = flagValue(source, appId, PermanentFlags.RESOURCE_LIMIT_DISK); this.resourceLimitMemory = flagValue(source, appId, PermanentFlags.RESOURCE_LIMIT_MEMORY); this.minNodeRatioPerGroup = flagValue(source, appId, Flags.MIN_NODE_RATIO_PER_GROUP); this.metricsproxyNumThreads = flagValue(source, appId, Flags.METRICSPROXY_NUM_THREADS); this.containerDumpHeapOnShutdownTimeout = flagValue(source, appId, Flags.CONTAINER_DUMP_HEAP_ON_SHUTDOWN_TIMEOUT); this.containerShutdownTimeout = flagValue(source, appId,Flags.CONTAINER_SHUTDOWN_TIMEOUT); - this.distributorMergeBusyWait = flagValue(source, appId, Flags.DISTRIBUTOR_MERGE_BUSY_WAIT); - this.distributorEnhancedMaintenanceScheduling = flagValue(source, appId, Flags.DISTRIBUTOR_ENHANCED_MAINTENANCE_SCHEDULING); this.maxUnCommittedMemory = flagValue(source, appId, Flags.MAX_UNCOMMITTED_MEMORY);; this.forwardIssuesAsErrors = flagValue(source, appId, PermanentFlags.FORWARD_ISSUES_AS_ERRORS); - this.asyncApplyBucketDiff = flagValue(source, appId, Flags.ASYNC_APPLY_BUCKET_DIFF); this.ignoreThreadStackSizes = flagValue(source, appId, Flags.IGNORE_THREAD_STACK_SIZES); this.unorderedMergeChaining = flagValue(source, appId, Flags.UNORDERED_MERGE_CHAINING); this.useV8GeoPositions = flagValue(source, appId, Flags.USE_V8_GEO_POSITIONS); @@ -269,18 +261,14 @@ public class ModelContextImpl implements ModelContext { } @Override public int maxConcurrentMergesPerNode() { return maxConcurrentMergesPerContentNode; } @Override public int maxMergeQueueSize() { return maxMergeQueueSize; } - @Override public boolean ignoreMergeQueueLimit() { return ignoreMergeQueueLimit; } @Override public double resourceLimitDisk() { return resourceLimitDisk; } @Override public double resourceLimitMemory() { return resourceLimitMemory; } @Override public double minNodeRatioPerGroup() { return minNodeRatioPerGroup; } @Override public int defaultPoolNumThreads() { return metricsproxyNumThreads; } @Override public double containerShutdownTimeout() { return containerShutdownTimeout; } @Override public boolean containerDumpHeapOnShutdownTimeout() { return containerDumpHeapOnShutdownTimeout; } - @Override public int distributorMergeBusyWait() { return distributorMergeBusyWait; } - @Override public boolean distributorEnhancedMaintenanceScheduling() { return distributorEnhancedMaintenanceScheduling; } @Override public int maxUnCommittedMemory() { return maxUnCommittedMemory; } @Override public boolean forwardIssuesAsErrors() { return forwardIssuesAsErrors; } - @Override public boolean asyncApplyBucketDiff() { return asyncApplyBucketDiff; } @Override public boolean ignoreThreadStackSizes() { return ignoreThreadStackSizes; } @Override public boolean unorderedMergeChaining() { return unorderedMergeChaining; } @Override public boolean useV8GeoPositions() { return useV8GeoPositions; } diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index c94bfec948e..b15fdf7e338 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -208,14 +208,6 @@ public class Flags { "Takes effect at redeploy", ZONE_ID, APPLICATION_ID); - public static final UnboundBooleanFlag IGNORE_MERGE_QUEUE_LIMIT = defineFeatureFlag( - "ignore-merge-queue-limit", true, - List.of("vekterli", "geirst"), "2021-10-06", "2022-03-01", - "Specifies if merges that are forwarded (chained) from another content node are always " + - "allowed to be enqueued even if the queue is otherwise full.", - "Takes effect at redeploy", - ZONE_ID, APPLICATION_ID); - public static final UnboundDoubleFlag MIN_NODE_RATIO_PER_GROUP = defineDoubleFlag( "min-node-ratio-per-group", 0.0, List.of("geirst", "vekterli"), "2021-07-16", "2022-03-01", @@ -277,28 +269,6 @@ public class Flags { TENANT_ID ); - public static final UnboundIntFlag DISTRIBUTOR_MERGE_BUSY_WAIT = defineIntFlag( - "distributor-merge-busy-wait", 1, - List.of("geirst", "vekterli"), "2021-10-04", "2022-03-01", - "Number of seconds that scheduling of new merge operations in the distributor should be inhibited " + - "towards a content node that has indicated merge busy", - "Takes effect at redeploy", - ZONE_ID, APPLICATION_ID); - - public static final UnboundBooleanFlag DISTRIBUTOR_ENHANCED_MAINTENANCE_SCHEDULING = defineFeatureFlag( - "distributor-enhanced-maintenance-scheduling", true, - List.of("vekterli", "geirst"), "2021-10-14", "2022-01-31", - "Enable enhanced maintenance operation scheduling semantics on the distributor", - "Takes effect at redeploy", - ZONE_ID, APPLICATION_ID); - - public static final UnboundBooleanFlag ASYNC_APPLY_BUCKET_DIFF = defineFeatureFlag( - "async-apply-bucket-diff", true, - List.of("geirst", "vekterli"), "2021-10-22", "2022-01-31", - "Whether portions of apply bucket diff handling will be performed asynchronously", - "Takes effect at redeploy", - ZONE_ID, APPLICATION_ID); - public static final UnboundBooleanFlag UNORDERED_MERGE_CHAINING = defineFeatureFlag( "unordered-merge-chaining", true, List.of("vekterli", "geirst"), "2021-11-15", "2022-03-01", -- cgit v1.2.3 From 7f4afd126f231f9fed62e5acedef1bfce3a3d63b Mon Sep 17 00:00:00 2001 From: Henning Baldersheim Date: Tue, 18 Jan 2022 07:01:57 +0100 Subject: Add flag to control number of processors a non-application jvm gets to use. Intention is to lower this number from 2 to 1 in order to reduce memory footprint for small containers. --- .../src/main/java/com/yahoo/config/model/api/ModelContext.java | 1 + .../java/com/yahoo/vespa/model/container/ContainerCluster.java | 4 +++- .../com/yahoo/vespa/config/server/deploy/ModelContextImpl.java | 5 ++++- flags/src/main/java/com/yahoo/vespa/flags/Flags.java | 7 +++++++ 4 files changed, 15 insertions(+), 2 deletions(-) (limited to 'config-model-api/src') diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java index f4cffd5a724..d509fa5dd6a 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java @@ -84,6 +84,7 @@ public interface ModelContext { @ModelFeatureFlag(owners = {"baldersheim"}) default double feedConcurrency() { throw new UnsupportedOperationException("TODO specify default value"); } @ModelFeatureFlag(owners = {"baldersheim"}, removeAfter = "7.527") default int metricsproxyNumThreads() { return defaultPoolNumThreads(); } @ModelFeatureFlag(owners = {"baldersheim"}) default int defaultPoolNumThreads() { return 2; } + @ModelFeatureFlag(owners = {"baldersheim"}) default int availableProcessors() { return 2; } @ModelFeatureFlag(owners = {"baldersheim"}, removeAfter = "7.527") default int largeRankExpressionLimit() { return 8192; } @ModelFeatureFlag(owners = {"baldersheim"}) default int maxUnCommittedMemory() { return 130000; } @ModelFeatureFlag(owners = {"baldersheim"}) default int maxConcurrentMergesPerNode() { return 16; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java index 7010d7b3d4e..982d886cbea 100755 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java @@ -146,6 +146,7 @@ public abstract class ContainerCluster private final ComponentGroup> componentGroup; private final boolean isHostedVespa; private final boolean zooKeeperLocalhostAffinity; + private final int numAvailableProcessors; private final Map concreteDocumentTypes = new LinkedHashMap<>(); @@ -166,6 +167,7 @@ public abstract class ContainerCluster this.isHostedVespa = stateIsHosted(deployState); this.zone = (deployState != null) ? deployState.zone() : Zone.defaultZone(); this.zooKeeperLocalhostAffinity = zooKeeperLocalhostAffinity; + numAvailableProcessors = deployState.featureFlags().availableProcessors(); componentGroup = new ComponentGroup<>(this, "component"); @@ -493,7 +495,7 @@ public abstract class ContainerCluster public void getConfig(QrStartConfig.Builder builder) { builder.jvm .verbosegc(false) - .availableProcessors(2) + .availableProcessors(numAvailableProcessors) .compressedClassSpaceSize(32) .minHeapsize(32) .heapsize(256) diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java index 03ba8fd5efd..b8f79682a03 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java @@ -187,6 +187,7 @@ public class ModelContextImpl implements ModelContext { private final double resourceLimitMemory; private final double minNodeRatioPerGroup; private final int metricsproxyNumThreads; + private final int availableProcessors; private final boolean containerDumpHeapOnShutdownTimeout; private final double containerShutdownTimeout; private final int maxUnCommittedMemory; @@ -225,9 +226,10 @@ public class ModelContextImpl implements ModelContext { this.resourceLimitMemory = flagValue(source, appId, PermanentFlags.RESOURCE_LIMIT_MEMORY); this.minNodeRatioPerGroup = flagValue(source, appId, Flags.MIN_NODE_RATIO_PER_GROUP); this.metricsproxyNumThreads = flagValue(source, appId, Flags.METRICSPROXY_NUM_THREADS); + this.availableProcessors = flagValue(source, appId, Flags.AVAILABLE_PROCESSORS); this.containerDumpHeapOnShutdownTimeout = flagValue(source, appId, Flags.CONTAINER_DUMP_HEAP_ON_SHUTDOWN_TIMEOUT); this.containerShutdownTimeout = flagValue(source, appId,Flags.CONTAINER_SHUTDOWN_TIMEOUT); - this.maxUnCommittedMemory = flagValue(source, appId, Flags.MAX_UNCOMMITTED_MEMORY);; + this.maxUnCommittedMemory = flagValue(source, appId, Flags.MAX_UNCOMMITTED_MEMORY); this.forwardIssuesAsErrors = flagValue(source, appId, PermanentFlags.FORWARD_ISSUES_AS_ERRORS); this.ignoreThreadStackSizes = flagValue(source, appId, Flags.IGNORE_THREAD_STACK_SIZES); this.unorderedMergeChaining = flagValue(source, appId, Flags.UNORDERED_MERGE_CHAINING); @@ -265,6 +267,7 @@ public class ModelContextImpl implements ModelContext { @Override public double resourceLimitMemory() { return resourceLimitMemory; } @Override public double minNodeRatioPerGroup() { return minNodeRatioPerGroup; } @Override public int defaultPoolNumThreads() { return metricsproxyNumThreads; } + @Override public int availableProcessors() { return availableProcessors; } @Override public double containerShutdownTimeout() { return containerShutdownTimeout; } @Override public boolean containerDumpHeapOnShutdownTimeout() { return containerDumpHeapOnShutdownTimeout; } @Override public int maxUnCommittedMemory() { return maxUnCommittedMemory; } diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index 1013cd229b0..ce9f1c80a1e 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -222,6 +222,13 @@ public class Flags { "Takes effect at redeployment", ZONE_ID, APPLICATION_ID); + public static final UnboundIntFlag AVAILABLE_PROCESSORS = defineIntFlag( + "available-processors", 2, + List.of("balder"), "2022-18-01", "2022-04-01", + "Number of processors the jvm sees in non-application clusters", + "Takes effect at redeployment", + ZONE_ID, APPLICATION_ID); + public static final UnboundBooleanFlag ENABLED_HORIZON_DASHBOARD = defineFeatureFlag( "enabled-horizon-dashboard", false, List.of("olaa"), "2021-09-13", "2022-02-01", -- cgit v1.2.3 From 74b4c0495907dee87491879455a81f748c29c236 Mon Sep 17 00:00:00 2001 From: Arne H Juul Date: Tue, 18 Jan 2022 13:26:56 +0000 Subject: add feature flag for old "qrserver" service name * we will remove the hack that uses "qrserver" as the service name in some cases, but we need to check what the impact will be first. --- .../src/main/java/com/yahoo/config/model/api/ModelContext.java | 1 + .../java/com/yahoo/vespa/model/container/ApplicationContainer.java | 4 +++- .../com/yahoo/vespa/config/server/deploy/ModelContextImpl.java | 3 +++ flags/src/main/java/com/yahoo/vespa/flags/Flags.java | 7 +++++++ 4 files changed, 14 insertions(+), 1 deletion(-) (limited to 'config-model-api/src') diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java index d509fa5dd6a..14cd6da1c2e 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java @@ -115,6 +115,7 @@ public interface ModelContext { @ModelFeatureFlag(owners = {"arnej", "andreer"}) default List ignoredHttpUserAgents() { return List.of(); } @ModelFeatureFlag(owners = {"bjorncs"}) default boolean enableServerOcspStapling() { return false; } @ModelFeatureFlag(owners = {"vekterli"}) default String persistenceAsyncThrottling() { throw new UnsupportedOperationException("TODO specify default value"); } + @ModelFeatureFlag(owners = {"arnej"}) default boolean useQrserverServiceName() { return true; } } /** Warning: As elsewhere in this package, do not make backwards incompatible changes that will break old config models! */ diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainer.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainer.java index 7c386875d02..43cb896c09b 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainer.java @@ -24,6 +24,7 @@ public final class ApplicationContainer extends Container implements private final boolean isHostedVespa; private final boolean enableServerOcspStapling; + private final boolean useQrserverServiceName; public ApplicationContainer(AbstractConfigProducer parent, String name, int index, DeployState deployState) { this(parent, name, false, index, deployState); @@ -33,6 +34,7 @@ public final class ApplicationContainer extends Container implements super(parent, name, retired, index, deployState); this.isHostedVespa = deployState.isHosted(); this.enableServerOcspStapling = deployState.featureFlags().enableServerOcspStapling(); + this.useQrserverServiceName = deployState.featureFlags().useQrserverServiceName(); addComponent(new SimpleComponent("com.yahoo.container.jdisc.messagebus.NetworkMultiplexerHolder")); addComponent(new SimpleComponent("com.yahoo.container.jdisc.messagebus.NetworkMultiplexerProvider")); @@ -56,7 +58,7 @@ public final class ApplicationContainer extends Container implements if (parent instanceof ContainerCluster) { ContainerCluster cluster = (ContainerCluster)parent; // TODO: The 'qrserver' name is retained for legacy reasons (e.g. system tests and log parsing). - if (cluster.getSearch() != null && cluster.getDocproc() == null && cluster.getDocumentApi() == null) { + if (useQrserverServiceName && cluster.getSearch() != null && cluster.getDocproc() == null && cluster.getDocumentApi() == null) { return ContainerServiceType.QRSERVER; } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java index b8f79682a03..fddb80610cd 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java @@ -201,6 +201,7 @@ public class ModelContextImpl implements ModelContext { private final List ignoredHttpUserAgents; private final boolean enableServerOcspStapling; private final String persistenceAsyncThrottling; + private final boolean useQrserverServiceName; public FeatureFlags(FlagSource source, ApplicationId appId) { this.defaultTermwiseLimit = flagValue(source, appId, Flags.DEFAULT_TERM_WISE_LIMIT); @@ -240,6 +241,7 @@ public class ModelContextImpl implements ModelContext { this.ignoredHttpUserAgents = flagValue(source, appId, PermanentFlags.IGNORED_HTTP_USER_AGENTS); this.enableServerOcspStapling = flagValue(source, appId, Flags.ENABLE_SERVER_OCSP_STAPLING); this.persistenceAsyncThrottling = flagValue(source, appId, Flags.PERSISTENCE_ASYNC_THROTTLING); + this.useQrserverServiceName = flagValue(source, appId, Flags.USE_QRSERVER_SERVICE_NAME); } @Override public double defaultTermwiseLimit() { return defaultTermwiseLimit; } @@ -281,6 +283,7 @@ public class ModelContextImpl implements ModelContext { @Override public List ignoredHttpUserAgents() { return ignoredHttpUserAgents; } @Override public boolean enableServerOcspStapling() { return enableServerOcspStapling; } @Override public String persistenceAsyncThrottling() { return persistenceAsyncThrottling; } + @Override public boolean useQrserverServiceName() { return useQrserverServiceName; } private static V flagValue(FlagSource source, ApplicationId appId, UnboundFlag flag) { return flag.bindTo(source) diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index 758e46f0e95..24039aa20f4 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -372,6 +372,13 @@ public class Flags { "Takes effect immediately", ZONE_ID, APPLICATION_ID); + public static final UnboundBooleanFlag USE_QRSERVER_SERVICE_NAME = defineFeatureFlag( + "use-qrserver-service-name", true, + List.of("arnej"), "2022-01-18", "2022-12-31", + "Use backwards-compatible 'qrserver' service name for containers with only 'search' API", + "Takes effect at redeployment", + ZONE_ID, APPLICATION_ID); + /** WARNING: public for testing: All flags should be defined in {@link Flags}. */ public static UnboundBooleanFlag defineFeatureFlag(String flagId, boolean defaultValue, List owners, String createdAt, String expiresAt, String description, -- cgit v1.2.3 From 54bfb797c727e16537ece247b0d834d30ab54ce1 Mon Sep 17 00:00:00 2001 From: Harald Musum Date: Tue, 18 Jan 2022 22:10:18 +0100 Subject: Add cluster name and getDeferChangesUntilRestart to ApplicationClusterInfo Needed for upcoming convergence checker work --- .../java/com/yahoo/config/model/api/ApplicationClusterInfo.java | 6 ++++++ .../yahoo/vespa/model/container/ApplicationContainerCluster.java | 3 +++ 2 files changed, 9 insertions(+) (limited to 'config-model-api/src') diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ApplicationClusterInfo.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ApplicationClusterInfo.java index 2cd2e980761..20fcb6b599b 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ApplicationClusterInfo.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ApplicationClusterInfo.java @@ -5,5 +5,11 @@ package com.yahoo.config.model.api; import java.util.List; public interface ApplicationClusterInfo { + List endpoints(); + + boolean getDeferChangesUntilRestart(); + + String name(); + } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java index 89c455269f4..780de5dbf04 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java @@ -343,6 +343,9 @@ public final class ApplicationContainerCluster extends ContainerCluster Date: Fri, 21 Jan 2022 09:25:08 +0100 Subject: Add upgrade revision strategy setting to deployment spec --- .../application/api/DeploymentInstanceSpec.java | 9 +++++++- .../config/application/api/DeploymentSpec.java | 9 ++++++++ .../api/xml/DeploymentSpecXmlReader.java | 25 ++++++++++++++++++++-- .../config/application/api/DeploymentSpecTest.java | 22 +++++++++++++++++-- .../src/main/resources/schema/deployment.rnc | 1 + 5 files changed, 61 insertions(+), 5 deletions(-) (limited to 'config-model-api/src') diff --git a/config-model-api/src/main/java/com/yahoo/config/application/api/DeploymentInstanceSpec.java b/config-model-api/src/main/java/com/yahoo/config/application/api/DeploymentInstanceSpec.java index 67ddb9ef83c..ea38860c29b 100644 --- a/config-model-api/src/main/java/com/yahoo/config/application/api/DeploymentInstanceSpec.java +++ b/config-model-api/src/main/java/com/yahoo/config/application/api/DeploymentInstanceSpec.java @@ -31,6 +31,7 @@ public class DeploymentInstanceSpec extends DeploymentSpec.Steps { private final InstanceName name; private final DeploymentSpec.UpgradePolicy upgradePolicy; + private final DeploymentSpec.UpgradeRevision upgradeRevision; private final DeploymentSpec.UpgradeRollout upgradeRollout; private final List changeBlockers; private final Optional globalServiceId; @@ -41,6 +42,7 @@ public class DeploymentInstanceSpec extends DeploymentSpec.Steps { public DeploymentInstanceSpec(InstanceName name, List steps, DeploymentSpec.UpgradePolicy upgradePolicy, + DeploymentSpec.UpgradeRevision upgradeRevision, DeploymentSpec.UpgradeRollout upgradeRollout, List changeBlockers, Optional globalServiceId, @@ -51,6 +53,7 @@ public class DeploymentInstanceSpec extends DeploymentSpec.Steps { super(steps); this.name = name; this.upgradePolicy = upgradePolicy; + this.upgradeRevision = upgradeRevision; this.upgradeRollout = upgradeRollout; this.changeBlockers = changeBlockers; this.globalServiceId = globalServiceId; @@ -150,6 +153,9 @@ public class DeploymentInstanceSpec extends DeploymentSpec.Steps { /** Returns the upgrade policy of this, which is defaultPolicy if none is specified */ public DeploymentSpec.UpgradePolicy upgradePolicy() { return upgradePolicy; } + /** Returns the upgrade revision strategy of this, which is separate if none is specified */ + public DeploymentSpec.UpgradeRevision upgradeRevision() { return upgradeRevision; } + /** Returns the upgrade rollout strategy of this, which is separate if none is specified */ public DeploymentSpec.UpgradeRollout upgradeRollout() { return upgradeRollout; } @@ -198,6 +204,7 @@ public class DeploymentInstanceSpec extends DeploymentSpec.Steps { DeploymentInstanceSpec other = (DeploymentInstanceSpec) o; return globalServiceId.equals(other.globalServiceId) && upgradePolicy == other.upgradePolicy && + upgradeRevision == other.upgradeRevision && upgradeRollout == other.upgradeRollout && changeBlockers.equals(other.changeBlockers) && steps().equals(other.steps()) && @@ -208,7 +215,7 @@ public class DeploymentInstanceSpec extends DeploymentSpec.Steps { @Override public int hashCode() { - return Objects.hash(globalServiceId, upgradePolicy, upgradeRollout, changeBlockers, steps(), athenzService, notifications, endpoints); + return Objects.hash(globalServiceId, upgradePolicy, upgradeRevision, upgradeRollout, changeBlockers, steps(), athenzService, notifications, endpoints); } @Override diff --git a/config-model-api/src/main/java/com/yahoo/config/application/api/DeploymentSpec.java b/config-model-api/src/main/java/com/yahoo/config/application/api/DeploymentSpec.java index 88363db6e49..4b019bd9f7a 100644 --- a/config-model-api/src/main/java/com/yahoo/config/application/api/DeploymentSpec.java +++ b/config-model-api/src/main/java/com/yahoo/config/application/api/DeploymentSpec.java @@ -550,6 +550,15 @@ public class DeploymentSpec { } + /** Determines when application changes deploy, when an older revision is already rolling out. */ + public enum UpgradeRevision { + /** Separate: Application changes wait for previous application changes to complete, unless they fail. */ + separate, + /** Latest: Application changes immediately supersede previous application changes, unless currently blocked. */ + latest + } + + /** Determines when application changes deploy, when there is already an ongoing platform upgrade. */ public enum UpgradeRollout { /** Separate: Application changes wait for upgrade to complete, unless upgrade fails. */ diff --git a/config-model-api/src/main/java/com/yahoo/config/application/api/xml/DeploymentSpecXmlReader.java b/config-model-api/src/main/java/com/yahoo/config/application/api/xml/DeploymentSpecXmlReader.java index 8f866654d56..b031af9faf2 100644 --- a/config-model-api/src/main/java/com/yahoo/config/application/api/xml/DeploymentSpecXmlReader.java +++ b/config-model-api/src/main/java/com/yahoo/config/application/api/xml/DeploymentSpecXmlReader.java @@ -165,6 +165,7 @@ public class DeploymentSpecXmlReader { // Values where the parent may provide a default DeploymentSpec.UpgradePolicy upgradePolicy = readUpgradePolicy(instanceTag, parentTag); + DeploymentSpec.UpgradeRevision upgradeRevision = readUpgradeRevision(instanceTag, parentTag); DeploymentSpec.UpgradeRollout upgradeRollout = readUpgradeRollout(instanceTag, parentTag); List changeBlockers = readChangeBlockers(instanceTag, parentTag); Optional athenzService = mostSpecificAttribute(instanceTag, athenzServiceAttribute).map(AthenzService::from); @@ -183,6 +184,7 @@ public class DeploymentSpecXmlReader { .map(name -> new DeploymentInstanceSpec(InstanceName.from(name), steps, upgradePolicy, + upgradeRevision, upgradeRollout, changeBlockers, globalServiceId.asOptional(), @@ -472,6 +474,25 @@ public class DeploymentSpecXmlReader { } } + private DeploymentSpec.UpgradeRevision readUpgradeRevision(Element parent, Element fallbackParent) { + Element upgradeElement = XML.getChild(parent, upgradeTag); + if (upgradeElement == null) + upgradeElement = XML.getChild(fallbackParent, upgradeTag); + if (upgradeElement == null) + return DeploymentSpec.UpgradeRevision.separate; + + String revision = upgradeElement.getAttribute("revision"); + if (revision.isEmpty()) + return DeploymentSpec.UpgradeRevision.separate; + + switch (revision) { + case "separate": return DeploymentSpec.UpgradeRevision.separate; + case "latest": return DeploymentSpec.UpgradeRevision.latest; + default: throw new IllegalArgumentException("Illegal upgrade revision '" + revision + "': " + + "Must be one of " + Arrays.toString(DeploymentSpec.UpgradeRevision.values())); + } + } + private DeploymentSpec.UpgradeRollout readUpgradeRollout(Element parent, Element fallbackParent) { Element upgradeElement = XML.getChild(parent, upgradeTag); if (upgradeElement == null) @@ -487,8 +508,8 @@ public class DeploymentSpecXmlReader { case "separate": return DeploymentSpec.UpgradeRollout.separate; case "leading": return DeploymentSpec.UpgradeRollout.leading; // case "simultaneous": return DeploymentSpec.UpgradePolicy.conservative; - default: throw new IllegalArgumentException("Illegal upgrade policy '" + rollout + "': " + - "Must be one of " + Arrays.toString(DeploymentSpec.UpgradePolicy.values())); + default: throw new IllegalArgumentException("Illegal upgrade rollout '" + rollout + "': " + + "Must be one of " + Arrays.toString(DeploymentSpec.UpgradeRollout.values())); } } diff --git a/config-model-api/src/test/java/com/yahoo/config/application/api/DeploymentSpecTest.java b/config-model-api/src/test/java/com/yahoo/config/application/api/DeploymentSpecTest.java index 2fa2ba83291..a97faf5995d 100644 --- a/config-model-api/src/test/java/com/yahoo/config/application/api/DeploymentSpecTest.java +++ b/config-model-api/src/test/java/com/yahoo/config/application/api/DeploymentSpecTest.java @@ -121,6 +121,7 @@ public class DeploymentSpecTest { assertFalse(spec.requireInstance("default").globalServiceId().isPresent()); assertEquals(DeploymentSpec.UpgradePolicy.defaultPolicy, spec.requireInstance("default").upgradePolicy()); + assertEquals(DeploymentSpec.UpgradeRevision.separate, spec.requireInstance("default").upgradeRevision()); assertEquals(DeploymentSpec.UpgradeRollout.separate, spec.requireInstance("default").upgradeRollout()); } @@ -363,6 +364,21 @@ public class DeploymentSpecTest { assertEquals("qrs", spec.requireInstance("default").globalServiceId().get()); } + @Test + public void productionSpecWithUpgradeRevision() { + StringReader r = new StringReader( + "" + + " " + + " " + + " " + + " " + + "" + ); + DeploymentSpec spec = DeploymentSpec.fromXml(r); + assertEquals("latest", spec.requireInstance("default").upgradeRevision().toString()); + assertEquals("separate", spec.requireInstance("custom").upgradeRevision().toString()); + } + @Test public void productionSpecWithUpgradeRollout() { StringReader r = new StringReader( @@ -397,10 +413,10 @@ public class DeploymentSpecTest { public void upgradePolicyDefault() { StringReader r = new StringReader( "" + - " " + + " " + " " + " " + - " " + + " " + " " + "" ); @@ -408,6 +424,8 @@ public class DeploymentSpecTest { DeploymentSpec spec = DeploymentSpec.fromXml(r); assertEquals("canary", spec.requireInstance("instance1").upgradePolicy().toString()); assertEquals("conservative", spec.requireInstance("instance2").upgradePolicy().toString()); + assertEquals("latest", spec.requireInstance("instance1").upgradeRevision().toString()); + assertEquals("separate", spec.requireInstance("instance2").upgradeRevision().toString()); assertEquals("leading", spec.requireInstance("instance1").upgradeRollout().toString()); assertEquals("separate", spec.requireInstance("instance2").upgradeRollout().toString()); } diff --git a/config-model/src/main/resources/schema/deployment.rnc b/config-model/src/main/resources/schema/deployment.rnc index 819e6b79fbb..3e751a379d4 100644 --- a/config-model/src/main/resources/schema/deployment.rnc +++ b/config-model/src/main/resources/schema/deployment.rnc @@ -52,6 +52,7 @@ ParallelInstances = element parallel { Upgrade = element upgrade { attribute policy { xsd:string }? & + attribute revision { xsd:string }? & attribute rollout { xsd:string }? } -- cgit v1.2.3 From a22135f0b5a9f2149b6c67c70fc6cf197e5c7d98 Mon Sep 17 00:00:00 2001 From: Jon Bratseth Date: Mon, 24 Jan 2022 15:37:52 +0100 Subject: Cleanup: No functional changes --- .../config/application/api/ApplicationPackage.java | 1 - .../com/yahoo/config/model/deploy/DeployState.java | 1 + .../config/model/deploy/SearchDocumentModel.java | 12 +-- .../yahoo/searchdefinition/DefaultRankProfile.java | 8 +- .../com/yahoo/searchdefinition/RankProfile.java | 85 ++++++++++++---------- .../searchdefinition/RankProfileRegistry.java | 15 ++-- .../java/com/yahoo/searchdefinition/Schema.java | 1 + .../yahoo/searchdefinition/derived/Deriver.java | 25 +++---- .../searchdefinition/derived/RankProfileList.java | 10 +-- .../searchdefinition/derived/RawRankProfile.java | 10 +-- .../expressiontransforms/OnnxModelTransformer.java | 4 +- .../processing/DiversitySettingsValidator.java | 2 +- .../processing/FilterFieldNames.java | 2 +- .../processing/MatchPhaseSettingsValidator.java | 2 +- .../processing/RankingExpressionTypeResolver.java | 8 +- .../processing/ReservedFunctionNames.java | 6 +- .../yahoo/vespa/documentmodel/DocumentModel.java | 9 +-- .../com/yahoo/vespa/model/ml/ConvertedModel.java | 2 +- .../src/test/derived/rankprofilemodularity/test.sd | 31 ++++++++ .../test/outside_schema1.profile | 11 +++ .../test/outside_schema2.profile | 11 +++ .../config/server/deploy/ZooKeeperClient.java | 6 +- 22 files changed, 152 insertions(+), 110 deletions(-) (limited to 'config-model-api/src') diff --git a/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationPackage.java b/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationPackage.java index d07df82fda1..7d7fff5bc2c 100644 --- a/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationPackage.java +++ b/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationPackage.java @@ -129,7 +129,6 @@ public interface ApplicationPackage { * Returns the files in a directory as readers. The readers must * be closed by the caller. * - * * @param pathFromRoot the relative path string from the root of the application package * @param suffix the suffix of files to return, or null to return all * @param recurse return files in all subdirectories (recursively) as well diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java b/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java index bb8c4141873..43b008ddab6 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java @@ -470,6 +470,7 @@ public class DeployState implements ConfigDefinitionStore { reindexing); } + // TODO: This should be moved into Application+SchemaBuilder private SearchDocumentModel createSearchDocumentModel(RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles, ValidationParameters validationParameters) { diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/SearchDocumentModel.java b/config-model/src/main/java/com/yahoo/config/model/deploy/SearchDocumentModel.java index aaba9a2b98c..ece3a662d94 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/SearchDocumentModel.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/SearchDocumentModel.java @@ -11,10 +11,11 @@ import java.util.List; import java.util.Map; /** - * Internal helper class to retrieve document model and search definitions. + * Internal helper class to retrieve document model and schemas. * * @author Ulf Lilleengen */ +// TODO: This should be removed in favor of Application public class SearchDocumentModel { private final DocumentModel documentModel; @@ -23,7 +24,6 @@ public class SearchDocumentModel { public SearchDocumentModel(DocumentModel documentModel, List schemas) { this.documentModel = documentModel; this.schemas = schemas; - } public DocumentModel getDocumentModel() { @@ -42,12 +42,4 @@ public class SearchDocumentModel { return new SearchDocumentModel(builder.getModel(), ret); } - public static SearchDocumentModel fromBuilder(SchemaBuilder builder) { - List ret = new ArrayList<>(); - for (Schema schema : builder.getSchemaList()) { - ret.add(new NamedSchema(schema.getName(), schema)); - } - return new SearchDocumentModel(builder.getModel(), ret); - } - } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/DefaultRankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/DefaultRankProfile.java index 56a739ced8b..4444636fbe8 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/DefaultRankProfile.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/DefaultRankProfile.java @@ -40,14 +40,14 @@ public class DefaultRankProfile extends RankProfile { RankSetting setting = super.getRankSetting(fieldOrIndex,type); if (setting != null) return setting; - ImmutableSDField field = getSearch().getConcreteField(fieldOrIndex); + ImmutableSDField field = schema().getConcreteField(fieldOrIndex); if (field != null) { setting = toRankSetting(field,type); if (setting != null) return setting; } - Index index = getSearch().getIndex(fieldOrIndex); + Index index = schema().getIndex(fieldOrIndex); if (index != null) { setting = toRankSetting(index,type); if (setting != null) @@ -89,7 +89,7 @@ public class DefaultRankProfile extends RankProfile { public Set rankSettings() { Set settings = new LinkedHashSet<>(20); settings.addAll(this.rankSettings); - for (ImmutableSDField field : getSearch().allConcreteFields() ) { + for (ImmutableSDField field : schema().allConcreteFields() ) { addSetting(field, RankSetting.Type.WEIGHT, settings); addSetting(field, RankSetting.Type.RANKTYPE, settings); addSetting(field, RankSetting.Type.LITERALBOOST, settings); @@ -97,7 +97,7 @@ public class DefaultRankProfile extends RankProfile { } // Foer settings that really pertains to indexes do the explicit indexes too - for (Index index : getSearch().getExplicitIndices()) { + for (Index index : schema().getExplicitIndices()) { addSetting(index, RankSetting.Type.PREFERBITVECTOR, settings); } return settings; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java index 5f6bde3b791..20234862643 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java @@ -57,11 +57,12 @@ public class RankProfile implements Cloneable { public final static String FIRST_PHASE = "firstphase"; public final static String SECOND_PHASE = "secondphase"; + /** The search definition-unique name of this rank profile */ private final String name; - /** The search definition owning this profile, or null if global (owned by a model) */ - private final ImmutableSchema search; + /** The schema owning this profile, or null if global (owned by a model) */ + private final ImmutableSchema schema; /** The name of the rank profile inherited by this */ private String inheritedName = null; @@ -132,20 +133,6 @@ public class RankProfile implements Cloneable { private final ApplicationPackage applicationPackage; private final DeployLogger deployLogger; - private static class CachedFunctions { - private final Map allRankingExpressionFunctions; - private final ImmutableMap allExpressionFunctions; - CachedFunctions(Map functions) { - allRankingExpressionFunctions = functions; - ImmutableMap.Builder mapBuilder = new ImmutableMap.Builder<>(); - for (var entry : functions.entrySet()) { - ExpressionFunction function = entry.getValue().function(); - mapBuilder.put(function.getName(), function); - } - allExpressionFunctions = mapBuilder.build(); - } - } - /** * Creates a new rank profile for a particular search definition * @@ -156,7 +143,7 @@ public class RankProfile implements Cloneable { */ public RankProfile(String name, Schema schema, RankProfileRegistry rankProfileRegistry, RankingConstants rankingConstants) { this.name = Objects.requireNonNull(name, "name cannot be null"); - this.search = Objects.requireNonNull(schema, "search cannot be null"); + this.schema = Objects.requireNonNull(schema, "search cannot be null"); this.onnxModels = null; this.rankingConstants = rankingConstants; this.rankProfileRegistry = rankProfileRegistry; @@ -172,7 +159,7 @@ public class RankProfile implements Cloneable { public RankProfile(String name, ApplicationPackage applicationPackage, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, RankingConstants rankingConstants, OnnxModels onnxModels) { this.name = Objects.requireNonNull(name, "name cannot be null"); - this.search = null; + this.schema = null; this.rankProfileRegistry = rankProfileRegistry; this.rankingConstants = rankingConstants; this.onnxModels = onnxModels; @@ -180,10 +167,10 @@ public class RankProfile implements Cloneable { this.deployLogger = deployLogger; } - public String getName() { return name; } + public String name() { return name; } /** Returns the search definition owning this, or null if it is global */ - public ImmutableSchema getSearch() { return search; } + public ImmutableSchema schema() { return schema; } /** Returns the application this is part of */ public ApplicationPackage applicationPackage() { @@ -196,19 +183,19 @@ public class RankProfile implements Cloneable { } public Map onnxModels() { - return search != null ? search.onnxModels().asMap() : onnxModels.asMap(); + return schema != null ? schema.onnxModels().asMap() : onnxModels.asMap(); } private Stream allFields() { - if (search == null) return Stream.empty(); + if (schema == null) return Stream.empty(); if (allFieldsList == null) { - allFieldsList = search.allFieldsList(); + allFieldsList = schema.allFieldsList(); } return allFieldsList.stream(); } private Stream allImportedFields() { - return search != null ? search.allImportedFields() : Stream.empty(); + return schema != null ? schema.allImportedFields() : Stream.empty(); } /** @@ -228,9 +215,9 @@ public class RankProfile implements Cloneable { if (inherited == null) { inherited = resolveInherited(); if (inherited == null) { - String msg = "rank-profile '" + getName() + "' inherits '" + inheritedName + - "', but it does not exist anywhere in the inheritance of search '" + - ((getSearch() != null) ? getSearch().getName() : " global rank profiles") + "'."; + String msg = "rank-profile '" + name() + "' inherits '" + inheritedName + + "', but it does not exist anywhere in the inheritance of search '" + + ((schema() != null) ? schema().getName() : " global rank profiles") + "'."; throw new IllegalArgumentException(msg); } else { List children = new ArrayList<>(); @@ -242,9 +229,9 @@ public class RankProfile implements Cloneable { } private String createFullyQualifiedName() { - return (search != null) - ? (search.getName() + "." + getName()) - : getName(); + return (schema != null) + ? (schema.getName() + "." + name()) + : name(); } private void verifyNoInheritanceCycle(List children, RankProfile parent) { @@ -275,8 +262,8 @@ public class RankProfile implements Cloneable { private RankProfile resolveInherited() { if (inheritedName == null) return null; - return (getSearch() != null) - ? resolveInherited(search) + return (schema() != null) + ? resolveInherited(schema) : rankProfileRegistry.getGlobal(inheritedName); } @@ -289,7 +276,7 @@ public class RankProfile implements Cloneable { public boolean inherits(String name) { RankProfile parent = getInherited(); while (parent != null) { - if (parent.getName().equals(name)) + if (parent.name().equals(name)) return true; parent = parent.getInherited(); } @@ -619,7 +606,7 @@ public class RankProfile implements Cloneable { @Override public String toString() { - return "rank profile '" + getName() + "'"; + return "rank profile '" + name() + "'"; } public int getRerankCount() { @@ -819,14 +806,14 @@ public class RankProfile implements Cloneable { private ExpressionFunction parseRankingExpression(String name, List arguments, String expression) throws ParseException { if (expression.trim().length() == 0) - throw new ParseException("Encountered an empty ranking expression in " + getName()+ ", " + name + "."); + throw new ParseException("Encountered an empty ranking expression in " + name() + ", " + name + "."); try (Reader rankingExpressionReader = openRankingExpressionReader(name, expression.trim())) { return new ExpressionFunction(name, arguments, new RankingExpression(name, rankingExpressionReader)); } catch (com.yahoo.searchlib.rankingexpression.parser.ParseException e) { ParseException exception = new ParseException("Could not parse ranking expression '" + expression.trim() + - "' in " + getName()+ ", " + name + "."); + "' in " + name() + ", " + name + "."); throw (ParseException)exception.initCause(e); } catch (IOException e) { @@ -848,10 +835,10 @@ public class RankProfile implements Cloneable { String fileName = extractFileName(expression); File file = new File(fileName); if (!file.isAbsolute() && file.getPath().contains("/")) // See ticket 4102122 - throw new IllegalArgumentException("In " + getName() + ", " + expName + ", ranking references file '" + file + - "' in subdirectory, which is not supported."); + throw new IllegalArgumentException("In " + name() + ", " + expName + ", ranking references file '" + file + + "' in subdirectory, which is not supported."); - return search.getRankingExpression(fileName); + return schema.getRankingExpression(fileName); } /** Shallow clones this */ @@ -888,7 +875,7 @@ public class RankProfile implements Cloneable { return compiled; } catch (IllegalArgumentException e) { - throw new IllegalArgumentException("Rank profile '" + getName() + "' is invalid", e); + throw new IllegalArgumentException("Rank profile '" + name() + "' is invalid", e); } } @@ -1321,4 +1308,22 @@ public class RankProfile implements Cloneable { } + private static class CachedFunctions { + + private final Map allRankingExpressionFunctions; + + private final ImmutableMap allExpressionFunctions; + + CachedFunctions(Map functions) { + allRankingExpressionFunctions = functions; + ImmutableMap.Builder mapBuilder = new ImmutableMap.Builder<>(); + for (var entry : functions.entrySet()) { + ExpressionFunction function = entry.getValue().function(); + mapBuilder.put(function.getName(), function); + } + allExpressionFunctions = mapBuilder.build(); + } + + } + } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfileRegistry.java b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfileRegistry.java index 08ae3d838ec..75c3aa313c0 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfileRegistry.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfileRegistry.java @@ -6,7 +6,6 @@ import com.yahoo.searchdefinition.document.SDDocumentType; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; -import java.util.HashMap; import java.util.HashSet; import java.util.LinkedHashMap; import java.util.List; @@ -44,22 +43,22 @@ public class RankProfileRegistry { /** Adds a rank profile to this registry */ public void add(RankProfile rankProfile) { - String searchName = extractName(rankProfile.getSearch()); - if ( ! rankProfiles.containsKey(searchName)) { - rankProfiles.put(searchName, new LinkedHashMap<>()); + String schemaName = extractName(rankProfile.schema()); + if ( ! rankProfiles.containsKey(schemaName)) { + rankProfiles.put(schemaName, new LinkedHashMap<>()); } checkForDuplicate(rankProfile); - rankProfiles.get(searchName).put(rankProfile.getName(), rankProfile); + rankProfiles.get(schemaName).put(rankProfile.name(), rankProfile); } private void checkForDuplicate(RankProfile rankProfile) { - String rankProfileName = rankProfile.getName(); - RankProfile existingRankProfileWithSameName = rankProfiles.get(extractName(rankProfile.getSearch())).get(rankProfileName); + String rankProfileName = rankProfile.name(); + RankProfile existingRankProfileWithSameName = rankProfiles.get(extractName(rankProfile.schema())).get(rankProfileName); if (existingRankProfileWithSameName == null) return; if ( ! overridableRankProfileNames.contains(rankProfileName)) { throw new IllegalArgumentException("Duplicate rank profile '" + rankProfileName + "' in " + - rankProfile.getSearch()); + rankProfile.schema()); } } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/Schema.java b/config-model/src/main/java/com/yahoo/searchdefinition/Schema.java index ddad67324ba..dec0a061b21 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/Schema.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/Schema.java @@ -124,6 +124,7 @@ public class Schema implements ImmutableSchema { DeployLogger deployLogger, ModelContext.Properties properties) { this(inherited, application, fileRegistry, deployLogger, properties, false); + if (1==1) throw new RuntimeException("df"); this.name = name; } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/Deriver.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/Deriver.java index 14e303522e0..b1fe4942f4a 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/Deriver.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/Deriver.java @@ -11,18 +11,17 @@ import java.util.Collections; import java.util.List; /** - * Auxiliary facade for deriving configs from search definitions + * Facade for deriving configs from schemas * * @author bratseth */ public class Deriver { - public static SchemaBuilder getSearchBuilder(List sds) { + public static SchemaBuilder getSchemaBuilder(List schemas) { SchemaBuilder builder = new SchemaBuilder(); try { - for (String s : sds) { - builder.importFile(s); - } + for (String schema : schemas) + builder.importFile(schema); } catch (ParseException | IOException e) { throw new IllegalArgumentException(e); } @@ -34,22 +33,22 @@ public class Deriver { return getDocumentManagerConfig(Collections.singletonList(sd)); } - public static DocumentmanagerConfig.Builder getDocumentManagerConfig(List sds) { - return new DocumentManager().produce(getSearchBuilder(sds).getModel(), new DocumentmanagerConfig.Builder()); + public static DocumentmanagerConfig.Builder getDocumentManagerConfig(List schemas) { + return new DocumentManager().produce(getSchemaBuilder(schemas).getModel(), new DocumentmanagerConfig.Builder()); } - public static DocumentmanagerConfig.Builder getDocumentManagerConfig(List sds, boolean useV8DocManagerCfg) { + public static DocumentmanagerConfig.Builder getDocumentManagerConfig(List schemas, boolean useV8DocManagerCfg) { return new DocumentManager() .useV8DocManagerCfg(useV8DocManagerCfg) - .produce(getSearchBuilder(sds).getModel(), new DocumentmanagerConfig.Builder()); + .produce(getSchemaBuilder(schemas).getModel(), new DocumentmanagerConfig.Builder()); } - public static DocumenttypesConfig.Builder getDocumentTypesConfig(String sd) { - return getDocumentTypesConfig(Collections.singletonList(sd)); + public static DocumenttypesConfig.Builder getDocumentTypesConfig(String schema) { + return getDocumentTypesConfig(Collections.singletonList(schema)); } - public static DocumenttypesConfig.Builder getDocumentTypesConfig(List sds) { - return new DocumentTypes().produce(getSearchBuilder(sds).getModel(), new DocumenttypesConfig.Builder()); + public static DocumenttypesConfig.Builder getDocumentTypesConfig(List schemas) { + return new DocumentTypes().produce(getSchemaBuilder(schemas).getModel(), new DocumenttypesConfig.Builder()); } } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/RankProfileList.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/RankProfileList.java index 3081637c975..86c1d478974 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/RankProfileList.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/RankProfileList.java @@ -75,7 +75,7 @@ public class RankProfileList extends Derived implements RankProfilesConfig.Produ private boolean areDependenciesReady(RankProfile rank, RankProfileRegistry registry) { return (rank.getInheritedName() == null) || rankProfiles.containsKey(rank.getInheritedName()) || - (rank.getSearch() != null && registry.resolve(rank.getSearch().getDocument(), rank.getInheritedName()) != null); + (rank.schema() != null && registry.resolve(rank.schema().getDocument(), rank.getInheritedName()) != null); } private void deriveRankProfiles(RankProfileRegistry rankProfileRegistry, @@ -92,7 +92,7 @@ public class RankProfileList extends Derived implements RankProfilesConfig.Produ } Map remaining = new LinkedHashMap<>(); - rankProfileRegistry.rankProfilesOf(schema).forEach(rank -> remaining.put(rank.getName(), rank)); + rankProfileRegistry.rankProfilesOf(schema).forEach(rank -> remaining.put(rank.name(), rank)); remaining.remove("default"); while (!remaining.isEmpty()) { List ready = new ArrayList<>(); @@ -100,7 +100,7 @@ public class RankProfileList extends Derived implements RankProfilesConfig.Produ if (areDependenciesReady(rank, rankProfileRegistry)) ready.add(rank); }); processRankProfiles(ready, queryProfiles, importedModels, schema, attributeFields, deployProperties, executor); - ready.forEach(rank -> remaining.remove(rank.getName())); + ready.forEach(rank -> remaining.remove(rank.name())); } } private void processRankProfiles(List ready, @@ -116,8 +116,8 @@ public class RankProfileList extends Derived implements RankProfilesConfig.Produ onnxModels.add(rank.onnxModels()); } - futureRawRankProfiles.put(rank.getName(), executor.submit(() -> new RawRankProfile(rank, largeRankExpressions, queryProfiles, importedModels, - attributeFields, deployProperties))); + futureRawRankProfiles.put(rank.name(), executor.submit(() -> new RawRankProfile(rank, largeRankExpressions, queryProfiles, importedModels, + attributeFields, deployProperties))); } try { for (Future rawFuture : futureRawRankProfiles.values()) { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java index f775c4e697d..a64f0939677 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java @@ -62,7 +62,7 @@ public class RawRankProfile implements RankProfilesConfig.Producer { public RawRankProfile(RankProfile rankProfile, LargeRankExpressions largeExpressions, QueryProfileRegistry queryProfiles, ImportedMlModels importedModels, AttributeFields attributeFields, ModelContext.Properties deployProperties) { - this.name = rankProfile.getName(); + this.name = rankProfile.name(); compressedProperties = compress(new Deriver(rankProfile.compile(queryProfiles, importedModels), attributeFields, deployProperties).derive(largeExpressions)); } @@ -158,7 +158,7 @@ public class RawRankProfile implements RankProfilesConfig.Producer { */ Deriver(RankProfile compiled, AttributeFields attributeFields, ModelContext.Properties deployProperties) { - rankprofileName = compiled.getName(); + rankprofileName = compiled.name(); attributeTypes = compiled.getAttributeTypes(); queryFeatureTypes = compiled.getQueryFeatureTypes(); firstPhaseRanking = compiled.getFirstPhaseRanking(); @@ -447,8 +447,8 @@ public class RawRankProfile implements RankProfilesConfig.Producer { } private void deriveOnnxModelFunctionsAndFeatures(RankProfile rankProfile) { - if (rankProfile.getSearch() == null) return; - if (rankProfile.getSearch().onnxModels().asMap().isEmpty()) return; + if (rankProfile.schema() == null) return; + if (rankProfile.schema().onnxModels().asMap().isEmpty()) return; replaceOnnxFunctionInputs(rankProfile); replaceImplicitOnnxConfigFeatures(summaryFeatures, rankProfile); replaceImplicitOnnxConfigFeatures(matchFeatures, rankProfile); @@ -457,7 +457,7 @@ public class RawRankProfile implements RankProfilesConfig.Producer { private void replaceOnnxFunctionInputs(RankProfile rankProfile) { Set functionNames = rankProfile.getFunctions().keySet(); if (functionNames.isEmpty()) return; - for (OnnxModel onnxModel: rankProfile.getSearch().onnxModels().asMap().values()) { + for (OnnxModel onnxModel: rankProfile.schema().onnxModels().asMap().values()) { for (Map.Entry mapping : onnxModel.getInputMap().entrySet()) { String source = mapping.getValue(); if (functionNames.contains(source)) { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/OnnxModelTransformer.java b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/OnnxModelTransformer.java index 6d9f4cdec92..35ee9ddb9ed 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/OnnxModelTransformer.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/OnnxModelTransformer.java @@ -49,12 +49,12 @@ public class OnnxModelTransformer extends ExpressionTransformer sds = app.getSchemas(); - if (sds.isEmpty()) return; + Collection schemas = app.getSchemas(); + if (schemas.isEmpty()) return; Path zkPath = getZooKeeperAppPath(USERAPP_ZK_SUBPATH).append(SCHEMAS_DIR); curator.create(zkPath); // Ensures that ranking expressions and other files are also written writeDir(app.getFile(ApplicationPackage.SEARCH_DEFINITIONS_DIR), zkPath, false); writeDir(app.getFile(ApplicationPackage.SCHEMAS_DIR), zkPath, false); - for (NamedReader sd : sds) { + for (NamedReader sd : schemas) { curator.set(zkPath.append(sd.getName()), Utf8.toBytes(com.yahoo.io.IOUtils.readAll(sd.getReader()))); sd.getReader().close(); } -- cgit v1.2.3 From c665b822bc3b1a7b8543e2cc3f31f26c28b258bc Mon Sep 17 00:00:00 2001 From: Henning Baldersheim Date: Mon, 24 Jan 2022 16:50:06 +0100 Subject: GC feature flag interfaces no longer in use. --- .../src/main/java/com/yahoo/config/model/api/ModelContext.java | 9 --------- 1 file changed, 9 deletions(-) (limited to 'config-model-api/src') diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java index 14cd6da1c2e..672414f2d47 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java @@ -82,18 +82,13 @@ public interface ModelContext { @ModelFeatureFlag(owners = {"baldersheim"}) default boolean skipMbusReplyThread() { throw new UnsupportedOperationException("TODO specify default value"); } @ModelFeatureFlag(owners = {"baldersheim"}) default boolean useAsyncMessageHandlingOnSchedule() { throw new UnsupportedOperationException("TODO specify default value"); } @ModelFeatureFlag(owners = {"baldersheim"}) default double feedConcurrency() { throw new UnsupportedOperationException("TODO specify default value"); } - @ModelFeatureFlag(owners = {"baldersheim"}, removeAfter = "7.527") default int metricsproxyNumThreads() { return defaultPoolNumThreads(); } @ModelFeatureFlag(owners = {"baldersheim"}) default int defaultPoolNumThreads() { return 2; } @ModelFeatureFlag(owners = {"baldersheim"}) default int availableProcessors() { return 2; } - @ModelFeatureFlag(owners = {"baldersheim"}, removeAfter = "7.527") default int largeRankExpressionLimit() { return 8192; } @ModelFeatureFlag(owners = {"baldersheim"}) default int maxUnCommittedMemory() { return 130000; } @ModelFeatureFlag(owners = {"baldersheim"}) default int maxConcurrentMergesPerNode() { return 16; } @ModelFeatureFlag(owners = {"baldersheim"}) default int maxMergeQueueSize() { return 100; } - @ModelFeatureFlag(owners = {"vekterli", "geirst"}, removeAfter = "7.528.3") default boolean ignoreMergeQueueLimit() { return true; } @ModelFeatureFlag(owners = {"baldersheim"}) default boolean containerDumpHeapOnShutdownTimeout() { throw new UnsupportedOperationException("TODO specify default value"); } @ModelFeatureFlag(owners = {"baldersheim"}) default double containerShutdownTimeout() { throw new UnsupportedOperationException("TODO specify default value"); } - @ModelFeatureFlag(owners = {"baldersheim"}, removeAfter = "7.527") default double diskBloatFactor() { return 0.25; } - @ModelFeatureFlag(owners = {"baldersheim"}, removeAfter = "7.527") default int docstoreCompressionLevel() { return 3; } @ModelFeatureFlag(owners = {"geirst"}) default boolean enableFeedBlockInDistributor() { return true; } @ModelFeatureFlag(owners = {"bjorncs", "tokle"}) default List allowedAthenzProxyIdentities() { return List.of(); } @ModelFeatureFlag(owners = {"vekterli"}) default int maxActivationInhibitedOutOfSyncGroups() { return 0; } @@ -101,17 +96,13 @@ public interface ModelContext { @ModelFeatureFlag(owners = {"hmusum"}) default double resourceLimitDisk() { return 0.8; } @ModelFeatureFlag(owners = {"hmusum"}) default double resourceLimitMemory() { return 0.8; } @ModelFeatureFlag(owners = {"geirst", "vekterli"}) default double minNodeRatioPerGroup() { return 0.0; } - @ModelFeatureFlag(owners = {"geirst", "vekterli"}, removeAfter = "7.528.3") default int distributorMergeBusyWait() { return 1; } - @ModelFeatureFlag(owners = {"vekterli", "geirst"}, removeAfter = "7.528.3") default boolean distributorEnhancedMaintenanceScheduling() { return true; } @ModelFeatureFlag(owners = {"arnej"}) default boolean forwardIssuesAsErrors() { return true; } - @ModelFeatureFlag(owners = {"geirst", "vekterli"}, removeAfter = "7.528.3") default boolean asyncApplyBucketDiff() { return true; } @ModelFeatureFlag(owners = {"arnej"}) default boolean ignoreThreadStackSizes() { return false; } @ModelFeatureFlag(owners = {"vekterli", "geirst"}) default boolean unorderedMergeChaining() { return true; } @ModelFeatureFlag(owners = {"arnej"}) default boolean useV8GeoPositions() { return false; } @ModelFeatureFlag(owners = {"arnej", "baldersheim"}) default boolean useV8DocManagerCfg() { return false; } @ModelFeatureFlag(owners = {"baldersheim", "geirst", "toregge"}) default int maxCompactBuffers() { return 1; } @ModelFeatureFlag(owners = {"hmusum"}) default boolean failDeploymentWithInvalidJvmOptions() { return false; } - @ModelFeatureFlag(owners = {"baldersheim"}, removeAfter = "7.527") default double tlsSizeFraction() { return 0.02; } @ModelFeatureFlag(owners = {"arnej", "andreer"}) default List ignoredHttpUserAgents() { return List.of(); } @ModelFeatureFlag(owners = {"bjorncs"}) default boolean enableServerOcspStapling() { return false; } @ModelFeatureFlag(owners = {"vekterli"}) default String persistenceAsyncThrottling() { throw new UnsupportedOperationException("TODO specify default value"); } -- cgit v1.2.3 From e68f0aa6f7cb59440f638dcb87e0d27bf53ce9a8 Mon Sep 17 00:00:00 2001 From: Tor Brede Vekterli Date: Tue, 25 Jan 2022 14:28:47 +0100 Subject: Add feature flag for merge throttling policy Two options; `STATIC` (default) and `DYNAMIC`. Does not currently set any window size configs based on the existing feature flag for max pending, so the config defaults will be used for these when dynamic throttling is enabled. --- .../java/com/yahoo/config/model/api/ModelContext.java | 1 + .../com/yahoo/config/model/deploy/TestProperties.java | 7 +++++++ .../content/storagecluster/StorServerProducer.java | 12 ++++++++++++ .../yahoo/vespa/model/content/StorageClusterTest.java | 19 +++++++++++++++++++ .../vespa/config/server/deploy/ModelContextImpl.java | 2 ++ flags/src/main/java/com/yahoo/vespa/flags/Flags.java | 8 ++++++++ 6 files changed, 49 insertions(+) (limited to 'config-model-api/src') diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java index 672414f2d47..3bb6c7c0511 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java @@ -106,6 +106,7 @@ public interface ModelContext { @ModelFeatureFlag(owners = {"arnej", "andreer"}) default List ignoredHttpUserAgents() { return List.of(); } @ModelFeatureFlag(owners = {"bjorncs"}) default boolean enableServerOcspStapling() { return false; } @ModelFeatureFlag(owners = {"vekterli"}) default String persistenceAsyncThrottling() { throw new UnsupportedOperationException("TODO specify default value"); } + @ModelFeatureFlag(owners = {"vekterli"}) default String mergeThrottlingPolicy() { throw new UnsupportedOperationException("TODO specify default value"); } @ModelFeatureFlag(owners = {"arnej"}) default boolean useQrserverServiceName() { return true; } } diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java index 1478854aecf..3aadf30e2f9 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java @@ -71,6 +71,7 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea private int maxCompactBuffers = 1; private boolean failDeploymentWithInvalidJvmOptions = false; private String persistenceAsyncThrottling = "UNLIMITED"; + private String mergeThrottlingPolicy = "STATIC"; @Override public ModelContext.FeatureFlags featureFlags() { return this; } @Override public boolean multitenant() { return multitenant; } @@ -121,6 +122,7 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea @Override public int maxCompactBuffers() { return maxCompactBuffers; } @Override public boolean failDeploymentWithInvalidJvmOptions() { return failDeploymentWithInvalidJvmOptions; } @Override public String persistenceAsyncThrottling() { return persistenceAsyncThrottling; } + @Override public String mergeThrottlingPolicy() { return mergeThrottlingPolicy; } public TestProperties maxUnCommittedMemory(int maxUnCommittedMemory) { this.maxUnCommittedMemory = maxUnCommittedMemory; @@ -313,6 +315,11 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea return this; } + public TestProperties setMergeThrottlingPolicy(String policy) { + this.mergeThrottlingPolicy = policy; + return this; + } + public static class Spec implements ConfigServerSpec { private final String hostName; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/StorServerProducer.java b/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/StorServerProducer.java index 6f9166f4493..2fca964a995 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/StorServerProducer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/StorServerProducer.java @@ -32,6 +32,7 @@ public class StorServerProducer implements StorServerConfig.Producer { private Integer maxMergesPerNode; private Integer queueSize; private Integer bucketDBStripeBits; + private StorServerConfig.Merge_throttling_policy.Type.Enum mergeThrottlingPolicyType; private StorServerProducer setMaxMergesPerNode(Integer value) { if (value != null) { @@ -50,10 +51,19 @@ public class StorServerProducer implements StorServerConfig.Producer { return this; } + private static StorServerConfig.Merge_throttling_policy.Type.Enum toThrottlePolicyType(String policyType) { + try { + return StorServerConfig.Merge_throttling_policy.Type.Enum.valueOf(policyType); + } catch (Throwable t) { + return StorServerConfig.Merge_throttling_policy.Type.STATIC; + } + } + StorServerProducer(String clusterName, ModelContext.FeatureFlags featureFlags) { this.clusterName = clusterName; maxMergesPerNode = featureFlags.maxConcurrentMergesPerNode(); queueSize = featureFlags.maxMergeQueueSize(); + mergeThrottlingPolicyType = toThrottlePolicyType(featureFlags.mergeThrottlingPolicy()); } @Override @@ -73,5 +83,7 @@ public class StorServerProducer implements StorServerConfig.Producer { if (bucketDBStripeBits != null) { builder.content_node_bucket_db_stripe_bits(bucketDBStripeBits); } + // TODO set throttle policy params based on existing or separate flags + builder.merge_throttling_policy(new StorServerConfig.Merge_throttling_policy.Builder().type(mergeThrottlingPolicyType)); } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java index 06b907ca2c2..2953b0cb32d 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java @@ -159,6 +159,25 @@ public class StorageClusterTest { assertEquals(1919, config.max_merge_queue_size()); } + @Test + public void merge_throttling_policy_config_defaults_to_static() { + var config = configFromProperties(new TestProperties()); + assertEquals(StorServerConfig.Merge_throttling_policy.Type.STATIC, config.merge_throttling_policy().type()); + } + + @Test + public void merge_throttling_policy_config_is_derived_from_flag() { + var config = configFromProperties(new TestProperties().setMergeThrottlingPolicy("STATIC")); + assertEquals(StorServerConfig.Merge_throttling_policy.Type.STATIC, config.merge_throttling_policy().type()); + + config = configFromProperties(new TestProperties().setMergeThrottlingPolicy("DYNAMIC")); + assertEquals(StorServerConfig.Merge_throttling_policy.Type.DYNAMIC, config.merge_throttling_policy().type()); + + // Invalid enum values fall back to the default + config = configFromProperties(new TestProperties().setMergeThrottlingPolicy("UKULELE")); + assertEquals(StorServerConfig.Merge_throttling_policy.Type.STATIC, config.merge_throttling_policy().type()); + } + @Test public void testVisitors() { StorVisitorConfig.Builder builder = new StorVisitorConfig.Builder(); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java index fddb80610cd..af092aec2eb 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java @@ -201,6 +201,7 @@ public class ModelContextImpl implements ModelContext { private final List ignoredHttpUserAgents; private final boolean enableServerOcspStapling; private final String persistenceAsyncThrottling; + private final String mergeThrottlingPolicy; private final boolean useQrserverServiceName; public FeatureFlags(FlagSource source, ApplicationId appId) { @@ -241,6 +242,7 @@ public class ModelContextImpl implements ModelContext { this.ignoredHttpUserAgents = flagValue(source, appId, PermanentFlags.IGNORED_HTTP_USER_AGENTS); this.enableServerOcspStapling = flagValue(source, appId, Flags.ENABLE_SERVER_OCSP_STAPLING); this.persistenceAsyncThrottling = flagValue(source, appId, Flags.PERSISTENCE_ASYNC_THROTTLING); + this.mergeThrottlingPolicy = flagValue(source, appId, Flags.MERGE_THROTTLING_POLICY); this.useQrserverServiceName = flagValue(source, appId, Flags.USE_QRSERVER_SERVICE_NAME); } diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index 30a7c2cd600..470bf9a0c1c 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -357,6 +357,14 @@ public class Flags { "Triggers restart, takes effect immediately", ZONE_ID, APPLICATION_ID); + public static final UnboundStringFlag MERGE_THROTTLING_POLICY = defineStringFlag( + "merge-throttling-policy", "STATIC", + List.of("vekterli"), "2022-01-25", "2022-05-01", + "Sets the policy used for merge throttling on the content nodes. " + + "Valid values: STATIC, DYNAMIC", + "Takes effect at redeployment", + ZONE_ID, APPLICATION_ID); + public static final UnboundBooleanFlag CHECK_CONFIG_CONVERGENCE_BEFORE_RESTARTING = defineFeatureFlag( "check-config-convergence-before-restart", false, List.of("hmusum"), "2022-01-16", "2022-02-16", -- cgit v1.2.3 From 5869796c08003f41f0ac8e4738414d136b075b1d Mon Sep 17 00:00:00 2001 From: Jon Bratseth Date: Wed, 26 Jan 2022 13:26:47 +0100 Subject: Modular rank profiles --- .../config/application/api/ApplicationPackage.java | 5 +- .../config/model/test/MockApplicationPackage.java | 16 ++++- .../yahoo/searchdefinition/ApplicationBuilder.java | 79 ++++++++++++++-------- .../com/yahoo/searchdefinition/RankProfile.java | 4 +- .../yahoo/searchdefinition/derived/Deriver.java | 2 +- .../derived/rankprofilemodularity/ilscripts.cfg | 0 .../rankprofilemodularity/rank-profiles.cfg | 30 ++++++++ .../src/test/derived/rankprofilemodularity/test.sd | 4 +- .../test/outside_schema1.profile | 6 +- .../AttributeSettingsTestCase.java | 24 +++---- .../yahoo/searchdefinition/DiversityTestCase.java | 6 +- .../searchdefinition/RankProfileTestCase.java | 10 +-- .../searchdefinition/RankPropertiesTestCase.java | 4 +- .../searchdefinition/RankingConstantTest.java | 12 ++-- .../RankingExpressionConstantsTestCase.java | 12 ++-- .../RankingExpressionInliningTestCase.java | 8 +-- .../RankingExpressionLoopDetectionTestCase.java | 16 ++--- .../RankingExpressionShadowingTestCase.java | 8 +-- .../RankingExpressionValidationTestCase.java | 2 +- .../searchdefinition/SchemaImporterTestCase.java | 2 +- .../UrlFieldValidationTestCase.java | 2 +- .../derived/AbstractExportingTestCase.java | 2 +- .../derived/ExportingTestCase.java | 2 +- .../derived/InheritanceTestCase.java | 10 +-- .../derived/SimpleInheritTestCase.java | 2 +- .../derived/StructInheritanceTestCase.java | 2 +- .../derived/SummaryMapTestCase.java | 2 +- .../searchdefinition/derived/SummaryTestCase.java | 2 +- .../derived/TwoStreamingStructsTestCase.java | 6 +- .../DisallowComplexMapAndWsetKeyTypesTestCase.java | 2 +- .../processing/FastAccessValidatorTest.java | 2 +- .../processing/ImplicitSchemaFieldsTestCase.java | 2 +- .../processing/ImportedFieldsTestCase.java | 8 +-- .../MatchedElementsOnlyResolverTestCase.java | 2 +- .../processing/RankProfileSearchFixture.java | 2 +- .../RankingExpressionTypeResolverTestCase.java | 20 +++--- .../RankingExpressionWithOnnxTestCase.java | 4 +- ...ingExpressionWithTransformerTokensTestCase.java | 2 +- .../processing/ReferenceFieldTestCase.java | 5 +- ...DocumentModelBuilderImportedFieldsTestCase.java | 2 +- .../DocumentModelBuilderReferenceTypeTestCase.java | 2 +- .../DocumentModelBuilderTestCase.java | 4 +- .../vespa/model/search/test/SchemaClusterTest.java | 2 +- .../config/server/deploy/ZooKeeperClient.java | 4 +- .../main/java/com/yahoo/vespa/DocumentGenMojo.java | 2 +- 45 files changed, 206 insertions(+), 139 deletions(-) delete mode 100644 config-model/src/test/derived/rankprofilemodularity/ilscripts.cfg create mode 100644 config-model/src/test/derived/rankprofilemodularity/rank-profiles.cfg (limited to 'config-model-api/src') diff --git a/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationPackage.java b/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationPackage.java index 7d7fff5bc2c..c40ce1ebeb1 100644 --- a/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationPackage.java +++ b/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationPackage.java @@ -25,9 +25,6 @@ import java.util.jar.JarEntry; * Represents an application package, that is, used as input when creating a VespaModel and as * a general reference to all contents in an application. * - * The class hides detail as to whether the source is local files or ZooKeeper - * data in config server. - * * @author Vegard Havdal */ public interface ApplicationPackage { @@ -140,7 +137,7 @@ public interface ApplicationPackage { /** Same as getFiles(pathFromRoot, suffix, false) */ default List getFiles(Path pathFromRoot, String suffix) { - return getFiles(pathFromRoot,suffix,false); + return getFiles(pathFromRoot, suffix, false); } /** Returns the major version this application is valid for, or empty if it is valid for all versions */ diff --git a/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java b/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java index 72a8f004b22..637bbe91e67 100644 --- a/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java +++ b/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java @@ -33,6 +33,7 @@ import java.io.InputStream; import java.io.Reader; import java.io.StringReader; import java.io.UncheckedIOException; +import java.nio.file.Files; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -145,7 +146,20 @@ public class MockApplicationPackage implements ApplicationPackage { @Override public List getFiles(Path dir, String fileSuffix, boolean recurse) { - return new ArrayList<>(); + try { + File dirFile = new File(root, dir.getName()); + if ( ! dirFile.exists()) return List.of(); + if (recurse) throw new RuntimeException("Recurse not implemented"); + List readers = new ArrayList<>(); + for (var i = Files.list(dirFile.toPath()).filter(p -> p.getFileName().toString().endsWith(fileSuffix)).iterator(); i.hasNext(); ) { + var file = i.next(); + readers.add(new NamedReader(file.toString(), IOUtils.createReader(file.toString()))); + } + return readers; + } + catch (IOException e) { + throw new RuntimeException(e); + } } @Override diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/ApplicationBuilder.java b/config-model/src/main/java/com/yahoo/searchdefinition/ApplicationBuilder.java index d75d940d491..ac4ac649a28 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/ApplicationBuilder.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/ApplicationBuilder.java @@ -12,6 +12,7 @@ import com.yahoo.config.model.test.MockApplicationPackage; import com.yahoo.document.DocumentTypeManager; import com.yahoo.io.IOUtils; import com.yahoo.io.reader.NamedReader; +import com.yahoo.path.Path; import com.yahoo.search.query.profile.QueryProfileRegistry; import com.yahoo.search.query.profile.config.QueryProfileXMLReader; import com.yahoo.searchdefinition.parser.ParseException; @@ -27,12 +28,10 @@ import java.io.File; import java.io.IOException; import java.io.Reader; import java.nio.file.Files; -import java.nio.file.Path; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.HashSet; -import java.util.Iterator; import java.util.List; import java.util.Set; @@ -40,7 +39,7 @@ import java.util.Set; * Application builder. Usage: * 1) Add all schemas, using the addXXX() methods, * 2) provide the available rank types and rank expressions, using the setRankXXX() methods, - * 3) invoke the {@link #build()} method + * 3) invoke the {@link #build} method */ public class ApplicationBuilder { @@ -143,13 +142,9 @@ public class ApplicationBuilder { return addSchema(IOUtils.readFile(file), file.getAbsoluteFile().getParent()); } - private Schema addSchemaFile(Path file) throws IOException, ParseException { - return addSchemaFile(file.toString()); - } - /** * Reads and parses the schema string provided by the given reader. Once all schemas have been - * imported, call {@link #build()}. + * imported, call {@link #build}. * * @param reader the reader whose content to import */ @@ -187,12 +182,12 @@ public class ApplicationBuilder { return addSchema(string, null); } - private Schema addSchema(String schemaString, String schemaDir) throws ParseException { - return add(createSchema(schemaString, schemaDir)); + private Schema addSchema(String schemaString, String schemaPath) throws ParseException { + return add(createSchema(schemaString, schemaPath)); } /** - * Registers the given schema to the application to be built during {@link #build()}. A + * Registers the given schema to the application to be built during {@link #build}. A * {@link Schema} object is considered to be "raw" if it has not already been processed. This is the case for most * programmatically constructed schemas used in unit tests. * @@ -206,13 +201,18 @@ public class ApplicationBuilder { return schema; } - private Schema createSchema(String schemaString, String schemaDir) throws ParseException { + private Schema createSchema(String schemaString, String schemaPath) throws ParseException { + Schema schema = parseSchema(schemaString, schemaPath); + addRankProfileFiles(schema, schemaPath); + return schema; + } + + private Schema parseSchema(String schemaString, String schemaPath) throws ParseException { SimpleCharStream stream = new SimpleCharStream(schemaString); try { - Schema schema = new SDParser(stream, applicationPackage, fileRegistry, deployLogger, properties, + return new SDParser(stream, applicationPackage, fileRegistry, deployLogger, properties, rankProfileRegistry, documentsOnly) - .schema(documentTypeManager, schemaDir); - return schema; + .schema(documentTypeManager, schemaPath); } catch (TokenMgrException e) { throw new ParseException("Unknown symbol: " + e.getMessage()); } catch (ParseException pe) { @@ -220,16 +220,41 @@ public class ApplicationBuilder { } } - /** - * Processes and finalizes the schemas of this. - * Only for testing. - * - * @throws IllegalStateException Thrown if this method has already been called. - */ - public Application build() { - return build(true); + private void addRankProfileFiles(Schema schema, String schemaPath) { + if (applicationPackage == null || schemaPath == null) return; + Path rankProfilePath = Path.fromString(schemaPath).append(schema.getName()); + System.out.println("Adding external rank profiles for " + schema + ": Have " + + applicationPackage.getFiles(rankProfilePath, ".profile").size() + + " files in " + rankProfilePath); + for (NamedReader reader : applicationPackage.getFiles(rankProfilePath, ".profile")) { + parseRankProfile(reader, schema); + } } + /** Parses the rank profile of the given reader and adds it to the rank profile registry for this schema. */ + private void parseRankProfile(NamedReader reader, Schema schema) { + try { + SimpleCharStream stream = new SimpleCharStream(IOUtils.readAll(reader.getReader())); + try { + new SDParser(stream, applicationPackage, fileRegistry, deployLogger, properties, + rankProfileRegistry, documentsOnly) + .rankProfile(schema); + } catch (TokenMgrException e) { + throw new ParseException("Unknown symbol: " + e.getMessage()); + } catch (ParseException pe) { + throw new ParseException(stream.formatException(Exceptions.toMessageString(pe))); + } + } + catch (IOException e) { + throw new IllegalArgumentException("Could not read rank profile " + reader.getName(), e); + } + catch (ParseException e) { + throw new IllegalArgumentException("Could not parse rank profile " + reader.getName(), e); + } + } + + + /** * Processes and finalizes the schemas of this. * @@ -278,7 +303,7 @@ public class ApplicationBuilder { * @param name the name of the schema to return, * or null to return the only one or throw an exception if there are multiple to choose from * @return the built object, or null if none with this name - * @throws IllegalStateException if {@link #build()} has not been called. + * @throws IllegalStateException if {@link #build} has not been called. */ public Schema getSchema(String name) { if (application == null) throw new IllegalStateException("Application not built"); @@ -426,8 +451,8 @@ public class ApplicationBuilder { properties, rankProfileRegistry, queryProfileRegistry); - for (Iterator i = Files.list(new File(dir).toPath()).filter(p -> p.getFileName().toString().endsWith(".sd")).iterator(); i.hasNext(); ) { - builder.addSchemaFile(i.next()); + for (var i = Files.list(new File(dir).toPath()).filter(p -> p.getFileName().toString().endsWith(".sd")).iterator(); i.hasNext(); ) { + builder.addSchemaFile(i.next().toString()); } builder.build(true); return builder; @@ -499,7 +524,7 @@ public class ApplicationBuilder { QueryProfileRegistry queryProfileRegistry) { ApplicationBuilder builder = new ApplicationBuilder(rankProfileRegistry, queryProfileRegistry); builder.add(rawSchema); - builder.build(); + builder.build(true); return builder; } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java index 20234862643..1b6c4aa9459 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java @@ -216,8 +216,8 @@ public class RankProfile implements Cloneable { inherited = resolveInherited(); if (inherited == null) { String msg = "rank-profile '" + name() + "' inherits '" + inheritedName + - "', but it does not exist anywhere in the inheritance of search '" + - ((schema() != null) ? schema().getName() : " global rank profiles") + "'."; + "', but this is not found in " + + ((schema() != null) ? schema() : " global rank profiles"); throw new IllegalArgumentException(msg); } else { List children = new ArrayList<>(); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/Deriver.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/Deriver.java index 475631f9241..bcdc9e656e1 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/Deriver.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/Deriver.java @@ -25,7 +25,7 @@ public class Deriver { } catch (ParseException | IOException e) { throw new IllegalArgumentException(e); } - builder.build(); + builder.build(true); return builder; } diff --git a/config-model/src/test/derived/rankprofilemodularity/ilscripts.cfg b/config-model/src/test/derived/rankprofilemodularity/ilscripts.cfg deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/config-model/src/test/derived/rankprofilemodularity/rank-profiles.cfg b/config-model/src/test/derived/rankprofilemodularity/rank-profiles.cfg new file mode 100644 index 00000000000..8953a8b6bcf --- /dev/null +++ b/config-model/src/test/derived/rankprofilemodularity/rank-profiles.cfg @@ -0,0 +1,30 @@ +rankprofile[].name "default" +rankprofile[].name "unranked" +rankprofile[].fef.property[].name "vespa.rank.firstphase" +rankprofile[].fef.property[].value "value(0)" +rankprofile[].fef.property[].name "vespa.hitcollector.heapsize" +rankprofile[].fef.property[].value "0" +rankprofile[].fef.property[].name "vespa.hitcollector.arraysize" +rankprofile[].fef.property[].value "0" +rankprofile[].fef.property[].name "vespa.dump.ignoredefaultfeatures" +rankprofile[].fef.property[].value "true" +rankprofile[].name "in_schema1" +rankprofile[].fef.property[].name "vespa.rank.firstphase" +rankprofile[].fef.property[].value "nativeRank" +rankprofile[].name "in_schema2" +rankprofile[].fef.property[].name "rankingExpression(fo2).rankingScript" +rankprofile[].fef.property[].value "random" +rankprofile[].fef.property[].name "rankingExpression(f2).rankingScript" +rankprofile[].fef.property[].value "fieldMatch(title) + rankingExpression(fo2)" +rankprofile[].fef.property[].name "vespa.rank.firstphase" +rankprofile[].fef.property[].value "rankingExpression(f2)" +rankprofile[].name "outside_schema1" +rankprofile[].fef.property[].name "rankingExpression(fo1).rankingScript" +rankprofile[].fef.property[].value "now" +rankprofile[].fef.property[].name "vespa.rank.firstphase" +rankprofile[].fef.property[].value "nativeRank" +rankprofile[].name "outside_schema2" +rankprofile[].fef.property[].name "rankingExpression(fo2).rankingScript" +rankprofile[].fef.property[].value "random" +rankprofile[].fef.property[].name "vespa.rank.firstphase" +rankprofile[].fef.property[].value "fieldMatch(title).completeness" diff --git a/config-model/src/test/derived/rankprofilemodularity/test.sd b/config-model/src/test/derived/rankprofilemodularity/test.sd index 29f39bef59d..720e2143f17 100644 --- a/config-model/src/test/derived/rankprofilemodularity/test.sd +++ b/config-model/src/test/derived/rankprofilemodularity/test.sd @@ -18,12 +18,12 @@ schema test { rank-profile in_schema2 inherits outside_schema2 { - function f1() { + function f2() { expression: fieldMatch(title) + fo2 } first-phase { - expression: f1 + expression: f2 } } diff --git a/config-model/src/test/derived/rankprofilemodularity/test/outside_schema1.profile b/config-model/src/test/derived/rankprofilemodularity/test/outside_schema1.profile index 8e9782da119..a8092689b7f 100644 --- a/config-model/src/test/derived/rankprofilemodularity/test/outside_schema1.profile +++ b/config-model/src/test/derived/rankprofilemodularity/test/outside_schema1.profile @@ -1,11 +1,7 @@ -rank-profile outside_schema1 { +rank-profile outside_schema1 inherits in_schema1 { function fo1() { expression: now } - first-phase { - expression: fieldMatch(title).completeness - } - } \ No newline at end of file diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/AttributeSettingsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/AttributeSettingsTestCase.java index 819a6f54b23..7b2ed2d7a7f 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/AttributeSettingsTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/AttributeSettingsTestCase.java @@ -98,15 +98,15 @@ public class AttributeSettingsTestCase extends AbstractSchemaTestCase { assertTrue(attr.isFastAccess()); } - private Schema getSearch(String sd) throws ParseException { + private Schema getSchema(String sd) throws ParseException { ApplicationBuilder builder = new ApplicationBuilder(); builder.addSchema(sd); - builder.build(); + builder.build(true); return builder.getSchema(); } private Attribute getAttributeF(String sd) throws ParseException { - Schema schema = getSearch(sd); + Schema schema = getSchema(sd); SDField field = (SDField) schema.getDocument().getField("f"); return field.getAttributes().get(field.getName()); } @@ -153,14 +153,14 @@ public class AttributeSettingsTestCase extends AbstractSchemaTestCase { @Test public void requireThatMutableCanNotbeSetInDocument() throws ParseException { try { - getSearch("search test {\n" + - " document test {\n" + - " field f type int {\n" + - " indexing: attribute\n" + - " attribute: mutable\n" + - " }\n" + - " }\n" + - "}\n"); + getSchema("search test {\n" + + " document test {\n" + + " field f type int {\n" + + " indexing: attribute\n" + + " attribute: mutable\n" + + " }\n" + + " }\n" + + "}\n"); fail(); } catch (IllegalArgumentException e) { assertEquals("Field 'f' in 'test' can not be marked mutable as it is inside the document clause.", e.getMessage()); @@ -185,7 +185,7 @@ public class AttributeSettingsTestCase extends AbstractSchemaTestCase { } private Schema getSearchWithMutables() throws ParseException { - return getSearch( + return getSchema( "search test {\n" + " document test { \n" + " field a type int { \n" + diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/DiversityTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/DiversityTestCase.java index 6d39a64cb21..ca7313a5c3f 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/DiversityTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/DiversityTestCase.java @@ -42,7 +42,7 @@ public class DiversityTestCase { " }\n" + " }\n" + "}\n"); - builder.build(); + builder.build(true); Schema s = builder.getSchema(); RankProfile.MatchPhaseSettings matchPhase = rankProfileRegistry.get(s, "parent").getMatchPhaseSettings(); RankProfile.DiversitySettings diversity = matchPhase.getDiversity(); @@ -63,7 +63,7 @@ public class DiversityTestCase { ApplicationBuilder builder = getSearchBuilder("field b type predicate { indexing: attribute }"); try { - builder.build(); + builder.build(true); fail("Should throw."); } catch (IllegalArgumentException e) { assertEquals(getMessagePrefix() + "must be single value numeric, or enumerated attribute, but it is 'predicate'", e.getMessage()); @@ -75,7 +75,7 @@ public class DiversityTestCase { ApplicationBuilder builder = getSearchBuilder("field b type array { indexing: attribute }"); try { - builder.build(); + builder.build(true); fail("Should throw."); } catch (IllegalArgumentException e) { assertEquals(getMessagePrefix() + "must be single value numeric, or enumerated attribute, but it is 'Array'", e.getMessage()); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/RankProfileTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/RankProfileTestCase.java index 4ebbf3f18cd..c88e8149e46 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/RankProfileTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/RankProfileTestCase.java @@ -79,7 +79,7 @@ public class RankProfileTestCase extends AbstractSchemaTestCase { builder.build(true); fail(); } catch (IllegalArgumentException e) { - assertEquals("rank-profile 'p1' inherits 'notexist', but it does not exist anywhere in the inheritance of search 'test'.", e.getMessage()); + assertEquals("rank-profile 'p1' inherits 'notexist', but this is not found in schema 'test'", e.getMessage()); } } @@ -166,7 +166,7 @@ public class RankProfileTestCase extends AbstractSchemaTestCase { builder.build(true); fail("Sideways inheritance should have been enforced"); } catch (IllegalArgumentException e) { - assertEquals("rank-profile 'child' inherits 'parent', but it does not exist anywhere in the inheritance of search 'child1'.", e.getMessage()); + assertEquals("rank-profile 'child' inherits 'parent', but this is not found in schema 'child1'", e.getMessage()); } } @@ -249,7 +249,7 @@ public class RankProfileTestCase extends AbstractSchemaTestCase { RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); ApplicationBuilder builder = new ApplicationBuilder(rankProfileRegistry); builder.addSchema(sd); - builder.build(); + builder.build(true); Schema schema = builder.getSchema(); AttributeFields attributeFields = new AttributeFields(schema); verifyRankProfile(rankProfileRegistry.get(schema, "parent"), attributeFields, deployProperties, termwiseLimit); @@ -291,7 +291,7 @@ public class RankProfileTestCase extends AbstractSchemaTestCase { " rank-profile p1 {}", " rank-profile p2 {}", "}")); - builder.build(); + builder.build(true); Schema schema = builder.getSchema(); assertEquals(4, registry.all().size()); @@ -311,7 +311,7 @@ public class RankProfileTestCase extends AbstractSchemaTestCase { " field a type tensor(x[]) { indexing: attribute }", " }", "}")); - builder.build(); + builder.build(true); } catch (IllegalArgumentException e) { assertEquals("Illegal type in field a type tensor(x[]): Dense tensor dimensions must have a size", diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/RankPropertiesTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/RankPropertiesTestCase.java index 963c91c8c23..6de890549c1 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/RankPropertiesTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/RankPropertiesTestCase.java @@ -48,7 +48,7 @@ public class RankPropertiesTestCase extends AbstractSchemaTestCase { " }", " }", "}")); - builder.build(); + builder.build(true); Schema schema = builder.getSchema(); AttributeFields attributeFields = new AttributeFields(schema); @@ -128,7 +128,7 @@ public class RankPropertiesTestCase extends AbstractSchemaTestCase { " }", " }", "}")); - builder.build(); + builder.build(true); Schema schema = builder.getSchema(); RankProfile a = rankProfileRegistry.get(schema, "a"); List operations = a.getMutateOperations(); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/RankingConstantTest.java b/config-model/src/test/java/com/yahoo/searchdefinition/RankingConstantTest.java index 78897bd612f..6eb74dede62 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/RankingConstantTest.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/RankingConstantTest.java @@ -43,7 +43,7 @@ public class RankingConstantTest { " }", "}" )); - schemaBuilder.build(); + schemaBuilder.build(true); Schema schema = schemaBuilder.getSchema(); Iterator constantIterator = schema.rankingConstants().asMap().values().iterator(); @@ -101,7 +101,7 @@ public class RankingConstantTest { " }", "}" )); - schemaBuilder.build(); + schemaBuilder.build(true); Schema schema = schemaBuilder.getSchema(); RankingConstant constant = schema.rankingConstants().asMap().values().iterator().next(); assertEquals("simplename", constant.getFileName()); @@ -120,7 +120,7 @@ public class RankingConstantTest { " }", "}" )); - schemaBuilder.build(); + schemaBuilder.build(true); Schema schema = schemaBuilder.getSchema(); RankingConstant constant = schema.rankingConstants().asMap().values().iterator().next(); assertEquals(RankingConstant.PathType.URI, constant.getPathType()); @@ -140,7 +140,7 @@ public class RankingConstantTest { " }", "}" )); - schemaBuilder.build(); + schemaBuilder.build(true); Schema schema = schemaBuilder.getSchema(); RankingConstant constant = schema.rankingConstants().asMap().values().iterator().next(); assertEquals(RankingConstant.PathType.URI, constant.getPathType()); @@ -160,7 +160,7 @@ public class RankingConstantTest { " }", "}" )); - schemaBuilder.build(); + schemaBuilder.build(true); Schema schema = schemaBuilder.getSchema(); RankingConstant constant = schema.rankingConstants().asMap().values().iterator().next(); assertEquals(RankingConstant.PathType.URI, constant.getPathType()); @@ -180,7 +180,7 @@ public class RankingConstantTest { " }", "}" )); - schemaBuilder.build(); + schemaBuilder.build(true); Schema schema = schemaBuilder.getSchema(); RankingConstant constant = schema.rankingConstants().asMap().values().iterator().next(); assertEquals(RankingConstant.PathType.URI, constant.getPathType()); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionConstantsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionConstantsTestCase.java index 19e471bacd3..4646ff2f394 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionConstantsTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionConstantsTestCase.java @@ -66,7 +66,7 @@ public class RankingExpressionConstantsTestCase extends AbstractSchemaTestCase { " }\n" + "\n" + "}\n"); - builder.build(); + builder.build(true); Schema s = builder.getSchema(); RankProfile parent = rankProfileRegistry.get(s, "parent").compile(queryProfileRegistry, new ImportedMlModels()); assertEquals("0.0", parent.getFirstPhaseRanking().getRoot().toString()); @@ -110,7 +110,7 @@ public class RankingExpressionConstantsTestCase extends AbstractSchemaTestCase { " }\n" + "\n" + "}\n"); - builder.build(); + builder.build(true); Schema s = builder.getSchema(); try { rankProfileRegistry.get(s, "test").compile(new QueryProfileRegistry(), new ImportedMlModels()); @@ -141,7 +141,7 @@ public class RankingExpressionConstantsTestCase extends AbstractSchemaTestCase { " }\n" + "\n" + "}\n"); - builder.build(); + builder.build(true); Schema s = builder.getSchema(); RankProfile profile = rankProfileRegistry.get(s, "test"); assertEquals("safeLog(popShareSlowDecaySignal,-9.21034037)", profile.getFunctions().get("POP_SLOW_SCORE").function().getBody().getRoot().toString()); @@ -169,7 +169,7 @@ public class RankingExpressionConstantsTestCase extends AbstractSchemaTestCase { " }\n" + "\n" + "}\n"); - builder.build(); + builder.build(true); Schema s = builder.getSchema(); RankProfile profile = rankProfileRegistry.get(s, "test"); assertEquals("safeLog(popShareSlowDecaySignal,myValue)", profile.getFunctions().get("POP_SLOW_SCORE").function().getBody().getRoot().toString()); @@ -193,7 +193,7 @@ public class RankingExpressionConstantsTestCase extends AbstractSchemaTestCase { " }\n" + "\n" + "}\n"); - builder.build(); + builder.build(true); Schema s = builder.getSchema(); RankProfile profile = rankProfileRegistry.get(s, "test"); assertEquals("k1 + (k2 + k3) / 1.0E8", @@ -219,7 +219,7 @@ public class RankingExpressionConstantsTestCase extends AbstractSchemaTestCase { " }\n" + "\n" + "}\n"); - builder.build(); + builder.build(true); Schema s = builder.getSchema(); RankProfile profile = rankProfileRegistry.get(s, "test"); assertEquals("0.5 + 50 * (attribute(rating_yelp) - 3)", diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionInliningTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionInliningTestCase.java index ab4cd3d2275..94d25deb16a 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionInliningTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionInliningTestCase.java @@ -65,7 +65,7 @@ public class RankingExpressionInliningTestCase extends AbstractSchemaTestCase { " }\n" + "\n" + "}\n"); - builder.build(); + builder.build(true); Schema s = builder.getSchema(); RankProfile parent = rankProfileRegistry.get(s, "parent").compile(new QueryProfileRegistry(), new ImportedMlModels()); @@ -125,7 +125,7 @@ public class RankingExpressionInliningTestCase extends AbstractSchemaTestCase { " }\n" + "\n" + "}\n"); - builder.build(); + builder.build(true); Schema s = builder.getSchema(); RankProfile parent = rankProfileRegistry.get(s, "parent").compile(new QueryProfileRegistry(), new ImportedMlModels()); @@ -181,7 +181,7 @@ public class RankingExpressionInliningTestCase extends AbstractSchemaTestCase { " }\n" + "\n" + "}\n"); - builder.build(); + builder.build(true); Schema s = builder.getSchema(); RankProfile test = rankProfileRegistry.get(s, "test").compile(new QueryProfileRegistry(), new ImportedMlModels()); @@ -215,7 +215,7 @@ public class RankingExpressionInliningTestCase extends AbstractSchemaTestCase { " }\n" + " }\n" + "}\n"); - builder.build(); + builder.build(true); Schema s = builder.getSchema(); RankProfile test = rankProfileRegistry.get(s, "test").compile(new QueryProfileRegistry(), new ImportedMlModels()); assertEquals("foo(2)", test.getFirstPhaseRanking().getRoot().toString()); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionLoopDetectionTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionLoopDetectionTestCase.java index 1e108a51a07..ff8a9df4076 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionLoopDetectionTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionLoopDetectionTestCase.java @@ -36,7 +36,7 @@ public class RankingExpressionLoopDetectionTestCase { "\n" + "}\n"); try { - builder.build(); + builder.build(true); fail("Excepted exception"); } catch (IllegalArgumentException e) { @@ -71,7 +71,7 @@ public class RankingExpressionLoopDetectionTestCase { "\n" + "}\n"); try { - builder.build(); + builder.build(true); fail("Excepted exception"); } catch (IllegalArgumentException e) { @@ -106,7 +106,7 @@ public class RankingExpressionLoopDetectionTestCase { "\n" + "}\n"); try { - builder.build(); + builder.build(true); fail("Excepted exception"); } catch (IllegalArgumentException e) { @@ -140,7 +140,7 @@ public class RankingExpressionLoopDetectionTestCase { " }\n" + "\n" + "}\n"); - builder.build(); + builder.build(true); } @Test @@ -168,7 +168,7 @@ public class RankingExpressionLoopDetectionTestCase { " }\n" + "\n" + "}\n"); - builder.build(); + builder.build(true); } @Test @@ -191,7 +191,7 @@ public class RankingExpressionLoopDetectionTestCase { " }\n" + " }\n" + "}\n"); - builder.build(); + builder.build(true); } @Test @@ -214,7 +214,7 @@ public class RankingExpressionLoopDetectionTestCase { " }\n" + " }\n" + "}\n"); - builder.build(); + builder.build(true); } @Test @@ -240,7 +240,7 @@ public class RankingExpressionLoopDetectionTestCase { " }\n" + " }\n" + "}\n"); - builder.build(); + builder.build(true); } } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionShadowingTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionShadowingTestCase.java index 76fc7118d45..8354eb59416 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionShadowingTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionShadowingTestCase.java @@ -45,7 +45,7 @@ public class RankingExpressionShadowingTestCase extends AbstractSchemaTestCase { " }\n" + "\n" + "}\n"); - builder.build(); + builder.build(true); Schema s = builder.getSchema(); RankProfile test = rankProfileRegistry.get(s, "test").compile(new QueryProfileRegistry(), new ImportedMlModels()); List> testRankProperties = createRawRankProfile(test, new QueryProfileRegistry(), s).configProperties(); @@ -86,7 +86,7 @@ public class RankingExpressionShadowingTestCase extends AbstractSchemaTestCase { " }\n" + "\n" + "}\n"); - builder.build(); + builder.build(true); Schema s = builder.getSchema(); RankProfile test = rankProfileRegistry.get(s, "test").compile(new QueryProfileRegistry(), new ImportedMlModels()); List> testRankProperties = createRawRankProfile(test, new QueryProfileRegistry(), s).configProperties(); @@ -132,7 +132,7 @@ public class RankingExpressionShadowingTestCase extends AbstractSchemaTestCase { " }\n" + "\n" + "}\n"); - builder.build(); + builder.build(true); Schema s = builder.getSchema(); RankProfile test = rankProfileRegistry.get(s, "test").compile(new QueryProfileRegistry(), new ImportedMlModels()); List> testRankProperties = createRawRankProfile(test, new QueryProfileRegistry(), s).configProperties(); @@ -193,7 +193,7 @@ public class RankingExpressionShadowingTestCase extends AbstractSchemaTestCase { " file: ignored.json\n" + " }\n" + "}\n"); - builder.build(); + builder.build(true); Schema s = builder.getSchema(); RankProfile test = rankProfileRegistry.get(s, "test").compile(queryProfiles, new ImportedMlModels()); List> testRankProperties = createRawRankProfile(test, queryProfiles, s).configProperties(); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionValidationTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionValidationTestCase.java index 48c3e8e74b7..ee3b4ab25e0 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionValidationTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionValidationTestCase.java @@ -45,7 +45,7 @@ public class RankingExpressionValidationTestCase extends AbstractSchemaTestCase " }" + " }" + "}"); - builder.build(); + builder.build(true); return builder.getSchema(); } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/SchemaImporterTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/SchemaImporterTestCase.java index 1d01fd8cafa..8a9300a83eb 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/SchemaImporterTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/SchemaImporterTestCase.java @@ -38,7 +38,7 @@ public class SchemaImporterTestCase extends AbstractSchemaTestCase { RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); ApplicationBuilder sb = new ApplicationBuilder(rankProfileRegistry, new QueryProfileRegistry()); sb.addSchemaFile("src/test/examples/simple.sd"); - sb.build(); + sb.build(true); Schema schema = sb.getSchema(); assertEquals("simple", schema.getName()); assertTrue(schema.hasDocument()); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/UrlFieldValidationTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/UrlFieldValidationTestCase.java index c5ead2b4064..f2d78796553 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/UrlFieldValidationTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/UrlFieldValidationTestCase.java @@ -22,7 +22,7 @@ public class UrlFieldValidationTestCase { " }" + "}"); try { - builder.build(); + builder.build(true); fail("Should have caused an exception"); // success } catch (IllegalArgumentException e) { diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/AbstractExportingTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/AbstractExportingTestCase.java index 7f913ef5b5f..a88ad929e80 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/AbstractExportingTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/AbstractExportingTestCase.java @@ -115,7 +115,7 @@ public abstract class AbstractExportingTestCase extends AbstractSchemaTestCase { * This will fail if the builder contains multiple search definitions. */ protected DerivedConfiguration assertCorrectDeriving(ApplicationBuilder builder, String dirName, DeployLogger logger) throws IOException { - builder.build(); + builder.build(true); DerivedConfiguration derived = derive(dirName, null, new TestProperties(), builder, logger); assertCorrectConfigFiles(dirName); return derived; diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/ExportingTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/ExportingTestCase.java index 48f0c862468..ddb627216b7 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/ExportingTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/ExportingTestCase.java @@ -147,7 +147,7 @@ public class ExportingTestCase extends AbstractExportingTestCase { ApplicationBuilder builder = new ApplicationBuilder(); builder.addSchemaFile(dir + "first.sd"); builder.addSchemaFile(dir + "second.sd"); - builder.build(); + builder.build(true); derive("tensor2", builder, builder.getSchema("second")); assertCorrectConfigFiles("tensor2"); } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/InheritanceTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/InheritanceTestCase.java index 13de3f95fa5..bcf68387294 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/InheritanceTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/InheritanceTestCase.java @@ -44,7 +44,7 @@ public class InheritanceTestCase extends AbstractExportingTestCase { ApplicationBuilder builder = new ApplicationBuilder(); builder.addSchemaFile(dir + "parent.sd"); builder.addSchemaFile(dir + "child.sd"); - builder.build(); + builder.build(true); derive("inheritstruct", builder, builder.getSchema("child")); assertCorrectConfigFiles("inheritstruct"); } @@ -69,7 +69,7 @@ public class InheritanceTestCase extends AbstractExportingTestCase { String fileName = files.get(fileIdx % files.size()); builder.addSchemaFile(dir + fileName); } - builder.build(); + builder.build(true); DocumentmanagerConfig.Builder b = new DocumentmanagerConfig.Builder(); DerivedConfiguration.exportDocuments(new DocumentManager().produce(builder.getModel(), b), outDir.getPath()); DocumentmanagerConfig dc = b.build(); @@ -114,7 +114,7 @@ public class InheritanceTestCase extends AbstractExportingTestCase { ApplicationBuilder builder = new ApplicationBuilder(); builder.addSchemaFile(dir + "parent.sd"); builder.addSchemaFile(dir + "child.sd"); - builder.build(); + builder.build(true); derive("inheritfromparent", builder, builder.getSchema("child")); assertCorrectConfigFiles("inheritfromparent"); } @@ -126,7 +126,7 @@ public class InheritanceTestCase extends AbstractExportingTestCase { builder.addSchemaFile(dir + "grandparent.sd"); builder.addSchemaFile(dir + "parent.sd"); builder.addSchemaFile(dir + "child.sd"); - builder.build(); + builder.build(true); derive("inheritfromgrandparent", builder, builder.getSchema("child")); assertCorrectConfigFiles("inheritfromgrandparent"); } @@ -139,7 +139,7 @@ public class InheritanceTestCase extends AbstractExportingTestCase { builder.addSchemaFile(dir + "father.sd"); builder.addSchemaFile(dir + "mother.sd"); builder.addSchemaFile(dir + "child.sd"); - builder.build(); + builder.build(true); derive("inheritance", builder, builder.getSchema("child")); assertCorrectConfigFiles("inheritance"); } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/SimpleInheritTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/SimpleInheritTestCase.java index 8a4e403072b..d89e5f2c957 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/SimpleInheritTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/SimpleInheritTestCase.java @@ -22,7 +22,7 @@ public class SimpleInheritTestCase extends AbstractExportingTestCase { ApplicationBuilder builder = new ApplicationBuilder(); builder.addSchemaFile(expectedResultsDirName + "parent.sd"); builder.addSchemaFile(expectedResultsDirName + "child.sd"); - builder.build(); + builder.build(true); Schema schema = builder.getSchema("child"); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/StructInheritanceTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/StructInheritanceTestCase.java index c684ec17e42..1f6c70c9383 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/StructInheritanceTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/StructInheritanceTestCase.java @@ -45,7 +45,7 @@ public class StructInheritanceTestCase extends AbstractExportingTestCase { String dir = "src/test/derived/structinheritance/"; ApplicationBuilder builder = new ApplicationBuilder(); builder.addSchemaFile(dir + "bad.sd"); - builder.build(); + builder.build(true); derive("structinheritance", builder, builder.getSchema("bad")); } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/SummaryMapTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/SummaryMapTestCase.java index fa8a31befc4..c0fa3e8311f 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/SummaryMapTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/SummaryMapTestCase.java @@ -196,7 +196,7 @@ public class SummaryMapTestCase extends AbstractSchemaTestCase { field, " }", "}")); - builder.build(); + builder.build(true); return builder.getSchema(); } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/SummaryTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/SummaryTestCase.java index b18b2120212..9a36ef90cd7 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/SummaryTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/SummaryTestCase.java @@ -149,7 +149,7 @@ public class SummaryTestCase extends AbstractSchemaTestCase { " summary other_campaign_ref type reference {}", " }", "}")); - builder.build(); + builder.build(true); return builder.getSchema("ad"); } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/TwoStreamingStructsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/TwoStreamingStructsTestCase.java index 53948e2d5c8..4e629fcb4d8 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/TwoStreamingStructsTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/TwoStreamingStructsTestCase.java @@ -13,6 +13,7 @@ import java.io.IOException; * @author arnej27959 */ public class TwoStreamingStructsTestCase extends AbstractExportingTestCase { + @Test public void testTwoStreamingStructsExporting() throws ParseException, IOException { @@ -20,13 +21,14 @@ public class TwoStreamingStructsTestCase extends AbstractExportingTestCase { ApplicationBuilder builder = new ApplicationBuilder(); builder.addSchemaFile(root + "/streamingstruct.sd"); builder.addSchemaFile(root + "/whatever.sd"); - builder.build(); + builder.build(true); assertCorrectDeriving(builder, builder.getSchema("streamingstruct"), root); builder = new ApplicationBuilder(); builder.addSchemaFile(root + "/streamingstruct.sd"); builder.addSchemaFile(root + "/whatever.sd"); - builder.build(); + builder.build(true); assertCorrectDeriving(builder, builder.getSchema("streamingstruct"), root); } + } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/DisallowComplexMapAndWsetKeyTypesTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/DisallowComplexMapAndWsetKeyTypesTestCase.java index 3bdda1ea0d8..b03aff455c5 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/DisallowComplexMapAndWsetKeyTypesTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/DisallowComplexMapAndWsetKeyTypesTestCase.java @@ -51,7 +51,7 @@ public class DisallowComplexMapAndWsetKeyTypesTestCase { " field a type " + fieldType + " {}\n" + " }\n" + "}\n"); - builder.build(); + builder.build(true); } } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/FastAccessValidatorTest.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/FastAccessValidatorTest.java index 6e01909ea44..0c25cef49a1 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/FastAccessValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/FastAccessValidatorTest.java @@ -55,7 +55,7 @@ public class FastAccessValidatorTest { "For schema 'test': The following attributes have a type that is incompatible " + "with fast-access: predicate_attribute, tensor_attribute, reference_attribute. " + "Predicate, tensor and reference attributes are incompatible with fast-access."); - builder.build(); + builder.build(true); } } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImplicitSchemaFieldsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImplicitSchemaFieldsTestCase.java index cfc1779ded0..b0b9ce81cc7 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImplicitSchemaFieldsTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImplicitSchemaFieldsTestCase.java @@ -84,7 +84,7 @@ public class ImplicitSchemaFieldsTestCase extends AbstractSchemaTestCase { public void testRequireThatDerivedConfigurationWorks() throws IOException, ParseException { ApplicationBuilder sb = new ApplicationBuilder(); sb.addSchemaFile("src/test/examples/nextgen/simple.sd"); - sb.build(); + sb.build(true); assertNotNull(sb.getSchema()); new DerivedConfiguration(sb.getSchema(), sb.getRankProfileRegistry()); } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImportedFieldsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImportedFieldsTestCase.java index e87ecd3294f..b15b81b717d 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImportedFieldsTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImportedFieldsTestCase.java @@ -82,7 +82,7 @@ public class ImportedFieldsTestCase { " }", "}")); builder.addSchema(sdContent); - builder.build(); + builder.build(true); return builder.getSchema("ad"); } @@ -315,7 +315,7 @@ public class ImportedFieldsTestCase { ApplicationBuilder builder = new ApplicationBuilder(); builder.addSchema(parentSdContent); builder.addSchema(sdContent); - builder.build(); + builder.build(true); return builder.getSchema("child"); } @@ -324,7 +324,7 @@ public class ImportedFieldsTestCase { builder.addSchema(grandParentSdContent); builder.addSchema(parentSdContent); builder.addSchema(sdContent); - builder.build(); + builder.build(true); return builder.getSchema("child"); } @@ -522,7 +522,7 @@ public class ImportedFieldsTestCase { "import field ref_parent_b.entries as entries_from_b {}", "}")); - builder.build(); + builder.build(true); return builder; } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/MatchedElementsOnlyResolverTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/MatchedElementsOnlyResolverTestCase.java index ea5102baf17..b48927d58a3 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/MatchedElementsOnlyResolverTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/MatchedElementsOnlyResolverTestCase.java @@ -186,7 +186,7 @@ public class MatchedElementsOnlyResolverTestCase { " }", summary, "}")); - builder.build(); + builder.build(true); return builder.getSchema(); } } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankProfileSearchFixture.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankProfileSearchFixture.java index dbf339dee4b..2d0bdb58122 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankProfileSearchFixture.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankProfileSearchFixture.java @@ -78,7 +78,7 @@ class RankProfileSearchFixture { "\n" + "}"; builder.addSchema(sdContent); - builder.build(); + builder.build(true); schema = builder.getSchema(); } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeResolverTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeResolverTestCase.java index ec8d9bc4261..7f5e8f8753a 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeResolverTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeResolverTestCase.java @@ -48,7 +48,7 @@ public class RankingExpressionTypeResolverTestCase { " }", "}" )); - builder.build(); + builder.build(true); fail("Expected exception"); } catch (IllegalArgumentException expected) { @@ -96,7 +96,7 @@ public class RankingExpressionTypeResolverTestCase { " }", "}" )); - builder.build(); + builder.build(true); fail("Expected exception"); } catch (IllegalArgumentException expected) { @@ -128,7 +128,7 @@ public class RankingExpressionTypeResolverTestCase { " }", "}" )); - builder.build(); + builder.build(true); fail("Expected exception"); } catch (IllegalArgumentException expected) { @@ -158,7 +158,7 @@ public class RankingExpressionTypeResolverTestCase { " }", "}" )); - schemaBuilder.build(); + schemaBuilder.build(true); fail("Expected exception"); } catch (IllegalArgumentException expected) { @@ -194,7 +194,7 @@ public class RankingExpressionTypeResolverTestCase { " }", "}" )); - builder.build(); + builder.build(true); RankProfile profile = builder.getRankProfileRegistry().get(builder.getSchema(), "my_rank_profile"); assertEquals(TensorType.fromSpec("tensor(x[10],y[3])"), @@ -236,7 +236,7 @@ public class RankingExpressionTypeResolverTestCase { " }", "}" )); - builder.build(); + builder.build(true); RankProfile profile = builder.getRankProfileRegistry().get(builder.getSchema(), "my_rank_profile"); assertEquals(TensorType.fromSpec("tensor(x[10],y[1])"), @@ -280,7 +280,7 @@ public class RankingExpressionTypeResolverTestCase { " first-phase { expression: commonfirstphase(eustaticrank) }", " }", "}")); - builder.build(); + builder.build(true); RankProfile profile = builder.getRankProfileRegistry().get(builder.getSchema(), "eurank"); } @@ -320,7 +320,7 @@ public class RankingExpressionTypeResolverTestCase { " }", "}" )); - builder.build(); + builder.build(true); RankProfile profile = builder.getRankProfileRegistry().get(builder.getSchema(), "my_rank_profile"); assertEquals(TensorType.fromSpec("tensor(x[10],y[1])"), @@ -345,7 +345,7 @@ public class RankingExpressionTypeResolverTestCase { " summary-features { test_func_via_func_with_expr }", " }", "}")); - builder.build(); + builder.build(true); RankProfile profile = builder.getRankProfileRegistry().get(builder.getSchema(), "test"); assertEquals(TensorType.fromSpec("tensor(y{})"), summaryFeatures(profile).get("test_func_via_func_with_expr").type(profile.typeContext(builder.getQueryProfileRegistry()))); @@ -378,7 +378,7 @@ public class RankingExpressionTypeResolverTestCase { " }", "}" )); - builder.build(); + builder.build(true); } @Test diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithOnnxTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithOnnxTestCase.java index 09aa09a626b..2a9d64767ba 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithOnnxTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithOnnxTestCase.java @@ -399,8 +399,10 @@ public class RankingExpressionWithOnnxTestCase { @Override public List getFiles(Path path, String suffix) { + File[] files = getFileReference(path).listFiles(); + if (files == null) return List.of(); List readers = new ArrayList<>(); - for (File file : getFileReference(path).listFiles()) { + for (File file : files) { if ( ! file.getName().endsWith(suffix)) continue; try { readers.add(new NamedReader(file.getName(), new FileReader(file))); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTransformerTokensTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTransformerTokensTestCase.java index 1cdc0aef1cc..ffc073ed434 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTransformerTokensTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTransformerTokensTestCase.java @@ -89,7 +89,7 @@ public class RankingExpressionWithTransformerTokensTestCase { "}"; ApplicationBuilder schemaBuilder = new ApplicationBuilder(application, new MockFileRegistry(), new BaseDeployLogger(), new TestProperties(), rankProfileRegistry, queryProfileRegistry); schemaBuilder.addSchema(sdContent); - schemaBuilder.build(); + schemaBuilder.build(true); Schema schema = schemaBuilder.getSchema(); RankProfile rp = rankProfileRegistry.get(schema, "my_profile"); return new RankProfileTransformContext(rp, queryProfileRegistry, Collections.emptyMap(), null, Collections.emptyMap(), Collections.emptyMap()); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ReferenceFieldTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ReferenceFieldTestCase.java index a2da13b3749..d60cb9040d8 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ReferenceFieldTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ReferenceFieldTestCase.java @@ -49,7 +49,7 @@ public class ReferenceFieldTestCase { builder.addSchema(campaignSdContent); builder.addSchema(salespersonSdContent); builder.addSchema(adSdContent); - builder.build(); + builder.build(true); Schema schema = builder.getSchema("ad"); assertSearchContainsReferenceField("campaign_ref", "campaign", schema.getDocument()); assertSearchContainsReferenceField("salesperson_ref", "salesperson", schema.getDocument()); @@ -74,7 +74,7 @@ public class ReferenceFieldTestCase { builder.addSchema(adSdContent); exceptionRule.expect(DocumentGraphValidator.DocumentGraphException.class); exceptionRule.expectMessage("Document dependency cycle detected: campaign->ad->campaign."); - builder.build(); + builder.build(true); } private static void assertSearchContainsReferenceField(String expectedFieldname, @@ -87,4 +87,5 @@ public class ReferenceFieldTestCase { ReferenceDataType refField = (ReferenceDataType) dataType; assertEquals(referencedDocType, refField.getTargetType().getName()); } + } diff --git a/config-model/src/test/java/com/yahoo/vespa/documentmodel/DocumentModelBuilderImportedFieldsTestCase.java b/config-model/src/test/java/com/yahoo/vespa/documentmodel/DocumentModelBuilderImportedFieldsTestCase.java index ad8e096e39c..33d84cf4313 100644 --- a/config-model/src/test/java/com/yahoo/vespa/documentmodel/DocumentModelBuilderImportedFieldsTestCase.java +++ b/config-model/src/test/java/com/yahoo/vespa/documentmodel/DocumentModelBuilderImportedFieldsTestCase.java @@ -47,7 +47,7 @@ public class DocumentModelBuilderImportedFieldsTestCase extends AbstractReferenc } public DocumentModel build(String adSdContent) throws ParseException { builder.addSchema(adSdContent); - builder.build(); + builder.build(true); return builder.getModel(); } } diff --git a/config-model/src/test/java/com/yahoo/vespa/documentmodel/DocumentModelBuilderReferenceTypeTestCase.java b/config-model/src/test/java/com/yahoo/vespa/documentmodel/DocumentModelBuilderReferenceTypeTestCase.java index febe0ecc8cd..00f39a23a8f 100644 --- a/config-model/src/test/java/com/yahoo/vespa/documentmodel/DocumentModelBuilderReferenceTypeTestCase.java +++ b/config-model/src/test/java/com/yahoo/vespa/documentmodel/DocumentModelBuilderReferenceTypeTestCase.java @@ -71,7 +71,7 @@ public class DocumentModelBuilderReferenceTypeTestCase extends AbstractReference } public DocumentModel build(String adSdContent) throws ParseException { builder.addSchema(adSdContent); - builder.build(); + builder.build(true); return builder.getModel(); } } diff --git a/config-model/src/test/java/com/yahoo/vespa/documentmodel/DocumentModelBuilderTestCase.java b/config-model/src/test/java/com/yahoo/vespa/documentmodel/DocumentModelBuilderTestCase.java index 857a1499264..dc88590a198 100644 --- a/config-model/src/test/java/com/yahoo/vespa/documentmodel/DocumentModelBuilderTestCase.java +++ b/config-model/src/test/java/com/yahoo/vespa/documentmodel/DocumentModelBuilderTestCase.java @@ -38,7 +38,7 @@ public class DocumentModelBuilderTestCase extends AbstractSchemaTestCase { ApplicationBuilder search = new ApplicationBuilder(); search.addSchemaFile("src/test/configmodel/types/other_doc.sd"); search.addSchemaFile("src/test/configmodel/types/type_with_doc_field.sd"); - search.build(); + search.build(true); DocumentModel model = search.getModel(); DocumenttypesConfig.Builder documenttypesCfg = new DocumentTypes().produce(model, new DocumenttypesConfig.Builder()); @@ -52,7 +52,7 @@ public class DocumentModelBuilderTestCase extends AbstractSchemaTestCase { search.addSchemaFile("src/test/cfg/search/data/travel/schemas/TTData.sd"); search.addSchemaFile("src/test/cfg/search/data/travel/schemas/TTEdge.sd"); search.addSchemaFile("src/test/cfg/search/data/travel/schemas/TTPOI.sd"); - search.build(); + search.build(true); } private DocumentModel createAndTestModel(String sd) throws IOException, ParseException { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaClusterTest.java index 27c2e5f8937..e58c29cc4fd 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaClusterTest.java @@ -60,7 +60,7 @@ public class SchemaClusterTest { ApplicationBuilder builder = new ApplicationBuilder(); builder.add(schema1); builder.add(schema2); - builder.build(); + builder.build(true); } @Test diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClient.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClient.java index 653f0173ba6..0acf32d79a7 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClient.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClient.java @@ -96,8 +96,8 @@ public class ZooKeeperClient { Path zkPath = getZooKeeperAppPath(USERAPP_ZK_SUBPATH).append(SCHEMAS_DIR); curator.create(zkPath); // Ensures that ranking expressions and other files are also written - writeDir(app.getFile(ApplicationPackage.SEARCH_DEFINITIONS_DIR), zkPath, false); - writeDir(app.getFile(ApplicationPackage.SCHEMAS_DIR), zkPath, false); + writeDir(app.getFile(ApplicationPackage.SEARCH_DEFINITIONS_DIR), zkPath, true); + writeDir(app.getFile(ApplicationPackage.SCHEMAS_DIR), zkPath, true); for (NamedReader sd : schemas) { curator.set(zkPath.append(sd.getName()), Utf8.toBytes(com.yahoo.io.IOUtils.readAll(sd.getReader()))); sd.getReader().close(); diff --git a/vespa-documentgen-plugin/src/main/java/com/yahoo/vespa/DocumentGenMojo.java b/vespa-documentgen-plugin/src/main/java/com/yahoo/vespa/DocumentGenMojo.java index 775b29c4abd..137860ba096 100644 --- a/vespa-documentgen-plugin/src/main/java/com/yahoo/vespa/DocumentGenMojo.java +++ b/vespa-documentgen-plugin/src/main/java/com/yahoo/vespa/DocumentGenMojo.java @@ -148,7 +148,7 @@ public class DocumentGenMojo extends AbstractMojo { throw new IllegalArgumentException(e); } } - builder.build(); + builder.build(true); for (Schema schema : builder.getSchemaList() ) { this.searches.put(schema.getName(), schema); } -- cgit v1.2.3 From d4081dfe4c40982d9657655ee33f0724ec2f2b21 Mon Sep 17 00:00:00 2001 From: Harald Musum Date: Thu, 27 Jan 2022 07:22:12 +0100 Subject: Revert "Add feature flag for merge throttling policy" --- .../java/com/yahoo/config/model/api/ModelContext.java | 1 - .../com/yahoo/config/model/deploy/TestProperties.java | 7 ------- .../content/storagecluster/StorServerProducer.java | 12 ------------ .../yahoo/vespa/model/content/StorageClusterTest.java | 19 ------------------- .../vespa/config/server/deploy/ModelContextImpl.java | 2 -- flags/src/main/java/com/yahoo/vespa/flags/Flags.java | 8 -------- 6 files changed, 49 deletions(-) (limited to 'config-model-api/src') diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java index 3bb6c7c0511..672414f2d47 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java @@ -106,7 +106,6 @@ public interface ModelContext { @ModelFeatureFlag(owners = {"arnej", "andreer"}) default List ignoredHttpUserAgents() { return List.of(); } @ModelFeatureFlag(owners = {"bjorncs"}) default boolean enableServerOcspStapling() { return false; } @ModelFeatureFlag(owners = {"vekterli"}) default String persistenceAsyncThrottling() { throw new UnsupportedOperationException("TODO specify default value"); } - @ModelFeatureFlag(owners = {"vekterli"}) default String mergeThrottlingPolicy() { throw new UnsupportedOperationException("TODO specify default value"); } @ModelFeatureFlag(owners = {"arnej"}) default boolean useQrserverServiceName() { return true; } } diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java index 3aadf30e2f9..1478854aecf 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java @@ -71,7 +71,6 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea private int maxCompactBuffers = 1; private boolean failDeploymentWithInvalidJvmOptions = false; private String persistenceAsyncThrottling = "UNLIMITED"; - private String mergeThrottlingPolicy = "STATIC"; @Override public ModelContext.FeatureFlags featureFlags() { return this; } @Override public boolean multitenant() { return multitenant; } @@ -122,7 +121,6 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea @Override public int maxCompactBuffers() { return maxCompactBuffers; } @Override public boolean failDeploymentWithInvalidJvmOptions() { return failDeploymentWithInvalidJvmOptions; } @Override public String persistenceAsyncThrottling() { return persistenceAsyncThrottling; } - @Override public String mergeThrottlingPolicy() { return mergeThrottlingPolicy; } public TestProperties maxUnCommittedMemory(int maxUnCommittedMemory) { this.maxUnCommittedMemory = maxUnCommittedMemory; @@ -315,11 +313,6 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea return this; } - public TestProperties setMergeThrottlingPolicy(String policy) { - this.mergeThrottlingPolicy = policy; - return this; - } - public static class Spec implements ConfigServerSpec { private final String hostName; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/StorServerProducer.java b/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/StorServerProducer.java index 2fca964a995..6f9166f4493 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/StorServerProducer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/StorServerProducer.java @@ -32,7 +32,6 @@ public class StorServerProducer implements StorServerConfig.Producer { private Integer maxMergesPerNode; private Integer queueSize; private Integer bucketDBStripeBits; - private StorServerConfig.Merge_throttling_policy.Type.Enum mergeThrottlingPolicyType; private StorServerProducer setMaxMergesPerNode(Integer value) { if (value != null) { @@ -51,19 +50,10 @@ public class StorServerProducer implements StorServerConfig.Producer { return this; } - private static StorServerConfig.Merge_throttling_policy.Type.Enum toThrottlePolicyType(String policyType) { - try { - return StorServerConfig.Merge_throttling_policy.Type.Enum.valueOf(policyType); - } catch (Throwable t) { - return StorServerConfig.Merge_throttling_policy.Type.STATIC; - } - } - StorServerProducer(String clusterName, ModelContext.FeatureFlags featureFlags) { this.clusterName = clusterName; maxMergesPerNode = featureFlags.maxConcurrentMergesPerNode(); queueSize = featureFlags.maxMergeQueueSize(); - mergeThrottlingPolicyType = toThrottlePolicyType(featureFlags.mergeThrottlingPolicy()); } @Override @@ -83,7 +73,5 @@ public class StorServerProducer implements StorServerConfig.Producer { if (bucketDBStripeBits != null) { builder.content_node_bucket_db_stripe_bits(bucketDBStripeBits); } - // TODO set throttle policy params based on existing or separate flags - builder.merge_throttling_policy(new StorServerConfig.Merge_throttling_policy.Builder().type(mergeThrottlingPolicyType)); } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java index 2953b0cb32d..06b907ca2c2 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java @@ -159,25 +159,6 @@ public class StorageClusterTest { assertEquals(1919, config.max_merge_queue_size()); } - @Test - public void merge_throttling_policy_config_defaults_to_static() { - var config = configFromProperties(new TestProperties()); - assertEquals(StorServerConfig.Merge_throttling_policy.Type.STATIC, config.merge_throttling_policy().type()); - } - - @Test - public void merge_throttling_policy_config_is_derived_from_flag() { - var config = configFromProperties(new TestProperties().setMergeThrottlingPolicy("STATIC")); - assertEquals(StorServerConfig.Merge_throttling_policy.Type.STATIC, config.merge_throttling_policy().type()); - - config = configFromProperties(new TestProperties().setMergeThrottlingPolicy("DYNAMIC")); - assertEquals(StorServerConfig.Merge_throttling_policy.Type.DYNAMIC, config.merge_throttling_policy().type()); - - // Invalid enum values fall back to the default - config = configFromProperties(new TestProperties().setMergeThrottlingPolicy("UKULELE")); - assertEquals(StorServerConfig.Merge_throttling_policy.Type.STATIC, config.merge_throttling_policy().type()); - } - @Test public void testVisitors() { StorVisitorConfig.Builder builder = new StorVisitorConfig.Builder(); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java index af092aec2eb..fddb80610cd 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java @@ -201,7 +201,6 @@ public class ModelContextImpl implements ModelContext { private final List ignoredHttpUserAgents; private final boolean enableServerOcspStapling; private final String persistenceAsyncThrottling; - private final String mergeThrottlingPolicy; private final boolean useQrserverServiceName; public FeatureFlags(FlagSource source, ApplicationId appId) { @@ -242,7 +241,6 @@ public class ModelContextImpl implements ModelContext { this.ignoredHttpUserAgents = flagValue(source, appId, PermanentFlags.IGNORED_HTTP_USER_AGENTS); this.enableServerOcspStapling = flagValue(source, appId, Flags.ENABLE_SERVER_OCSP_STAPLING); this.persistenceAsyncThrottling = flagValue(source, appId, Flags.PERSISTENCE_ASYNC_THROTTLING); - this.mergeThrottlingPolicy = flagValue(source, appId, Flags.MERGE_THROTTLING_POLICY); this.useQrserverServiceName = flagValue(source, appId, Flags.USE_QRSERVER_SERVICE_NAME); } diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index 470bf9a0c1c..30a7c2cd600 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -357,14 +357,6 @@ public class Flags { "Triggers restart, takes effect immediately", ZONE_ID, APPLICATION_ID); - public static final UnboundStringFlag MERGE_THROTTLING_POLICY = defineStringFlag( - "merge-throttling-policy", "STATIC", - List.of("vekterli"), "2022-01-25", "2022-05-01", - "Sets the policy used for merge throttling on the content nodes. " + - "Valid values: STATIC, DYNAMIC", - "Takes effect at redeployment", - ZONE_ID, APPLICATION_ID); - public static final UnboundBooleanFlag CHECK_CONFIG_CONVERGENCE_BEFORE_RESTARTING = defineFeatureFlag( "check-config-convergence-before-restart", false, List.of("hmusum"), "2022-01-16", "2022-02-16", -- cgit v1.2.3 From a0d9f4d2066d08a3fe1dd826f19e39ab5cfd6055 Mon Sep 17 00:00:00 2001 From: Tor Brede Vekterli Date: Tue, 25 Jan 2022 14:28:47 +0100 Subject: Add feature flag for merge throttling policy Two options; `STATIC` (default) and `DYNAMIC`. Does not currently set any window size configs based on the existing feature flag for max pending, so the config defaults will be used for these when dynamic throttling is enabled. --- .../java/com/yahoo/config/model/api/ModelContext.java | 1 + .../com/yahoo/config/model/deploy/TestProperties.java | 7 +++++++ .../content/storagecluster/StorServerProducer.java | 12 ++++++++++++ .../yahoo/vespa/model/content/StorageClusterTest.java | 19 +++++++++++++++++++ .../vespa/config/server/deploy/ModelContextImpl.java | 2 ++ flags/src/main/java/com/yahoo/vespa/flags/Flags.java | 8 ++++++++ 6 files changed, 49 insertions(+) (limited to 'config-model-api/src') diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java index 672414f2d47..3bb6c7c0511 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java @@ -106,6 +106,7 @@ public interface ModelContext { @ModelFeatureFlag(owners = {"arnej", "andreer"}) default List ignoredHttpUserAgents() { return List.of(); } @ModelFeatureFlag(owners = {"bjorncs"}) default boolean enableServerOcspStapling() { return false; } @ModelFeatureFlag(owners = {"vekterli"}) default String persistenceAsyncThrottling() { throw new UnsupportedOperationException("TODO specify default value"); } + @ModelFeatureFlag(owners = {"vekterli"}) default String mergeThrottlingPolicy() { throw new UnsupportedOperationException("TODO specify default value"); } @ModelFeatureFlag(owners = {"arnej"}) default boolean useQrserverServiceName() { return true; } } diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java index 1478854aecf..3aadf30e2f9 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java @@ -71,6 +71,7 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea private int maxCompactBuffers = 1; private boolean failDeploymentWithInvalidJvmOptions = false; private String persistenceAsyncThrottling = "UNLIMITED"; + private String mergeThrottlingPolicy = "STATIC"; @Override public ModelContext.FeatureFlags featureFlags() { return this; } @Override public boolean multitenant() { return multitenant; } @@ -121,6 +122,7 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea @Override public int maxCompactBuffers() { return maxCompactBuffers; } @Override public boolean failDeploymentWithInvalidJvmOptions() { return failDeploymentWithInvalidJvmOptions; } @Override public String persistenceAsyncThrottling() { return persistenceAsyncThrottling; } + @Override public String mergeThrottlingPolicy() { return mergeThrottlingPolicy; } public TestProperties maxUnCommittedMemory(int maxUnCommittedMemory) { this.maxUnCommittedMemory = maxUnCommittedMemory; @@ -313,6 +315,11 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea return this; } + public TestProperties setMergeThrottlingPolicy(String policy) { + this.mergeThrottlingPolicy = policy; + return this; + } + public static class Spec implements ConfigServerSpec { private final String hostName; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/StorServerProducer.java b/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/StorServerProducer.java index 6f9166f4493..2fca964a995 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/StorServerProducer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/StorServerProducer.java @@ -32,6 +32,7 @@ public class StorServerProducer implements StorServerConfig.Producer { private Integer maxMergesPerNode; private Integer queueSize; private Integer bucketDBStripeBits; + private StorServerConfig.Merge_throttling_policy.Type.Enum mergeThrottlingPolicyType; private StorServerProducer setMaxMergesPerNode(Integer value) { if (value != null) { @@ -50,10 +51,19 @@ public class StorServerProducer implements StorServerConfig.Producer { return this; } + private static StorServerConfig.Merge_throttling_policy.Type.Enum toThrottlePolicyType(String policyType) { + try { + return StorServerConfig.Merge_throttling_policy.Type.Enum.valueOf(policyType); + } catch (Throwable t) { + return StorServerConfig.Merge_throttling_policy.Type.STATIC; + } + } + StorServerProducer(String clusterName, ModelContext.FeatureFlags featureFlags) { this.clusterName = clusterName; maxMergesPerNode = featureFlags.maxConcurrentMergesPerNode(); queueSize = featureFlags.maxMergeQueueSize(); + mergeThrottlingPolicyType = toThrottlePolicyType(featureFlags.mergeThrottlingPolicy()); } @Override @@ -73,5 +83,7 @@ public class StorServerProducer implements StorServerConfig.Producer { if (bucketDBStripeBits != null) { builder.content_node_bucket_db_stripe_bits(bucketDBStripeBits); } + // TODO set throttle policy params based on existing or separate flags + builder.merge_throttling_policy(new StorServerConfig.Merge_throttling_policy.Builder().type(mergeThrottlingPolicyType)); } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java index 06b907ca2c2..2953b0cb32d 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java @@ -159,6 +159,25 @@ public class StorageClusterTest { assertEquals(1919, config.max_merge_queue_size()); } + @Test + public void merge_throttling_policy_config_defaults_to_static() { + var config = configFromProperties(new TestProperties()); + assertEquals(StorServerConfig.Merge_throttling_policy.Type.STATIC, config.merge_throttling_policy().type()); + } + + @Test + public void merge_throttling_policy_config_is_derived_from_flag() { + var config = configFromProperties(new TestProperties().setMergeThrottlingPolicy("STATIC")); + assertEquals(StorServerConfig.Merge_throttling_policy.Type.STATIC, config.merge_throttling_policy().type()); + + config = configFromProperties(new TestProperties().setMergeThrottlingPolicy("DYNAMIC")); + assertEquals(StorServerConfig.Merge_throttling_policy.Type.DYNAMIC, config.merge_throttling_policy().type()); + + // Invalid enum values fall back to the default + config = configFromProperties(new TestProperties().setMergeThrottlingPolicy("UKULELE")); + assertEquals(StorServerConfig.Merge_throttling_policy.Type.STATIC, config.merge_throttling_policy().type()); + } + @Test public void testVisitors() { StorVisitorConfig.Builder builder = new StorVisitorConfig.Builder(); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java index fddb80610cd..af092aec2eb 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java @@ -201,6 +201,7 @@ public class ModelContextImpl implements ModelContext { private final List ignoredHttpUserAgents; private final boolean enableServerOcspStapling; private final String persistenceAsyncThrottling; + private final String mergeThrottlingPolicy; private final boolean useQrserverServiceName; public FeatureFlags(FlagSource source, ApplicationId appId) { @@ -241,6 +242,7 @@ public class ModelContextImpl implements ModelContext { this.ignoredHttpUserAgents = flagValue(source, appId, PermanentFlags.IGNORED_HTTP_USER_AGENTS); this.enableServerOcspStapling = flagValue(source, appId, Flags.ENABLE_SERVER_OCSP_STAPLING); this.persistenceAsyncThrottling = flagValue(source, appId, Flags.PERSISTENCE_ASYNC_THROTTLING); + this.mergeThrottlingPolicy = flagValue(source, appId, Flags.MERGE_THROTTLING_POLICY); this.useQrserverServiceName = flagValue(source, appId, Flags.USE_QRSERVER_SERVICE_NAME); } diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index 30a7c2cd600..470bf9a0c1c 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -357,6 +357,14 @@ public class Flags { "Triggers restart, takes effect immediately", ZONE_ID, APPLICATION_ID); + public static final UnboundStringFlag MERGE_THROTTLING_POLICY = defineStringFlag( + "merge-throttling-policy", "STATIC", + List.of("vekterli"), "2022-01-25", "2022-05-01", + "Sets the policy used for merge throttling on the content nodes. " + + "Valid values: STATIC, DYNAMIC", + "Takes effect at redeployment", + ZONE_ID, APPLICATION_ID); + public static final UnboundBooleanFlag CHECK_CONFIG_CONVERGENCE_BEFORE_RESTARTING = defineFeatureFlag( "check-config-convergence-before-restart", false, List.of("hmusum"), "2022-01-16", "2022-02-16", -- cgit v1.2.3 From 69dcc2d3a9ff069da85b6744a9274d7944d26c69 Mon Sep 17 00:00:00 2001 From: Tor Brede Vekterli Date: Wed, 26 Jan 2022 14:04:23 +0100 Subject: Add feature flags for dynamic persistence throttling window size tuning Added flags: * Window size decrement factor (default 1.2) * Window size backoff (default 0.95) --- .../com/yahoo/config/model/api/ModelContext.java | 6 +++-- .../yahoo/config/model/deploy/TestProperties.java | 14 +++++++++++ .../content/storagecluster/FileStorProducer.java | 25 ++++++++++++++----- .../vespa/model/content/StorageClusterTest.java | 28 ++++++++++++++++++---- .../config/server/deploy/ModelContextImpl.java | 6 +++++ .../src/main/java/com/yahoo/vespa/flags/Flags.java | 16 +++++++++++++ 6 files changed, 83 insertions(+), 12 deletions(-) (limited to 'config-model-api/src') diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java index 3bb6c7c0511..952ff090693 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java @@ -105,8 +105,10 @@ public interface ModelContext { @ModelFeatureFlag(owners = {"hmusum"}) default boolean failDeploymentWithInvalidJvmOptions() { return false; } @ModelFeatureFlag(owners = {"arnej", "andreer"}) default List ignoredHttpUserAgents() { return List.of(); } @ModelFeatureFlag(owners = {"bjorncs"}) default boolean enableServerOcspStapling() { return false; } - @ModelFeatureFlag(owners = {"vekterli"}) default String persistenceAsyncThrottling() { throw new UnsupportedOperationException("TODO specify default value"); } - @ModelFeatureFlag(owners = {"vekterli"}) default String mergeThrottlingPolicy() { throw new UnsupportedOperationException("TODO specify default value"); } + @ModelFeatureFlag(owners = {"vekterli"}) default String persistenceAsyncThrottling() { throw new UnsupportedOperationException("TODO specify default value"); } + @ModelFeatureFlag(owners = {"vekterli"}) default String mergeThrottlingPolicy() { throw new UnsupportedOperationException("TODO specify default value"); } + @ModelFeatureFlag(owners = {"vekterli"}) default double persistenceThrottlingWsDecrementFactor() { throw new UnsupportedOperationException("TODO specify default value"); } + @ModelFeatureFlag(owners = {"vekterli"}) default double persistenceThrottlingWsBackoff() { throw new UnsupportedOperationException("TODO specify default value"); } @ModelFeatureFlag(owners = {"arnej"}) default boolean useQrserverServiceName() { return true; } } diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java index 3aadf30e2f9..a492ee47030 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java @@ -72,6 +72,8 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea private boolean failDeploymentWithInvalidJvmOptions = false; private String persistenceAsyncThrottling = "UNLIMITED"; private String mergeThrottlingPolicy = "STATIC"; + private double persistenceThrottlingWsDecrementFactor = 1.2; + private double persistenceThrottlingWsBackoff = 0.95; @Override public ModelContext.FeatureFlags featureFlags() { return this; } @Override public boolean multitenant() { return multitenant; } @@ -123,6 +125,8 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea @Override public boolean failDeploymentWithInvalidJvmOptions() { return failDeploymentWithInvalidJvmOptions; } @Override public String persistenceAsyncThrottling() { return persistenceAsyncThrottling; } @Override public String mergeThrottlingPolicy() { return mergeThrottlingPolicy; } + @Override public double persistenceThrottlingWsDecrementFactor() { return persistenceThrottlingWsDecrementFactor; } + @Override public double persistenceThrottlingWsBackoff() { return persistenceThrottlingWsBackoff; } public TestProperties maxUnCommittedMemory(int maxUnCommittedMemory) { this.maxUnCommittedMemory = maxUnCommittedMemory; @@ -320,6 +324,16 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea return this; } + public TestProperties setPersistenceThrottlingWsDecrementFactor(double factor) { + this.persistenceThrottlingWsDecrementFactor = factor; + return this; + } + + public TestProperties setPersistenceThrottlingWsBackoff(double backoff) { + this.persistenceThrottlingWsBackoff = backoff; + return this; + } + public static class Spec implements ConfigServerSpec { private final String hostName; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/FileStorProducer.java b/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/FileStorProducer.java index 5be06306f89..f3263658717 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/FileStorProducer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/FileStorProducer.java @@ -46,7 +46,9 @@ public class FileStorProducer implements StorFilestorConfig.Producer { private final ContentCluster cluster; private final int reponseNumThreads; private final StorFilestorConfig.Response_sequencer_type.Enum responseSequencerType; - private final StorFilestorConfig.Async_operation_throttler_type.Enum asyncOperationThrottlerType; + private final StorFilestorConfig.Async_operation_throttler.Type.Enum asyncOperationThrottlerType; + private final double persistenceThrottlingWsDecrementFactor; + private final double persistenceThrottlingWsBackoff; private final boolean useAsyncMessageHandlingOnSchedule; private static StorFilestorConfig.Response_sequencer_type.Enum convertResponseSequencerType(String sequencerType) { @@ -57,11 +59,11 @@ public class FileStorProducer implements StorFilestorConfig.Producer { } } - private static StorFilestorConfig.Async_operation_throttler_type.Enum toAsyncOperationThrottlerType(String throttlerType) { + private static StorFilestorConfig.Async_operation_throttler.Type.Enum toAsyncOperationThrottlerType(String throttlerType) { try { - return StorFilestorConfig.Async_operation_throttler_type.Enum.valueOf(throttlerType); + return StorFilestorConfig.Async_operation_throttler.Type.Enum.valueOf(throttlerType); } catch (Throwable t) { - return StorFilestorConfig.Async_operation_throttler_type.UNLIMITED; + return StorFilestorConfig.Async_operation_throttler.Type.UNLIMITED; } } @@ -71,7 +73,9 @@ public class FileStorProducer implements StorFilestorConfig.Producer { this.reponseNumThreads = featureFlags.defaultNumResponseThreads(); this.responseSequencerType = convertResponseSequencerType(featureFlags.responseSequencerType()); this.asyncOperationThrottlerType = toAsyncOperationThrottlerType(featureFlags.persistenceAsyncThrottling()); - useAsyncMessageHandlingOnSchedule = featureFlags.useAsyncMessageHandlingOnSchedule(); + this.persistenceThrottlingWsDecrementFactor = featureFlags.persistenceThrottlingWsDecrementFactor(); + this.persistenceThrottlingWsBackoff = featureFlags.persistenceThrottlingWsBackoff(); + this.useAsyncMessageHandlingOnSchedule = featureFlags.useAsyncMessageHandlingOnSchedule(); } @Override @@ -83,7 +87,16 @@ public class FileStorProducer implements StorFilestorConfig.Producer { builder.num_response_threads(reponseNumThreads); builder.response_sequencer_type(responseSequencerType); builder.use_async_message_handling_on_schedule(useAsyncMessageHandlingOnSchedule); - builder.async_operation_throttler_type(asyncOperationThrottlerType); + // TODO remove deprecated throttler type config + builder.async_operation_throttler_type((asyncOperationThrottlerType == StorFilestorConfig.Async_operation_throttler.Type.DYNAMIC) + ? StorFilestorConfig.Async_operation_throttler_type.Enum.DYNAMIC + : StorFilestorConfig.Async_operation_throttler_type.Enum.UNLIMITED); + + var throttleBuilder = new StorFilestorConfig.Async_operation_throttler.Builder(); + throttleBuilder.type(asyncOperationThrottlerType); + throttleBuilder.window_size_decrement_factor(persistenceThrottlingWsDecrementFactor); + throttleBuilder.window_size_backoff(persistenceThrottlingWsBackoff); + builder.async_operation_throttler(throttleBuilder); } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java index 2953b0cb32d..509055df7f4 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java @@ -309,20 +309,40 @@ public class StorageClusterTest { @Test public void persistence_async_throttle_config_defaults_to_unlimited() { var config = filestorConfigFromProducer(simpleCluster(new TestProperties())); - assertEquals(StorFilestorConfig.Async_operation_throttler_type.UNLIMITED, config.async_operation_throttler_type()); + assertEquals(StorFilestorConfig.Async_operation_throttler_type.UNLIMITED, config.async_operation_throttler_type()); // TODO remove + assertEquals(StorFilestorConfig.Async_operation_throttler.Type.UNLIMITED, config.async_operation_throttler().type()); } @Test public void persistence_async_throttle_config_is_derived_from_flag() { var config = filestorConfigFromProducer(simpleCluster(new TestProperties().setPersistenceAsyncThrottling("UNLIMITED"))); - assertEquals(StorFilestorConfig.Async_operation_throttler_type.UNLIMITED, config.async_operation_throttler_type()); + assertEquals(StorFilestorConfig.Async_operation_throttler_type.UNLIMITED, config.async_operation_throttler_type()); // TODO remove + assertEquals(StorFilestorConfig.Async_operation_throttler.Type.UNLIMITED, config.async_operation_throttler().type()); config = filestorConfigFromProducer(simpleCluster(new TestProperties().setPersistenceAsyncThrottling("DYNAMIC"))); - assertEquals(StorFilestorConfig.Async_operation_throttler_type.DYNAMIC, config.async_operation_throttler_type()); + assertEquals(StorFilestorConfig.Async_operation_throttler_type.DYNAMIC, config.async_operation_throttler_type()); // TODO remove + assertEquals(StorFilestorConfig.Async_operation_throttler.Type.DYNAMIC, config.async_operation_throttler().type()); // Invalid enum values fall back to the default config = filestorConfigFromProducer(simpleCluster(new TestProperties().setPersistenceAsyncThrottling("BANANAS"))); - assertEquals(StorFilestorConfig.Async_operation_throttler_type.UNLIMITED, config.async_operation_throttler_type()); + assertEquals(StorFilestorConfig.Async_operation_throttler_type.UNLIMITED, config.async_operation_throttler_type()); // TODO remove + assertEquals(StorFilestorConfig.Async_operation_throttler.Type.UNLIMITED, config.async_operation_throttler().type()); + } + + @Test + public void persistence_dynamic_throttling_parameters_have_sane_defaults() { + var config = filestorConfigFromProducer(simpleCluster(new TestProperties())); + assertEquals(1.2, config.async_operation_throttler().window_size_decrement_factor(), 0.0001); + assertEquals(0.95, config.async_operation_throttler().window_size_backoff(), 0.0001); + } + + @Test + public void persistence_dynamic_throttling_parameters_can_be_set_through_feature_flags() { + var config = filestorConfigFromProducer(simpleCluster(new TestProperties() + .setPersistenceThrottlingWsDecrementFactor(1.5) + .setPersistenceThrottlingWsBackoff(0.8))); + assertEquals(1.5, config.async_operation_throttler().window_size_decrement_factor(), 0.0001); + assertEquals(0.8, config.async_operation_throttler().window_size_backoff(), 0.0001); } @Test diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java index 89d3318d710..bc0f66d933c 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java @@ -202,6 +202,8 @@ public class ModelContextImpl implements ModelContext { private final boolean enableServerOcspStapling; private final String persistenceAsyncThrottling; private final String mergeThrottlingPolicy; + private final double persistenceThrottlingWsDecrementFactor; + private final double persistenceThrottlingWsBackoff; private final boolean useQrserverServiceName; public FeatureFlags(FlagSource source, ApplicationId appId) { @@ -243,6 +245,8 @@ public class ModelContextImpl implements ModelContext { this.enableServerOcspStapling = flagValue(source, appId, Flags.ENABLE_SERVER_OCSP_STAPLING); this.persistenceAsyncThrottling = flagValue(source, appId, Flags.PERSISTENCE_ASYNC_THROTTLING); this.mergeThrottlingPolicy = flagValue(source, appId, Flags.MERGE_THROTTLING_POLICY); + this.persistenceThrottlingWsDecrementFactor = flagValue(source, appId, Flags.PERSISTENCE_THROTTLING_WS_DECREMENT_FACTOR); + this.persistenceThrottlingWsBackoff = flagValue(source, appId, Flags.PERSISTENCE_THROTTLING_WS_BACKOFF); this.useQrserverServiceName = flagValue(source, appId, Flags.USE_QRSERVER_SERVICE_NAME); } @@ -286,6 +290,8 @@ public class ModelContextImpl implements ModelContext { @Override public boolean enableServerOcspStapling() { return enableServerOcspStapling; } @Override public String persistenceAsyncThrottling() { return persistenceAsyncThrottling; } @Override public String mergeThrottlingPolicy() { return mergeThrottlingPolicy; } + @Override public double persistenceThrottlingWsDecrementFactor() { return persistenceThrottlingWsDecrementFactor; } + @Override public double persistenceThrottlingWsBackoff() { return persistenceThrottlingWsBackoff; } @Override public boolean useQrserverServiceName() { return useQrserverServiceName; } private static V flagValue(FlagSource source, ApplicationId appId, UnboundFlag flag) { diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index 470bf9a0c1c..e2c7d376a0d 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -365,6 +365,22 @@ public class Flags { "Takes effect at redeployment", ZONE_ID, APPLICATION_ID); + public static final UnboundDoubleFlag PERSISTENCE_THROTTLING_WS_DECREMENT_FACTOR = defineDoubleFlag( + "persistence-throttling-ws-decrement-factor", 1.2, + List.of("vekterli"), "2022-01-27", "2022-05-01", + "Sets the dynamic throttle policy window size decrement factor for persistence " + + "async throttling. Only applies if DYNAMIC policy is used.", + "Takes effect on redeployment", + ZONE_ID, APPLICATION_ID); + + public static final UnboundDoubleFlag PERSISTENCE_THROTTLING_WS_BACKOFF = defineDoubleFlag( + "persistence-throttling-ws-backoff", 0.95, + List.of("vekterli"), "2022-01-27", "2022-05-01", + "Sets the dynamic throttle policy window size backoff for persistence " + + "async throttling. Only applies if DYNAMIC policy is used. Valid range [0, 1]", + "Takes effect on redeployment", + ZONE_ID, APPLICATION_ID); + public static final UnboundBooleanFlag CHECK_CONFIG_CONVERGENCE_BEFORE_RESTARTING = defineFeatureFlag( "check-config-convergence-before-restart", false, List.of("hmusum"), "2022-01-16", "2022-02-16", -- cgit v1.2.3 From 4bd4e26765074e36199ad0428bbf758d7dc29f82 Mon Sep 17 00:00:00 2001 From: Jon Marius Venstad Date: Tue, 25 Jan 2022 13:28:17 +0100 Subject: Add "simultaneous" upgrade rollout strategy --- config-model-api/abi-spec.json | 3 ++- .../main/java/com/yahoo/config/application/api/DeploymentSpec.java | 4 ++-- .../com/yahoo/config/application/api/xml/DeploymentSpecXmlReader.java | 2 +- .../java/com/yahoo/config/application/api/DeploymentSpecTest.java | 4 ++++ .../yahoo/vespa/hosted/controller/deployment/DeploymentStatus.java | 2 +- .../java/com/yahoo/vespa/hosted/controller/deployment/RunList.java | 2 +- 6 files changed, 11 insertions(+), 6 deletions(-) (limited to 'config-model-api/src') diff --git a/config-model-api/abi-spec.json b/config-model-api/abi-spec.json index d946dd972f4..ae9e7a22129 100644 --- a/config-model-api/abi-spec.json +++ b/config-model-api/abi-spec.json @@ -397,7 +397,8 @@ ], "fields": [ "public static final enum com.yahoo.config.application.api.DeploymentSpec$UpgradeRollout separate", - "public static final enum com.yahoo.config.application.api.DeploymentSpec$UpgradeRollout leading" + "public static final enum com.yahoo.config.application.api.DeploymentSpec$UpgradeRollout leading", + "public static final enum com.yahoo.config.application.api.DeploymentSpec$UpgradeRollout simultaneous" ] }, "com.yahoo.config.application.api.DeploymentSpec": { diff --git a/config-model-api/src/main/java/com/yahoo/config/application/api/DeploymentSpec.java b/config-model-api/src/main/java/com/yahoo/config/application/api/DeploymentSpec.java index 4b019bd9f7a..8ad42b1d4a8 100644 --- a/config-model-api/src/main/java/com/yahoo/config/application/api/DeploymentSpec.java +++ b/config-model-api/src/main/java/com/yahoo/config/application/api/DeploymentSpec.java @@ -564,9 +564,9 @@ public class DeploymentSpec { /** Separate: Application changes wait for upgrade to complete, unless upgrade fails. */ separate, /** Leading: Application changes are allowed to start and catch up to the platform upgrade. */ - leading + leading, // /** Simultaneous: Application changes deploy independently of platform upgrades. */ - // simultaneous + simultaneous } diff --git a/config-model-api/src/main/java/com/yahoo/config/application/api/xml/DeploymentSpecXmlReader.java b/config-model-api/src/main/java/com/yahoo/config/application/api/xml/DeploymentSpecXmlReader.java index b031af9faf2..b12d4024591 100644 --- a/config-model-api/src/main/java/com/yahoo/config/application/api/xml/DeploymentSpecXmlReader.java +++ b/config-model-api/src/main/java/com/yahoo/config/application/api/xml/DeploymentSpecXmlReader.java @@ -507,7 +507,7 @@ public class DeploymentSpecXmlReader { switch (rollout) { case "separate": return DeploymentSpec.UpgradeRollout.separate; case "leading": return DeploymentSpec.UpgradeRollout.leading; - // case "simultaneous": return DeploymentSpec.UpgradePolicy.conservative; + case "simultaneous": return DeploymentSpec.UpgradeRollout.simultaneous; default: throw new IllegalArgumentException("Illegal upgrade rollout '" + rollout + "': " + "Must be one of " + Arrays.toString(DeploymentSpec.UpgradeRollout.values())); } diff --git a/config-model-api/src/test/java/com/yahoo/config/application/api/DeploymentSpecTest.java b/config-model-api/src/test/java/com/yahoo/config/application/api/DeploymentSpecTest.java index a97faf5995d..f6af155ffc2 100644 --- a/config-model-api/src/test/java/com/yahoo/config/application/api/DeploymentSpecTest.java +++ b/config-model-api/src/test/java/com/yahoo/config/application/api/DeploymentSpecTest.java @@ -386,12 +386,16 @@ public class DeploymentSpecTest { " " + " " + " " + + " " + + " " + + " " + " " + "" ); DeploymentSpec spec = DeploymentSpec.fromXml(r); assertEquals("leading", spec.requireInstance("default").upgradeRollout().toString()); assertEquals("separate", spec.requireInstance("custom").upgradeRollout().toString()); + assertEquals("simultaneous", spec.requireInstance("aggressive").upgradeRollout().toString()); } @Test diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatus.java index ad237a6aa6e..05242ed744e 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatus.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatus.java @@ -129,7 +129,7 @@ public class DeploymentStatus { /** * The set of jobs that need to run for the changes of each instance of the application to be considered complete, - * and any test jobs for any oustanding change, which will likely be needed to lated deploy this change. + * and any test jobs for any outstanding change, which will likely be needed to later deploy this change. */ public Map> jobsToRun() { Map changes = new LinkedHashMap<>(); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunList.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunList.java index 925bd500199..00cd4bd5c6c 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunList.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunList.java @@ -26,7 +26,7 @@ public class RunList extends AbstractFilteringList { return from(job.runs().descendingMap().values()); } - /** Returns the jobs with runs matching the given versions — targets only for system test, everything present otherwise. */ + /** Returns the jobs with runs matching the given versions — targets only for system test, everything present otherwise. */ public RunList on(Versions versions) { return matching(run -> matchingVersions(run, versions)); } -- cgit v1.2.3 From afc48a47ba8a3fbb31da97aa562b097ba2868b05 Mon Sep 17 00:00:00 2001 From: Bjørn Christian Seime Date: Mon, 31 Jan 2022 16:39:14 +0100 Subject: Add feature flag to enable preshutdown command for jdisc --- .../main/java/com/yahoo/config/model/api/ModelContext.java | 1 + .../yahoo/vespa/model/container/ApplicationContainer.java | 14 ++++++++++++++ .../java/com/yahoo/vespa/model/container/Container.java | 2 +- .../yahoo/vespa/model/container/http/JettyHttpServer.java | 5 +++++ .../yahoo/vespa/config/server/deploy/ModelContextImpl.java | 3 +++ flags/src/main/java/com/yahoo/vespa/flags/Flags.java | 7 +++++++ 6 files changed, 31 insertions(+), 1 deletion(-) (limited to 'config-model-api/src') diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java index 952ff090693..fb878f2c75f 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java @@ -110,6 +110,7 @@ public interface ModelContext { @ModelFeatureFlag(owners = {"vekterli"}) default double persistenceThrottlingWsDecrementFactor() { throw new UnsupportedOperationException("TODO specify default value"); } @ModelFeatureFlag(owners = {"vekterli"}) default double persistenceThrottlingWsBackoff() { throw new UnsupportedOperationException("TODO specify default value"); } @ModelFeatureFlag(owners = {"arnej"}) default boolean useQrserverServiceName() { return true; } + @ModelFeatureFlag(owners = {"bjorncs", "baldersheim"}) default boolean enableJdiscPreshutdownCommand() { return false; } } /** Warning: As elsewhere in this package, do not make backwards incompatible changes that will break old config models! */ diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainer.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainer.java index 43cb896c09b..54b9a4511b0 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainer.java @@ -11,6 +11,10 @@ import com.yahoo.config.provision.NodeResources; import com.yahoo.search.config.QrStartConfig; import com.yahoo.vespa.model.container.component.SimpleComponent; +import java.util.Optional; + +import static com.yahoo.vespa.defaults.Defaults.getDefaults; + /** * A container that is typically used by container clusters set up from the user application. * @@ -25,6 +29,7 @@ public final class ApplicationContainer extends Container implements private final boolean isHostedVespa; private final boolean enableServerOcspStapling; private final boolean useQrserverServiceName; + private final boolean enablePreshutdownCommand; public ApplicationContainer(AbstractConfigProducer parent, String name, int index, DeployState deployState) { this(parent, name, false, index, deployState); @@ -35,6 +40,7 @@ public final class ApplicationContainer extends Container implements this.isHostedVespa = deployState.isHosted(); this.enableServerOcspStapling = deployState.featureFlags().enableServerOcspStapling(); this.useQrserverServiceName = deployState.featureFlags().useQrserverServiceName(); + this.enablePreshutdownCommand = deployState.featureFlags().enableJdiscPreshutdownCommand(); addComponent(new SimpleComponent("com.yahoo.container.jdisc.messagebus.NetworkMultiplexerHolder")); addComponent(new SimpleComponent("com.yahoo.container.jdisc.messagebus.NetworkMultiplexerProvider")); @@ -101,4 +107,12 @@ public final class ApplicationContainer extends Container implements return featureFlags.jvmOmitStackTraceInFastThrowOption(ClusterSpec.Type.container); } + @Override + public Optional getPreShutdownCommand() { + if (!enablePreshutdownCommand) return Optional.empty(); + int preshutdownTimeoutSeconds = 360; + int rpcTimeoutSeconds = preshutdownTimeoutSeconds + 10; + String rpcParams = "-t " + rpcTimeoutSeconds + " tcp/localhost:" + getRpcPort() + " prepareStop d:" + preshutdownTimeoutSeconds; + return Optional.of(getDefaults().underVespaHome("bin/vespa-rpc-invoke") + " " + rpcParams); + } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java b/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java index 5c5b065ac29..2e5126dcc44 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java @@ -281,7 +281,7 @@ public abstract class Container extends AbstractService implements } protected int allocatedRpcPort = 0; - private int getRpcPort() { + protected int getRpcPort() { return allocatedRpcPort; } protected int numRpcPorts() { return rpcServerEnabled() ? 1 : 0; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/JettyHttpServer.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/JettyHttpServer.java index 7b91b4b3244..e397e6ab399 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/http/JettyHttpServer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/JettyHttpServer.java @@ -22,6 +22,7 @@ import java.util.List; public class JettyHttpServer extends SimpleComponent implements ServerConfig.Producer { private final ContainerCluster cluster; + private final boolean enablePreshutdownCommand; private volatile boolean isHostedVespa; private final List connectorFactories = new ArrayList<>(); private final List ignoredUserAgentsList = new ArrayList<>(); @@ -36,6 +37,7 @@ public class JettyHttpServer extends SimpleComponent implements ServerConfig.Pro for (String agent : deployState.featureFlags().ignoredHttpUserAgents()) { addIgnoredUserAgent(agent); } + this.enablePreshutdownCommand = deployState.featureFlags().enableJdiscPreshutdownCommand(); } public void setHostedVespa(boolean isHostedVespa) { this.isHostedVespa = isHostedVespa; } @@ -75,6 +77,9 @@ public class JettyHttpServer extends SimpleComponent implements ServerConfig.Pro .remotePortHeaders(List.of("X-Forwarded-Port", "y-rp"))); } configureJettyThreadpool(builder); + if (enablePreshutdownCommand) { + builder.stopTimeout(300); + } } private void configureJettyThreadpool(ServerConfig.Builder builder) { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java index bc0f66d933c..056ca4b6c0d 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java @@ -205,6 +205,7 @@ public class ModelContextImpl implements ModelContext { private final double persistenceThrottlingWsDecrementFactor; private final double persistenceThrottlingWsBackoff; private final boolean useQrserverServiceName; + private final boolean enableJdiscPreshutdownCommand; public FeatureFlags(FlagSource source, ApplicationId appId) { this.defaultTermwiseLimit = flagValue(source, appId, Flags.DEFAULT_TERM_WISE_LIMIT); @@ -248,6 +249,7 @@ public class ModelContextImpl implements ModelContext { this.persistenceThrottlingWsDecrementFactor = flagValue(source, appId, Flags.PERSISTENCE_THROTTLING_WS_DECREMENT_FACTOR); this.persistenceThrottlingWsBackoff = flagValue(source, appId, Flags.PERSISTENCE_THROTTLING_WS_BACKOFF); this.useQrserverServiceName = flagValue(source, appId, Flags.USE_QRSERVER_SERVICE_NAME); + this.enableJdiscPreshutdownCommand = flagValue(source, appId, Flags.ENABLE_JDISC_PRESHUTDOWN_COMMAND); } @Override public double defaultTermwiseLimit() { return defaultTermwiseLimit; } @@ -293,6 +295,7 @@ public class ModelContextImpl implements ModelContext { @Override public double persistenceThrottlingWsDecrementFactor() { return persistenceThrottlingWsDecrementFactor; } @Override public double persistenceThrottlingWsBackoff() { return persistenceThrottlingWsBackoff; } @Override public boolean useQrserverServiceName() { return useQrserverServiceName; } + @Override public boolean enableJdiscPreshutdownCommand() { return enableJdiscPreshutdownCommand; } private static V flagValue(FlagSource source, ApplicationId appId, UnboundFlag flag) { return flag.bindTo(source) diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index e2c7d376a0d..9b7dcea6c18 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -395,6 +395,13 @@ public class Flags { "Takes effect at redeployment", ZONE_ID, APPLICATION_ID); + public static final UnboundBooleanFlag ENABLE_JDISC_PRESHUTDOWN_COMMAND = defineFeatureFlag( + "enable-jdisc-preshutdown-command", false, + List.of("bjorncs", "baldersheim"), "2022-01-31", "2022-05-31", + "Enable pre-shutdown command for jdisc", + "Takes effect at redeployment", + APPLICATION_ID, HOSTNAME, TENANT_ID); + /** WARNING: public for testing: All flags should be defined in {@link Flags}. */ public static UnboundBooleanFlag defineFeatureFlag(String flagId, boolean defaultValue, List owners, String createdAt, String expiresAt, String description, -- cgit v1.2.3 From 2c4ea0a81062c3effa25ec42d44d0fc86adb1980 Mon Sep 17 00:00:00 2001 From: Henning Baldersheim Date: Mon, 7 Feb 2022 12:09:52 +0100 Subject: - Adde featureflag for controlling environment variables. - Use general environment mechanism for existing ad-hoc environment variables. --- .../com/yahoo/config/model/api/ModelContext.java | 1 + .../yahoo/config/model/deploy/TestProperties.java | 10 +- .../com/yahoo/vespa/model/AbstractService.java | 124 +++++++++------------ .../java/com/yahoo/vespa/model/admin/Admin.java | 37 +++--- .../model/builder/xml/dom/DomAdminBuilderBase.java | 4 +- .../model/builder/xml/dom/DomAdminV2Builder.java | 4 +- .../model/builder/xml/dom/DomAdminV4Builder.java | 17 ++- .../model/builder/xml/dom/VespaDomBuilder.java | 2 +- .../com/yahoo/vespa/model/container/Container.java | 4 +- .../model/container/xml/ContainerModelBuilder.java | 15 ++- .../com/yahoo/vespa/model/content/Content.java | 2 +- .../vespa/model/content/ContentSearchCluster.java | 4 +- .../yahoo/vespa/model/content/StorageGroup.java | 16 ++- .../model/content/cluster/ContentCluster.java | 2 +- .../com/yahoo/vespa/model/search/SearchNode.java | 6 +- .../com/yahoo/vespa/model/HostResourceTest.java | 2 +- .../model/builder/xml/dom/ContentBuilderTest.java | 120 +++++++++++++------- .../model/container/ContainerClusterTest.java | 12 +- .../vespa/model/search/test/SchemaNodeTest.java | 8 +- .../vespa/model/test/ModelAmendingTestCase.java | 8 +- .../config/server/deploy/ModelContextImpl.java | 8 +- .../java/com/yahoo/vespa/flags/PermanentFlags.java | 8 ++ 22 files changed, 223 insertions(+), 191 deletions(-) (limited to 'config-model-api/src') diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java index fb878f2c75f..c4a120fc10c 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java @@ -156,6 +156,7 @@ public interface ModelContext { default List tlsCiphersOverride() { return List.of(); } default List zoneDnsSuffixes() { return List.of(); } + List environmentVariables(); } @Retention(RetentionPolicy.RUNTIME) diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java index 62e78db0e6f..f76cb6e43cc 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java @@ -1,7 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.config.model.deploy; -import com.google.common.collect.ImmutableList; import com.yahoo.config.model.api.ConfigServerSpec; import com.yahoo.config.model.api.ContainerEndpoint; import com.yahoo.config.model.api.EndpointCertificateSecrets; @@ -75,6 +74,7 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea private double persistenceThrottlingWsDecrementFactor = 1.2; private double persistenceThrottlingWsBackoff = 0.95; private boolean useV8GeoPositions = false; + private List environmentVariables = List.of(); @Override public ModelContext.FeatureFlags featureFlags() { return this; } @Override public boolean multitenant() { return multitenant; } @@ -129,6 +129,7 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea @Override public double persistenceThrottlingWsDecrementFactor() { return persistenceThrottlingWsDecrementFactor; } @Override public double persistenceThrottlingWsBackoff() { return persistenceThrottlingWsBackoff; } @Override public boolean useV8GeoPositions() { return useV8GeoPositions; } + @Override public List environmentVariables() { return environmentVariables; } public TestProperties maxUnCommittedMemory(int maxUnCommittedMemory) { this.maxUnCommittedMemory = maxUnCommittedMemory; @@ -222,7 +223,7 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea } public TestProperties setConfigServerSpecs(List configServerSpecs) { - this.configServerSpecs = ImmutableList.copyOf(configServerSpecs); + this.configServerSpecs = List.copyOf(configServerSpecs); return this; } @@ -341,6 +342,11 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea return this; } + public TestProperties setEnvironmentVariables(List value) { + this.environmentVariables = value; + return this; + } + public static class Spec implements ConfigServerSpec { private final String hostName; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/AbstractService.java b/config-model/src/main/java/com/yahoo/vespa/model/AbstractService.java index 78cbb722bc7..6777e2fb741 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/AbstractService.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/AbstractService.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.model; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.model.api.PortInfo; import com.yahoo.config.model.api.ServiceInfo; +import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.producer.AbstractConfigProducer; import com.yahoo.vespa.defaults.Defaults; @@ -16,6 +17,8 @@ import java.util.HashMap; import java.util.Map; import java.util.Optional; import java.util.Set; +import java.util.TreeMap; +import java.util.stream.Collectors; import static com.yahoo.text.Lowercase.toLowerCase; @@ -54,19 +57,7 @@ public abstract class AbstractService extends AbstractConfigProducer environmentVariables = new TreeMap<>(); /** The ports metainfo object */ protected PortsMeta portsMeta = new PortsMeta(); @@ -97,6 +88,8 @@ public abstract class AbstractService extends AbstractConfigProducer parent, String name) { super(parent, name); + environmentVariables.put("OMP_NUM_THREADS", 1); + environmentVariables.put("VESPA_SILENCE_CORE_ON_OOM", true); } /** @@ -105,7 +98,7 @@ public abstract class AbstractService extends AbstractConfigProducer= 0L) - ? "VESPA_MMAP_NOCORE_LIMIT=" + getMMapNoCoreLimit() + " " - : ""; - } - - public String getCoreOnOOMEnvVariable() { - return getCoreOnOOM() ? "" : "VESPA_SILENCE_CORE_ON_OOM=true "; - } - public String getOmpNumThreadsEnvVariable() { - return (getOmpNumThreads() == 0) - ? "" - : "OMP_NUM_THREADS=" + getOmpNumThreads() + " "; - } - public String getNoVespaMallocEnvVariable() { - return "".equals(getNoVespaMalloc()) - ? "" - : "VESPA_USE_NO_VESPAMALLOC=\"" + getNoVespaMalloc() + "\" "; - } - public String getVespaMallocEnvVariable() { - return "".equals(getVespaMalloc()) - ? "" - : "VESPA_USE_VESPAMALLOC=\"" + getVespaMalloc() + "\" "; - } - public String getVespaMallocDebugEnvVariable() { - return "".equals(getVespaMallocDebug()) - ? "" - : "VESPA_USE_VESPAMALLOC_D=\"" + getVespaMallocDebug() + "\" "; - } - public String getVespaMallocDebugStackTraceEnvVariable() { - return "".equals(getVespaMallocDebugStackTrace()) - ? "" - : "VESPA_USE_VESPAMALLOC_DST=\"" + getVespaMallocDebugStackTrace() + "\" "; - } - - public String getEnvVariables() { - return getCoreOnOOMEnvVariable() + getOmpNumThreadsEnvVariable() + getMMapNoCoreEnvVariable() + getNoVespaMallocEnvVariable() + - getVespaMallocEnvVariable() + getVespaMallocDebugEnvVariable() + getVespaMallocDebugStackTraceEnvVariable(); + /** If larger or equal to 0 it mean that explicit mmaps shall not be included in coredump.*/ + public void setMMapNoCoreLimit(long noCoreLimit) { + if (noCoreLimit >= 0) { + environmentVariables.put("VESPA_MMAP_NOCORE_LIMIT", noCoreLimit); + } else { + environmentVariables.remove("VESPA_MMAP_NOCORE_LIMIT"); + } + } + public void setCoreOnOOM(boolean coreOnOOM) { + if ( ! coreOnOOM) { + environmentVariables.put("VESPA_SILENCE_CORE_ON_OOM", true); + } else { + environmentVariables.remove("VESPA_SILENCE_CORE_ON_OOM"); + } + } + + public void setNoVespaMalloc(String s) { environmentVariables.put("VESPA_USE_NO_VESPAMALLOC", s); } + public void setVespaMalloc(String s) { environmentVariables.put("VESPA_USE_VESPAMALLOC", s); } + public void setVespaMallocDebug(String s) { environmentVariables.put("VESPA_USE_VESPAMALLOC_D", s); } + public void setVespaMallocDebugStackTrace(String s) { environmentVariables.put("VESPA_USE_VESPAMALLOC_DST", s); } + + private static String toEnvValue(Object o) { + if (o instanceof Number || o instanceof Boolean) { + return o.toString(); + } + return '"' + o.toString() + '"'; + } + + public void addEnvironmentVariable(String nameAndValue) { + int pos = nameAndValue.indexOf('='); + environmentVariables.put(nameAndValue.substring(0, pos), nameAndValue.substring(pos+1)); + } + public void addEnvironmentVariable(String name, Object value) { + environmentVariables.put(name, value); + } + + public String getEnv() { + return environmentVariables.entrySet().stream().map(e -> e.getKey() + '=' + toEnvValue(e.getValue())).collect(Collectors.joining(" ")); } /** diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java index 2692d676d2b..e299689d1c7 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java @@ -5,7 +5,6 @@ import com.yahoo.config.model.api.ModelContext; import com.yahoo.cloud.config.SlobroksConfig; import com.yahoo.cloud.config.ZookeepersConfig; import com.yahoo.cloud.config.log.LogdConfig; -import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.model.ConfigModelContext.ApplicationType; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.producer.AbstractConfigProducer; @@ -168,7 +167,7 @@ public class Admin extends AbstractConfigProducer implements Serializable Slobrok slobrok = new Slobrok(this, clusterController.index(), deployState.featureFlags()); slobrok.setHostResource(clusterController.getHostResource()); slobroks.add(slobrok); - slobrok.initService(deployState.getDeployLogger()); + slobrok.initService(deployState); } return slobroks; } @@ -237,16 +236,16 @@ public class Admin extends AbstractConfigProducer implements Serializable // Send hostname to be used in configId (instead of index), as the sorting of hosts seems to be unstable // between config changes, even when the set of hosts is unchanged. var container = new MetricsProxyContainer(metricsProxyCluster, host, index, deployState); - addAndInitializeService(deployState.getDeployLogger(), host, container); + addAndInitializeService(deployState, host, container); metricsProxyCluster.addContainer(container); } } private void addCommonServices(HostResource host, DeployState deployState) { - addConfigSentinel(deployState.getDeployLogger(), host, deployState.getProperties().applicationId(), deployState.zone(), + addConfigSentinel(deployState, host, deployState.getProperties().applicationId(), deployState.zone(), deployState.featureFlags()); - addLogd(deployState.getDeployLogger(), host); - addConfigProxy(deployState.getDeployLogger(), host); + addLogd(deployState, host); + addConfigProxy(deployState, host); addFileDistribution(host); if (logForwarderConfig != null) { boolean actuallyAdd = true; @@ -259,34 +258,34 @@ public class Admin extends AbstractConfigProducer implements Serializable } } if (actuallyAdd) { - addLogForwarder(deployState.getDeployLogger(), host); + addLogForwarder(deployState, host); } } } - private void addConfigSentinel(DeployLogger deployLogger, HostResource host, + private void addConfigSentinel(DeployState deployState, HostResource host, ApplicationId applicationId, Zone zone, ModelContext.FeatureFlags featureFlags) { ConfigSentinel configSentinel = new ConfigSentinel(host.getHost(), applicationId, zone, featureFlags); - addAndInitializeService(deployLogger, host, configSentinel); + addAndInitializeService(deployState, host, configSentinel); host.getHost().setConfigSentinel(configSentinel); } - private void addLogForwarder(DeployLogger deployLogger, HostResource host) { - addAndInitializeService(deployLogger, host, new LogForwarder(host.getHost(), logForwarderConfig)); + private void addLogForwarder(DeployState deployState, HostResource host) { + addAndInitializeService(deployState, host, new LogForwarder(host.getHost(), logForwarderConfig)); } - private void addLogd(DeployLogger deployLogger, HostResource host) { - addAndInitializeService(deployLogger, host, new Logd(host.getHost())); + private void addLogd(DeployState deployState, HostResource host) { + addAndInitializeService(deployState, host, new Logd(host.getHost())); } - private void addConfigProxy(DeployLogger deployLogger, HostResource host) { - addAndInitializeService(deployLogger, host, new ConfigProxy(host.getHost())); + private void addConfigProxy(DeployState deployState, HostResource host) { + addAndInitializeService(deployState, host, new ConfigProxy(host.getHost())); } - public void addAndInitializeService(DeployLogger deployLogger, HostResource host, AbstractService service) { + public void addAndInitializeService(DeployState deployState, HostResource host, AbstractService service) { service.setHostResource(host); - service.initService(deployLogger); + service.initService(deployState); } private void addFileDistribution(HostResource host) { @@ -300,7 +299,7 @@ public class Admin extends AbstractConfigProducer implements Serializable List slobs = new ArrayList<>(); if (logserver != null) { Slobrok slobrok = new Slobrok(this, 0, deployState.featureFlags()); - addAndInitializeService(deployState.getDeployLogger(), logserver.getHostResource(), slobrok); + addAndInitializeService(deployState, logserver.getHostResource(), slobrok); slobs.add(slobrok); } @@ -309,7 +308,7 @@ public class Admin extends AbstractConfigProducer implements Serializable HostResource host = hosts.get(n); if ((logserver== null || host != logserver.getHostResource()) && ! host.getHost().runsConfigServer()) { Slobrok newSlobrok = new Slobrok(this, slobs.size(), deployState.featureFlags()); - addAndInitializeService(deployState.getDeployLogger(), host, newSlobrok); + addAndInitializeService(deployState, host, newSlobrok); slobs.add(newSlobrok); } n++; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminBuilderBase.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminBuilderBase.java index 56f462805bd..4c74282c061 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminBuilderBase.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminBuilderBase.java @@ -45,7 +45,7 @@ public abstract class DomAdminBuilderBase extends VespaDomBuilder.DomConfigProdu this.configServerSpecs = configServerSpecs; } - List getConfigServersFromSpec(DeployLogger deployLogger, AbstractConfigProducer parent) { + List getConfigServersFromSpec(DeployState deployState, AbstractConfigProducer parent) { List configservers = new ArrayList<>(); for (ConfigServerSpec spec : configServerSpecs) { HostSystem hostSystem = parent.hostSystem(); @@ -54,7 +54,7 @@ public abstract class DomAdminBuilderBase extends VespaDomBuilder.DomConfigProdu Configserver configserver = new Configserver(parent, spec.getHostName(), spec.getConfigServerPort()); configserver.setHostResource(host); configserver.setBasePort(configserver.getWantedPort()); - configserver.initService(deployLogger); + configserver.initService(deployState); configservers.add(configserver); } return configservers; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2Builder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2Builder.java index 5aaba9550d2..8e7c543b67b 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2Builder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2Builder.java @@ -53,7 +53,7 @@ public class DomAdminV2Builder extends DomAdminBuilderBase { private List parseConfigservers(DeployState deployState, Admin admin, Element adminE) { List configservers; if (multitenant) - configservers = getConfigServersFromSpec(deployState.getDeployLogger(), admin); + configservers = getConfigServersFromSpec(deployState, admin); else configservers = getConfigServers(deployState, admin, adminE); if (configservers.isEmpty() && ! multitenant) @@ -129,7 +129,7 @@ public class DomAdminV2Builder extends DomAdminBuilderBase { SimpleConfigProducer configServers = new SimpleConfigProducer<>(parent, "configservers"); Configserver configServer = new Configserver(configServers, "configserver", Configserver.defaultRpcPort); configServer.setHostResource(parent.hostSystem().getHost(Container.SINGLENODE_CONTAINER_SERVICESPEC)); - configServer.initService(deployState.getDeployLogger()); + configServer.initService(deployState); return List.of(configServer); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java index 41baf2db3aa..a91a7949ad8 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java @@ -1,7 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.builder.xml.dom; -import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.model.ConfigModelContext; import com.yahoo.config.model.api.ConfigServerSpec; import com.yahoo.config.model.deploy.DeployState; @@ -44,7 +43,7 @@ public class DomAdminV4Builder extends DomAdminBuilderBase { @Override protected void doBuildAdmin(DeployState deployState, Admin admin, Element w3cAdminElement) { ModelElement adminElement = new ModelElement(w3cAdminElement); - admin.addConfigservers(getConfigServersFromSpec(deployState.getDeployLogger(), admin)); + admin.addConfigservers(getConfigServersFromSpec(deployState, admin)); // Note: These two elements only exists in admin version 4.0 // This build handles admin version 3.0 by ignoring its content (as the content is not useful) @@ -79,13 +78,13 @@ public class DomAdminV4Builder extends DomAdminBuilderBase { Collection hosts = allocateHosts(admin.hostSystem(), "logserver", nodesSpecification); if (hosts.isEmpty()) return; // No log server can be created (and none is needed) - Logserver logserver = createLogserver(deployState.getDeployLogger(), admin, hosts); + Logserver logserver = createLogserver(deployState, admin, hosts); createContainerOnLogserverHost(deployState, admin, logserver.getHostResource()); } else if (containerModels.iterator().hasNext()) { List hosts = sortedContainerHostsFrom(containerModels.iterator().next(), nodesSpecification.minResources().nodes(), false); if (hosts.isEmpty()) return; // No log server can be created (and none is needed) - createLogserver(deployState.getDeployLogger(), admin, hosts); + createLogserver(deployState, admin, hosts); } else { context.getDeployLogger().logApplicationPackage(Level.INFO, "No container host available to use for running logserver"); } @@ -110,9 +109,9 @@ public class DomAdminV4Builder extends DomAdminBuilderBase { LogserverContainer container = new LogserverContainer(logServerCluster, deployState); container.setHostResource(hostResource); - container.initService(deployState.getDeployLogger()); + container.initService(deployState); logServerCluster.addContainer(container); - admin.addAndInitializeService(deployState.getDeployLogger(), hostResource, container); + admin.addAndInitializeService(deployState, hostResource, container); admin.setLogserverContainerCluster(logServerCluster); context.getConfigModelRepoAdder().add(logserverClusterModel); } @@ -169,11 +168,11 @@ public class DomAdminV4Builder extends DomAdminBuilderBase { return hosts.subList(0, Math.min(count, hosts.size())); } - private Logserver createLogserver(DeployLogger deployLogger, Admin admin, Collection hosts) { + private Logserver createLogserver(DeployState deployState, Admin admin, Collection hosts) { Logserver logserver = new Logserver(admin); logserver.setHostResource(hosts.iterator().next()); admin.setLogserver(logserver); - logserver.initService(deployLogger); + logserver.initService(deployState); return logserver; } @@ -185,7 +184,7 @@ public class DomAdminV4Builder extends DomAdminBuilderBase { Slobrok slobrok = new Slobrok(admin, index++, deployState.featureFlags()); slobrok.setHostResource(host); slobroks.add(slobrok); - slobrok.initService(deployState.getDeployLogger()); + slobrok.initService(deployState); } admin.addSlobroks(slobroks); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/VespaDomBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/VespaDomBuilder.java index cb19df16fd8..0be3ac01f71 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/VespaDomBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/VespaDomBuilder.java @@ -183,7 +183,7 @@ public class VespaDomBuilder extends VespaModelBuilder { // This depends on which constructor in AbstractService is used, but the best way // is to let this method do initialize. if (!t.isInitialized()) { - t.initService(deployState.getDeployLogger()); + t.initService(deployState); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java b/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java index 2e5126dcc44..4b1c03a170c 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java @@ -168,11 +168,11 @@ public abstract class Container extends AbstractService implements public void addBuiltinHandlers() { } @Override - public void initService(DeployLogger deployLogger) { + public void initService(DeployState deployState) { if (isInitialized()) return; // XXX: Must be called first, to set the baseport - super.initService(deployLogger); + super.initService(deployState); if (getHttp() == null) { initDefaultJettyConnector(); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java index b89c48931b2..8e856e5e962 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java @@ -814,7 +814,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder { false, !deployState.getProperties().isBootstrap()); var hosts = hostSystem.allocateHosts(clusterSpec, capacity, log); - return createNodesFromHosts(log, hosts, cluster, context.getDeployState()); + return createNodesFromHosts(hosts, cluster, context.getDeployState()); } else { return singleHostContainerCluster(cluster, hostSystem.getHost(Container.SINGLENODE_CONTAINER_SERVICESPEC), context); @@ -824,7 +824,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder { private List singleHostContainerCluster(ApplicationContainerCluster cluster, HostResource host, ConfigModelContext context) { ApplicationContainer node = new ApplicationContainer(cluster, "container.0", 0, context.getDeployState()); node.setHostResource(host); - node.initService(context.getDeployLogger()); + node.initService(context.getDeployState()); return List.of(node); } @@ -835,7 +835,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder { ClusterSpec.Id.from(cluster.getName()), log, hasZooKeeper(containerElement)); - return createNodesFromHosts(context.getDeployLogger(), hosts, cluster, context.getDeployState()); + return createNodesFromHosts(hosts, cluster, context.getDeployState()); } private List createNodesFromNodeType(ApplicationContainerCluster cluster, Element nodesElement, ConfigModelContext context) { @@ -847,7 +847,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder { Map hosts = cluster.getRoot().hostSystem().allocateHosts(clusterSpec, Capacity.fromRequiredNodeType(type), log); - return createNodesFromHosts(context.getDeployLogger(), hosts, cluster, context.getDeployState()); + return createNodesFromHosts(hosts, cluster, context.getDeployState()); } private List createNodesFromContentServiceReference(ApplicationContainerCluster cluster, Element nodesElement, ConfigModelContext context) { @@ -865,11 +865,10 @@ public class ContainerModelBuilder extends ConfigModelBuilder { referenceId, cluster.getRoot().hostSystem(), context.getDeployLogger()); - return createNodesFromHosts(context.getDeployLogger(), hosts, cluster, context.getDeployState()); + return createNodesFromHosts(hosts, cluster, context.getDeployState()); } - private List createNodesFromHosts(DeployLogger deployLogger, - Map hosts, + private List createNodesFromHosts(Map hosts, ApplicationContainerCluster cluster, DeployState deployState) { List nodes = new ArrayList<>(); @@ -877,7 +876,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder { String id = "container." + entry.getValue().index(); ApplicationContainer container = new ApplicationContainer(cluster, id, entry.getValue().retired(), entry.getValue().index(), deployState); container.setHostResource(entry.getKey()); - container.initService(deployLogger); + container.initService(deployState); nodes.add(container); } return nodes; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/Content.java b/config-model/src/main/java/com/yahoo/vespa/model/content/Content.java index e69dd9abf47..dcf0be48f7b 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/Content.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/Content.java @@ -312,7 +312,7 @@ public class Content extends ConfigModel { index++; docprocService.useDynamicPorts(); docprocService.setHostResource(host); - docprocService.initService(modelContext.getDeployLogger()); + docprocService.initService(modelContext.getDeployState()); nodes.add(docprocService); processedHosts.add(host); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java index 6d21e0ad10e..22b752777e9 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java @@ -300,11 +300,11 @@ public class ContentSearchCluster extends AbstractConfigProducer clusterName, node, flushOnShutdown, tuning, resourceLimits, parentGroup.isHosted(), fractionOfMemoryReserved); searchNode.setHostResource(node.getHostResource()); - searchNode.initService(deployState.getDeployLogger()); + searchNode.initService(deployState); tls = new TransactionLogServer(searchNode, clusterName, syncTransactionLog); tls.setHostResource(searchNode.getHostResource()); - tls.initService(deployState.getDeployLogger()); + tls.initService(deployState); } else { searchNode = new SearchNode.Builder(""+node.getDistributionKey(), spec, clusterName, node, flushOnShutdown, tuning, resourceLimits, fractionOfMemoryReserved) diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/StorageGroup.java b/config-model/src/main/java/com/yahoo/vespa/model/content/StorageGroup.java index de50ba6ea21..88739e92567 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/StorageGroup.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/StorageGroup.java @@ -280,15 +280,13 @@ public class StorageGroup { /** The nodes explicitly specified as a nodes tag in this group, or empty if none */ private final Optional nodeRequirement; - private final DeployLogger deployLogger; private GroupBuilder(StorageGroup storageGroup, List subGroups, List nodeBuilders, - Optional nodeRequirement, DeployLogger deployLogger) { + Optional nodeRequirement) { this.storageGroup = storageGroup; this.subGroups = subGroups; this.nodeBuilders = nodeBuilders; this.nodeRequirement = nodeRequirement; - this.deployLogger = deployLogger; } /** @@ -319,11 +317,11 @@ public class StorageGroup { StorageNode searchNode = new StorageNode(deployState.getProperties(), parent.getStorageCluster(), 1.0, distributionKey , false); searchNode.setHostResource(parent.hostSystem().getHost(Container.SINGLENODE_CONTAINER_SERVICESPEC)); PersistenceEngine provider = parent.getPersistence().create(deployState, searchNode, storageGroup, null); - searchNode.initService(deployLogger); + searchNode.initService(deployState); Distributor distributor = new Distributor(deployState.getProperties(), parent.getDistributorNodes(), distributionKey, null, provider); distributor.setHostResource(searchNode.getHostResource()); - distributor.initService(deployLogger); + distributor.initService(deployState); return searchNode; } @@ -339,7 +337,7 @@ public class StorageGroup { throw new IllegalArgumentException("Specifying individual groups is not supported on hosted applications"); Map hostMapping = nodeRequirement.isPresent() ? - provisionHosts(nodeRequirement.get(), owner.getStorageCluster().getClusterName(), owner.getRoot().hostSystem(), deployLogger) : + provisionHosts(nodeRequirement.get(), owner.getStorageCluster().getClusterName(), owner.getRoot().hostSystem(), deployState.getDeployLogger()) : Collections.emptyMap(); Map, Map> hostGroups = collectAllocatedSubgroups(hostMapping); @@ -450,7 +448,7 @@ public class StorageGroup { else // Nodes or groups explicitly listed - resolve in GroupBuilder nodeRequirement = Optional.empty(); - return new GroupBuilder(group, subGroups, explicitNodes, nodeRequirement, context.getDeployLogger()); + return new GroupBuilder(group, subGroups, explicitNodes, nodeRequirement); } private Optional childAsString(Optional element, String childTagName) { @@ -503,13 +501,13 @@ public class StorageGroup { private static StorageNode createStorageNode(DeployState deployState, ContentCluster parent, HostResource hostResource, StorageGroup parentGroup, ClusterMembership clusterMembership) { StorageNode sNode = new StorageNode(deployState.getProperties(), parent.getStorageCluster(), null, clusterMembership.index(), clusterMembership.retired()); sNode.setHostResource(hostResource); - sNode.initService(deployState.getDeployLogger()); + sNode.initService(deployState); // TODO: Supplying null as XML is not very nice PersistenceEngine provider = parent.getPersistence().create(deployState, sNode, parentGroup, null); Distributor d = new Distributor(deployState.getProperties(), parent.getDistributorNodes(), clusterMembership.index(), null, provider); d.setHostResource(sNode.getHostResource()); - d.initService(deployState.getDeployLogger()); + d.initService(deployState); return sNode; } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java index 4a88e02be45..eda70ffb2bc 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java @@ -367,7 +367,7 @@ public class ContentCluster extends AbstractConfigProducer tuning, Optional resourceLimits, boolean isHostedVespa, double fractionOfMemoryReserved) { super(parent, name); - setOmpNumThreads(1); this.isHostedVespa = isHostedVespa; this.fractionOfMemoryReserved = fractionOfMemoryReserved; this.nodeSpec = nodeSpec; @@ -239,10 +238,7 @@ public class SearchNode extends AbstractService implements @Override public String getStartupCommand() { - if (getOmpNumThreads() != 1) { - throw new IllegalStateException("ompNumThreads must be 1"); - } - String startup = getEnvVariables() + "exec $ROOT/sbin/vespa-proton " + "--identity " + getConfigId(); + String startup = getEnv() + " exec $ROOT/sbin/vespa-proton " + "--identity " + getConfigId(); if (serviceLayerService != null) { startup = startup + " --serviceidentity " + serviceLayerService.getConfigId(); } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/HostResourceTest.java b/config-model/src/test/java/com/yahoo/vespa/model/HostResourceTest.java index 80a388a822e..4d8dd3dacfd 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/HostResourceTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/HostResourceTest.java @@ -26,7 +26,7 @@ public class HostResourceTest { TestService service = new TestService(root, 1); try { - service.initService(root.deployLogger()); + service.initService(root.getDeployState()); } catch (RuntimeException e) { assertTrue(e.getMessage().endsWith("No host found for service 'hostresourcetest$testservice0'. " + "The hostalias is probably missing from hosts.xml.")); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java index c746db168ee..77fa0b685fb 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java @@ -13,6 +13,7 @@ import com.yahoo.vespa.config.ConfigDefinitionKey; import com.yahoo.vespa.config.ConfigPayloadBuilder; import com.yahoo.vespa.config.GenericConfig; import com.yahoo.vespa.config.search.core.ProtonConfig; +import com.yahoo.vespa.model.AbstractService; import com.yahoo.vespa.model.HostResource; import com.yahoo.vespa.model.Service; import com.yahoo.vespa.model.VespaModel; @@ -35,7 +36,6 @@ import java.util.List; import static com.yahoo.config.model.api.container.ContainerServiceType.CLUSTERCONTROLLER_CONTAINER; import static com.yahoo.config.model.api.container.ContainerServiceType.METRICS_PROXY_CONTAINER; import static com.yahoo.vespa.config.search.core.ProtonConfig.Feeding.Shared_field_writer_executor; -import static org.hamcrest.Matchers.containsString; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; @@ -328,10 +328,64 @@ public class ContentBuilderTest extends DomBuilderTest { assertEquals(200000, b.getRootGroup().getMmapNoCoreLimit().get().longValue()); assertEquals(2, s.getSearchNodes().size()); - assertEquals(200000, s.getSearchNodes().get(0).getMMapNoCoreLimit()); - assertEquals(200000, s.getSearchNodes().get(1).getMMapNoCoreLimit()); - assertEquals("VESPA_MMAP_NOCORE_LIMIT=200000 ", s.getSearchNodes().get(0).getMMapNoCoreEnvVariable()); - assertEquals("VESPA_MMAP_NOCORE_LIMIT=200000 ", s.getSearchNodes().get(1).getMMapNoCoreEnvVariable()); + assertTrue(s.getSearchNodes().get(0).getEnv().contains("VESPA_MMAP_NOCORE_LIMIT=200000")); + assertTrue(s.getSearchNodes().get(1).getEnv().contains("VESPA_MMAP_NOCORE_LIMIT=200000")); + } + + @Test + public void canAddEnvironmentVariable() { + ContentCluster b = createContent( + "" + + " 1" + + " " + + " " + + " " + + " " + + " " + + " " + + ""); + ContentSearchCluster s; + + s = b.getSearch(); + assertTrue(s.hasIndexedCluster()); + assertNotNull(s.getIndexed()); + assertEquals(1, b.getStorageCluster().getChildren().size()); + + assertEquals(1, s.getSearchNodes().size()); + AbstractService node = s.getSearchNodes().get(0); + node.addEnvironmentVariable("MY_ENV_1", 7); + node.addEnvironmentVariable("MY_ENV_2", "7 8"); + assertTrue(node.getEnv().contains("MY_ENV_1=7")); + assertTrue(node.getEnv().contains("MY_ENV_2=\"7 8\"")); + node.addEnvironmentVariable("MY_PARSED_ENV_1=7"); + node.addEnvironmentVariable("MY_PARSED_ENV_2=7 8"); + assertTrue(node.getEnv().contains("MY_PARSED_ENV_1=\"7\"")); + assertTrue(node.getEnv().contains("MY_PARSED_ENV_2=\"7 8\"")); + } + + @Test + public void addsEnvironmentVariablesfromFeatureFlag() { + ContentCluster b = createContent( + "" + + " 1" + + " " + + " " + + " " + + " " + + " " + + " " + + "", new TestProperties().setEnvironmentVariables(List.of("MY_1_ENV=xyz abc", "MY_2_ENV=2"))); + ContentSearchCluster s; + + s = b.getSearch(); + assertTrue(s.hasIndexedCluster()); + assertNotNull(s.getIndexed()); + assertEquals(1, b.getStorageCluster().getChildren().size()); + + assertEquals(1, s.getSearchNodes().size()); + AbstractService node = s.getSearchNodes().get(0); + assertTrue(node.getEnv().contains("MY_1_ENV=\"xyz abc\"")); + assertTrue(node.getEnv().contains("MY_2_ENV=\"2\"")); } @Test @@ -357,10 +411,8 @@ public class ContentBuilderTest extends DomBuilderTest { assertTrue(b.getRootGroup().getCoreOnOOM().get()); assertEquals(2, s.getSearchNodes().size()); - assertTrue(s.getSearchNodes().get(0).getCoreOnOOM()); - assertTrue(s.getSearchNodes().get(1).getCoreOnOOM()); - assertEquals("", s.getSearchNodes().get(0).getCoreOnOOMEnvVariable()); - assertEquals("", s.getSearchNodes().get(1).getCoreOnOOMEnvVariable()); + assertFalse(s.getSearchNodes().get(0).getEnv().contains("VESPA_SILENCE_CORE_ON_OOM=true")); + assertFalse(s.getSearchNodes().get(1).getEnv().contains("VESPA_SILENCE_CORE_ON_OOM=true")); } @Test @@ -383,10 +435,8 @@ public class ContentBuilderTest extends DomBuilderTest { assertFalse(b.getRootGroup().getCoreOnOOM().isPresent()); assertEquals(2, s.getSearchNodes().size()); - assertFalse(s.getSearchNodes().get(0).getCoreOnOOM()); - assertFalse(s.getSearchNodes().get(1).getCoreOnOOM()); - assertEquals("VESPA_SILENCE_CORE_ON_OOM=true ", s.getSearchNodes().get(0).getCoreOnOOMEnvVariable()); - assertEquals("VESPA_SILENCE_CORE_ON_OOM=true ", s.getSearchNodes().get(1).getCoreOnOOMEnvVariable()); + assertTrue(s.getSearchNodes().get(0).getEnv().contains("VESPA_SILENCE_CORE_ON_OOM=true")); + assertTrue(s.getSearchNodes().get(1).getEnv().contains("VESPA_SILENCE_CORE_ON_OOM=true")); } @Test @@ -409,10 +459,8 @@ public class ContentBuilderTest extends DomBuilderTest { assertFalse(b.getRootGroup().getMmapNoCoreLimit().isPresent()); assertEquals(2, s.getSearchNodes().size()); - assertEquals(200000, s.getSearchNodes().get(0).getMMapNoCoreLimit()); - assertEquals(-1, s.getSearchNodes().get(1).getMMapNoCoreLimit()); - assertEquals("VESPA_MMAP_NOCORE_LIMIT=200000 ", s.getSearchNodes().get(0).getMMapNoCoreEnvVariable()); - assertEquals("", s.getSearchNodes().get(1).getMMapNoCoreEnvVariable()); + assertTrue(s.getSearchNodes().get(0).getEnv().contains("VESPA_MMAP_NOCORE_LIMIT=200000")); + assertFalse(s.getSearchNodes().get(1).getEnv().contains("VESPA_MMAP_NOCORE_LIMIT=")); } @Test @@ -435,10 +483,8 @@ public class ContentBuilderTest extends DomBuilderTest { assertFalse(b.getRootGroup().getCoreOnOOM().isPresent()); assertEquals(2, s.getSearchNodes().size()); - assertTrue(s.getSearchNodes().get(0).getCoreOnOOM()); - assertFalse(s.getSearchNodes().get(1).getCoreOnOOM()); - assertEquals("", s.getSearchNodes().get(0).getCoreOnOOMEnvVariable()); - assertEquals("VESPA_SILENCE_CORE_ON_OOM=true ", s.getSearchNodes().get(1).getCoreOnOOMEnvVariable()); + assertFalse(s.getSearchNodes().get(0).getEnv().contains("VESPA_SILENCE_CORE_ON_OOM=true")); + assertTrue(s.getSearchNodes().get(1).getEnv().contains("VESPA_SILENCE_CORE_ON_OOM=true")); } @Test @@ -471,15 +517,7 @@ public class ContentBuilderTest extends DomBuilderTest { assertEquals(4, s.getSearchNodes().size()); for (SearchNode n : s.getSearchNodes()) { - assertEquals("proton", n.getNoVespaMalloc()); - assertEquals("VESPA_USE_NO_VESPAMALLOC=\"proton\" ", n.getNoVespaMallocEnvVariable()); - assertEquals("distributord", n.getVespaMallocDebug()); - assertEquals("VESPA_USE_VESPAMALLOC=\"storaged\" ", n.getVespaMallocEnvVariable()); - assertEquals("all", n.getVespaMallocDebugStackTrace()); - assertEquals("VESPA_USE_VESPAMALLOC_D=\"distributord\" ", n.getVespaMallocDebugEnvVariable()); - assertEquals("storaged", n.getVespaMalloc()); - assertEquals("VESPA_USE_VESPAMALLOC_DST=\"all\" ", n.getVespaMallocDebugStackTraceEnvVariable()); - assertEquals("VESPA_SILENCE_CORE_ON_OOM=true OMP_NUM_THREADS=1 VESPA_USE_NO_VESPAMALLOC=\"proton\" VESPA_USE_VESPAMALLOC=\"storaged\" VESPA_USE_VESPAMALLOC_D=\"distributord\" VESPA_USE_VESPAMALLOC_DST=\"all\" ", n.getEnvVariables()); + assertEquals("OMP_NUM_THREADS=1 VESPA_SILENCE_CORE_ON_OOM=true VESPA_USE_NO_VESPAMALLOC=\"proton\" VESPA_USE_VESPAMALLOC=\"storaged\" VESPA_USE_VESPAMALLOC_D=\"distributord\" VESPA_USE_VESPAMALLOC_DST=\"all\"", n.getEnv()); } } @@ -508,10 +546,10 @@ public class ContentBuilderTest extends DomBuilderTest { assertFalse(b.getRootGroup().getVespaMallocDebugStackTrace().isPresent()); assertEquals(4, s.getSearchNodes().size()); - assertEquals("VESPA_SILENCE_CORE_ON_OOM=true OMP_NUM_THREADS=1 VESPA_USE_NO_VESPAMALLOC=\"proton\" ", s.getSearchNodes().get(0).getEnvVariables()); - assertEquals("VESPA_SILENCE_CORE_ON_OOM=true OMP_NUM_THREADS=1 VESPA_USE_VESPAMALLOC_D=\"distributord\" ", s.getSearchNodes().get(1).getEnvVariables()); - assertEquals("VESPA_SILENCE_CORE_ON_OOM=true OMP_NUM_THREADS=1 VESPA_USE_VESPAMALLOC_DST=\"all\" ", s.getSearchNodes().get(2).getEnvVariables()); - assertEquals("VESPA_SILENCE_CORE_ON_OOM=true OMP_NUM_THREADS=1 VESPA_USE_VESPAMALLOC=\"storaged\" ", s.getSearchNodes().get(3).getEnvVariables()); + assertEquals("OMP_NUM_THREADS=1 VESPA_SILENCE_CORE_ON_OOM=true VESPA_USE_NO_VESPAMALLOC=\"proton\"", s.getSearchNodes().get(0).getEnv()); + assertEquals("OMP_NUM_THREADS=1 VESPA_SILENCE_CORE_ON_OOM=true VESPA_USE_VESPAMALLOC_D=\"distributord\"", s.getSearchNodes().get(1).getEnv()); + assertEquals("OMP_NUM_THREADS=1 VESPA_SILENCE_CORE_ON_OOM=true VESPA_USE_VESPAMALLOC_DST=\"all\"", s.getSearchNodes().get(2).getEnv()); + assertEquals("OMP_NUM_THREADS=1 VESPA_SILENCE_CORE_ON_OOM=true VESPA_USE_VESPAMALLOC=\"storaged\"", s.getSearchNodes().get(3).getEnv()); } @Test @@ -908,6 +946,9 @@ public class ContentBuilderTest extends DomBuilderTest { } private ContentCluster createContent(String xml) { + return createContent(xml, new TestProperties()); + } + private ContentCluster createContent(String xml, TestProperties props) { String combined = "" + ""+ " " + @@ -917,12 +958,13 @@ public class ContentBuilderTest extends DomBuilderTest { ""; + var deployStateBuilder = new DeployState.Builder().properties(props); VespaModel m = new VespaModelCreatorWithMockPkg(new MockApplicationPackage.Builder() - .withHosts(getHosts()) - .withServices(combined) - .withSearchDefinition(MockApplicationPackage.MUSIC_SEARCHDEFINITION) - .build()) - .create(); + .withHosts(getHosts()) + .withServices(combined) + .withSearchDefinition(MockApplicationPackage.MUSIC_SEARCHDEFINITION) + .build()) + .create(deployStateBuilder); return m.getContentClusters().isEmpty() ? null diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java index 14a90130a57..36adee5ff15 100755 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java @@ -6,7 +6,6 @@ import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.cloud.config.CuratorConfig; import com.yahoo.cloud.config.ZookeeperServerConfig; import com.yahoo.component.ComponentId; -import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.model.api.ApplicationClusterEndpoint; import com.yahoo.config.model.api.ContainerEndpoint; import com.yahoo.config.model.deploy.DeployState; @@ -178,7 +177,7 @@ public class ContainerClusterTest { public void testClusterControllerResourceUsage() { MockRoot root = createRoot(false); ClusterControllerContainerCluster cluster = createClusterControllerCluster(root); - addClusterController(root.deployLogger(), cluster, "host-c1", root.getDeployState()); + addClusterController(cluster, "host-c1", root.getDeployState()); assertEquals(1, cluster.getContainers().size()); QrStartConfig.Builder qrBuilder = new QrStartConfig.Builder(); cluster.getConfig(qrBuilder); @@ -198,7 +197,7 @@ public class ContainerClusterTest { public void testThatLinguisticsIsExcludedForClusterControllerCluster() { MockRoot root = createRoot(false); ClusterControllerContainerCluster cluster = createClusterControllerCluster(root); - addClusterController(root.deployLogger(), cluster, "host-c1", root.getDeployState()); + addClusterController(cluster, "host-c1", root.getDeployState()); assertFalse(contains("com.yahoo.language.provider.DefaultLinguisticsProvider", cluster.getAllComponents())); } @@ -493,17 +492,16 @@ public class ContainerClusterTest { HostResource hostResource) { ApplicationContainer container = new ApplicationContainer(cluster, name, 0, root.getDeployState()); container.setHostResource(hostResource); - container.initService(root.deployLogger()); + container.initService(root.getDeployState()); cluster.addContainer(container); } - private static void addClusterController(DeployLogger deployLogger, - ClusterControllerContainerCluster cluster, + private static void addClusterController(ClusterControllerContainerCluster cluster, String hostName, DeployState deployState) { ClusterControllerContainer container = new ClusterControllerContainer(cluster, 1, false, deployState, false); container.setHostResource(new HostResource(new Host(null, hostName))); - container.initService(deployLogger); + container.initService(deployState); cluster.addContainer(container); } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaNodeTest.java b/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaNodeTest.java index 226045f4d8a..a347c0a9c43 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaNodeTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaNodeTest.java @@ -40,11 +40,11 @@ public class SchemaNodeTest { TransactionLogServer tls = new TransactionLogServer(root, "mycluster", useFsync); tls.setHostResource(new HostResource(host)); tls.setBasePort(100); - tls.initService(root.deployLogger()); + tls.initService(root.getDeployState()); node.setTls(tls); node.setHostResource(new HostResource(host)); node.setBasePort(200); - node.initService(root.deployLogger()); + node.initService(root.getDeployState()); root.freezeModelTopology(); } @@ -78,7 +78,7 @@ public class SchemaNodeTest { MockRoot root = new MockRoot(""); SearchNode node = createSearchNode(root, "mynode", 3, new NodeSpec(7, 5), false, root.getDeployState().isHosted()); node.setHostResource(new HostResource(new Host(node, "mynbode"))); - node.initService(root.deployLogger()); + node.initService(root.getDeployState()); assertFalse(node.getPreShutdownCommand().isPresent()); } @@ -87,7 +87,7 @@ public class SchemaNodeTest { MockRoot root = new MockRoot(""); SearchNode node = createSearchNode(root, "mynode2", 4, new NodeSpec(7, 5), true, root.getDeployState().isHosted()); node.setHostResource(new HostResource(new Host(node, "mynbode2"))); - node.initService(root.deployLogger()); + node.initService(root.getDeployState()); assertTrue(node.getPreShutdownCommand().isPresent()); assertTrue(node.getPreShutdownCommand().get().contains("vespa-proton-cmd " + node.getRpcPort() + " prepareRestart")); } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/test/ModelAmendingTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/test/ModelAmendingTestCase.java index 3388f9d2670..3de8cfe540f 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/test/ModelAmendingTestCase.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/test/ModelAmendingTestCase.java @@ -2,7 +2,6 @@ package com.yahoo.vespa.model.test; import com.yahoo.component.ComponentId; -import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.model.ConfigModel; import com.yahoo.config.model.ConfigModelContext; import com.yahoo.config.model.ConfigModelRegistry; @@ -10,6 +9,7 @@ import com.yahoo.config.model.MapConfigModelRegistry; import com.yahoo.config.model.admin.AdminModel; import com.yahoo.config.model.builder.xml.ConfigModelBuilder; import com.yahoo.config.model.builder.xml.ConfigModelId; +import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.producer.AbstractConfigProducer; import com.yahoo.vespa.defaults.Defaults; import com.yahoo.vespa.model.AbstractService; @@ -152,13 +152,13 @@ public class ModelAmendingTestCase { @Override public void doBuild(AdminModelAmender model, Element spec, ConfigModelContext modelContext) { for (AdminModel adminModel : model.adminModels) - amend(modelContext.getDeployLogger(), adminModel); + amend(modelContext.getDeployState(), adminModel); } - private void amend(DeployLogger deployLogger, AdminModel adminModel) { + private void amend(DeployState deployState, AdminModel adminModel) { for (HostResource host : adminModel.getAdmin().hostSystem().getHosts()) { if ( ! host.getHost().getChildrenByTypeRecursive(AmendedService.class).isEmpty()) continue; // already amended - adminModel.getAdmin().addAndInitializeService(deployLogger, host, new AmendedService(host.getHost())); + adminModel.getAdmin().addAndInitializeService(deployState, host, new AmendedService(host.getHost())); } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java index 056ca4b6c0d..547494efcba 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java @@ -357,6 +357,7 @@ public class ModelContextImpl implements ModelContext { private final List operatorCertificates; private final List tlsCiphersOverride; private final List zoneDnsSuffixes; + private final List environmentVariables; public Properties(ApplicationId applicationId, ConfigserverConfig configserverConfig, @@ -389,13 +390,15 @@ public class ModelContextImpl implements ModelContext { this.tenantSecretStores = tenantSecretStores; this.secretStore = secretStore; this.jvmGCOptionsFlag = PermanentFlags.JVM_GC_OPTIONS.bindTo(flagSource) - .with(FetchVector.Dimension.APPLICATION_ID, applicationId.serializedForm()); + .with(FetchVector.Dimension.APPLICATION_ID, applicationId.serializedForm()); this.allowDisableMtls = PermanentFlags.ALLOW_DISABLE_MTLS.bindTo(flagSource) .with(FetchVector.Dimension.APPLICATION_ID, applicationId.serializedForm()).value(); this.operatorCertificates = operatorCertificates; this.tlsCiphersOverride = PermanentFlags.TLS_CIPHERS_OVERRIDE.bindTo(flagSource) .with(FetchVector.Dimension.APPLICATION_ID, applicationId.serializedForm()).value(); this.zoneDnsSuffixes = configserverConfig.zoneDnsSuffixes(); + this.environmentVariables = PermanentFlags.ENVIRONMENT_VARIABLES.bindTo(flagSource) + .with(FetchVector.Dimension.APPLICATION_ID, applicationId.serializedForm()).value(); } @Override public ModelContext.FeatureFlags featureFlags() { return featureFlags; } @@ -477,6 +480,9 @@ public class ModelContextImpl implements ModelContext { .value(); } + @Override + public List environmentVariables() { return environmentVariables; } + } } diff --git a/flags/src/main/java/com/yahoo/vespa/flags/PermanentFlags.java b/flags/src/main/java/com/yahoo/vespa/flags/PermanentFlags.java index 75d6cab273a..ceb5e483934 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/PermanentFlags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/PermanentFlags.java @@ -198,6 +198,14 @@ public class PermanentFlags { APPLICATION_ID, HOSTNAME ); + public static final UnboundListFlag ENVIRONMENT_VARIABLES = defineListFlag( + "environment-variables", List.of(), String.class, + "A list of environment variables set for all services. " + + "Each item should be on the form =", + "Takes effect on service restart", + ZONE_ID, APPLICATION_ID + ); + public static final UnboundStringFlag CONFIG_PROXY_JVM_ARGS = defineStringFlag( "config-proxy-jvm-args", "", "Sets jvm args for config proxy (added at the end of startup command, will override existing ones)", -- cgit v1.2.3 From 27e0c481d57d6825e3c7ebe641f56ca573fbdf8a Mon Sep 17 00:00:00 2001 From: Bjørn Christian Seime Date: Mon, 7 Feb 2022 16:28:03 +0100 Subject: Enable pre-shutdown command for jdisc by default --- .../src/main/java/com/yahoo/config/model/api/ModelContext.java | 2 +- .../java/com/yahoo/vespa/model/container/ApplicationContainer.java | 3 --- .../java/com/yahoo/vespa/model/container/http/JettyHttpServer.java | 6 +----- .../java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java | 3 --- 4 files changed, 2 insertions(+), 12 deletions(-) (limited to 'config-model-api/src') diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java index c4a120fc10c..0cf8b343b6a 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java @@ -110,7 +110,7 @@ public interface ModelContext { @ModelFeatureFlag(owners = {"vekterli"}) default double persistenceThrottlingWsDecrementFactor() { throw new UnsupportedOperationException("TODO specify default value"); } @ModelFeatureFlag(owners = {"vekterli"}) default double persistenceThrottlingWsBackoff() { throw new UnsupportedOperationException("TODO specify default value"); } @ModelFeatureFlag(owners = {"arnej"}) default boolean useQrserverServiceName() { return true; } - @ModelFeatureFlag(owners = {"bjorncs", "baldersheim"}) default boolean enableJdiscPreshutdownCommand() { return false; } + @ModelFeatureFlag(owners = {"bjorncs", "baldersheim"}) default boolean enableJdiscPreshutdownCommand() { return true; } } /** Warning: As elsewhere in this package, do not make backwards incompatible changes that will break old config models! */ diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainer.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainer.java index 54b9a4511b0..ba5e8ed5544 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainer.java @@ -29,7 +29,6 @@ public final class ApplicationContainer extends Container implements private final boolean isHostedVespa; private final boolean enableServerOcspStapling; private final boolean useQrserverServiceName; - private final boolean enablePreshutdownCommand; public ApplicationContainer(AbstractConfigProducer parent, String name, int index, DeployState deployState) { this(parent, name, false, index, deployState); @@ -40,7 +39,6 @@ public final class ApplicationContainer extends Container implements this.isHostedVespa = deployState.isHosted(); this.enableServerOcspStapling = deployState.featureFlags().enableServerOcspStapling(); this.useQrserverServiceName = deployState.featureFlags().useQrserverServiceName(); - this.enablePreshutdownCommand = deployState.featureFlags().enableJdiscPreshutdownCommand(); addComponent(new SimpleComponent("com.yahoo.container.jdisc.messagebus.NetworkMultiplexerHolder")); addComponent(new SimpleComponent("com.yahoo.container.jdisc.messagebus.NetworkMultiplexerProvider")); @@ -109,7 +107,6 @@ public final class ApplicationContainer extends Container implements @Override public Optional getPreShutdownCommand() { - if (!enablePreshutdownCommand) return Optional.empty(); int preshutdownTimeoutSeconds = 360; int rpcTimeoutSeconds = preshutdownTimeoutSeconds + 10; String rpcParams = "-t " + rpcTimeoutSeconds + " tcp/localhost:" + getRpcPort() + " prepareStop d:" + preshutdownTimeoutSeconds; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/JettyHttpServer.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/JettyHttpServer.java index e397e6ab399..33cbb328c52 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/http/JettyHttpServer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/JettyHttpServer.java @@ -22,7 +22,6 @@ import java.util.List; public class JettyHttpServer extends SimpleComponent implements ServerConfig.Producer { private final ContainerCluster cluster; - private final boolean enablePreshutdownCommand; private volatile boolean isHostedVespa; private final List connectorFactories = new ArrayList<>(); private final List ignoredUserAgentsList = new ArrayList<>(); @@ -37,7 +36,6 @@ public class JettyHttpServer extends SimpleComponent implements ServerConfig.Pro for (String agent : deployState.featureFlags().ignoredHttpUserAgents()) { addIgnoredUserAgent(agent); } - this.enablePreshutdownCommand = deployState.featureFlags().enableJdiscPreshutdownCommand(); } public void setHostedVespa(boolean isHostedVespa) { this.isHostedVespa = isHostedVespa; } @@ -77,9 +75,7 @@ public class JettyHttpServer extends SimpleComponent implements ServerConfig.Pro .remotePortHeaders(List.of("X-Forwarded-Port", "y-rp"))); } configureJettyThreadpool(builder); - if (enablePreshutdownCommand) { - builder.stopTimeout(300); - } + builder.stopTimeout(300); } private void configureJettyThreadpool(ServerConfig.Builder builder) { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java index 547494efcba..c50b75b3460 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java @@ -205,7 +205,6 @@ public class ModelContextImpl implements ModelContext { private final double persistenceThrottlingWsDecrementFactor; private final double persistenceThrottlingWsBackoff; private final boolean useQrserverServiceName; - private final boolean enableJdiscPreshutdownCommand; public FeatureFlags(FlagSource source, ApplicationId appId) { this.defaultTermwiseLimit = flagValue(source, appId, Flags.DEFAULT_TERM_WISE_LIMIT); @@ -249,7 +248,6 @@ public class ModelContextImpl implements ModelContext { this.persistenceThrottlingWsDecrementFactor = flagValue(source, appId, Flags.PERSISTENCE_THROTTLING_WS_DECREMENT_FACTOR); this.persistenceThrottlingWsBackoff = flagValue(source, appId, Flags.PERSISTENCE_THROTTLING_WS_BACKOFF); this.useQrserverServiceName = flagValue(source, appId, Flags.USE_QRSERVER_SERVICE_NAME); - this.enableJdiscPreshutdownCommand = flagValue(source, appId, Flags.ENABLE_JDISC_PRESHUTDOWN_COMMAND); } @Override public double defaultTermwiseLimit() { return defaultTermwiseLimit; } @@ -295,7 +293,6 @@ public class ModelContextImpl implements ModelContext { @Override public double persistenceThrottlingWsDecrementFactor() { return persistenceThrottlingWsDecrementFactor; } @Override public double persistenceThrottlingWsBackoff() { return persistenceThrottlingWsBackoff; } @Override public boolean useQrserverServiceName() { return useQrserverServiceName; } - @Override public boolean enableJdiscPreshutdownCommand() { return enableJdiscPreshutdownCommand; } private static V flagValue(FlagSource source, ApplicationId appId, UnboundFlag flag) { return flag.bindTo(source) -- cgit v1.2.3 From d5dc5cf116c60091684ac0efaf98716f9678b7ad Mon Sep 17 00:00:00 2001 From: Geir Storli Date: Wed, 9 Feb 2022 15:21:47 +0000 Subject: Remove feature flag "enable-feed-block-in-distributor". This has been the default for several months. --- .../com/yahoo/config/model/api/ModelContext.java | 2 +- .../yahoo/config/model/deploy/TestProperties.java | 7 ---- .../model/content/ClusterControllerConfig.java | 8 +--- .../vespa/model/content/ClusterResourceLimits.java | 25 +++++------- .../model/content/cluster/ContentCluster.java | 4 +- .../model/content/ClusterResourceLimitsTest.java | 44 +++++----------------- .../model/content/ContentSchemaClusterTest.java | 17 +-------- .../model/content/FleetControllerClusterTest.java | 20 ++-------- .../config/server/deploy/ModelContextImpl.java | 3 -- .../src/main/java/com/yahoo/vespa/flags/Flags.java | 7 ---- 10 files changed, 27 insertions(+), 110 deletions(-) (limited to 'config-model-api/src') diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java index c4a120fc10c..b3a54445a23 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java @@ -89,7 +89,7 @@ public interface ModelContext { @ModelFeatureFlag(owners = {"baldersheim"}) default int maxMergeQueueSize() { return 100; } @ModelFeatureFlag(owners = {"baldersheim"}) default boolean containerDumpHeapOnShutdownTimeout() { throw new UnsupportedOperationException("TODO specify default value"); } @ModelFeatureFlag(owners = {"baldersheim"}) default double containerShutdownTimeout() { throw new UnsupportedOperationException("TODO specify default value"); } - @ModelFeatureFlag(owners = {"geirst"}) default boolean enableFeedBlockInDistributor() { return true; } + @ModelFeatureFlag(owners = {"geirst"}, removeAfter = "7.541") default boolean enableFeedBlockInDistributor() { return true; } @ModelFeatureFlag(owners = {"bjorncs", "tokle"}) default List allowedAthenzProxyIdentities() { return List.of(); } @ModelFeatureFlag(owners = {"vekterli"}) default int maxActivationInhibitedOutOfSyncGroups() { return 0; } @ModelFeatureFlag(owners = {"hmusum"}) default String jvmOmitStackTraceInFastThrowOption(ClusterSpec.Type type) { return ""; } diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java index f76cb6e43cc..c9ddcdf38eb 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java @@ -51,7 +51,6 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea private Quota quota = Quota.unlimited(); private boolean useAsyncMessageHandlingOnSchedule = false; private double feedConcurrency = 0.5; - private boolean enableFeedBlockInDistributor = true; private int maxActivationInhibitedOutOfSyncGroups = 0; private List tenantSecretStores = Collections.emptyList(); private String jvmOmitStackTraceInFastThrowOption; @@ -106,7 +105,6 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea @Override public Quota quota() { return quota; } @Override public boolean useAsyncMessageHandlingOnSchedule() { return useAsyncMessageHandlingOnSchedule; } @Override public double feedConcurrency() { return feedConcurrency; } - @Override public boolean enableFeedBlockInDistributor() { return enableFeedBlockInDistributor; } @Override public int maxActivationInhibitedOutOfSyncGroups() { return maxActivationInhibitedOutOfSyncGroups; } @Override public List tenantSecretStores() { return tenantSecretStores; } @Override public String jvmOmitStackTraceInFastThrowOption(ClusterSpec.Type type) { return jvmOmitStackTraceInFastThrowOption; } @@ -252,11 +250,6 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea return this; } - public TestProperties enableFeedBlockInDistributor(boolean enabled) { - enableFeedBlockInDistributor = enabled; - return this; - } - public TestProperties maxActivationInhibitedOutOfSyncGroups(int nGroups) { maxActivationInhibitedOutOfSyncGroups = nGroups; return this; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/ClusterControllerConfig.java b/config-model/src/main/java/com/yahoo/vespa/model/content/ClusterControllerConfig.java index 25e00dc05dd..f0b32f9140d 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/ClusterControllerConfig.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/ClusterControllerConfig.java @@ -41,7 +41,6 @@ public class ClusterControllerConfig extends AbstractConfigProducer"); ClusterResourceLimits.Builder builder = new ClusterResourceLimits.Builder(true, - true, featureFlags.resourceLimitDisk(), featureFlags.resourceLimitMemory()); return builder.build(new ModelElement((limitsInXml ? clusterXml : noLimitsXml).getDocumentElement())); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentSchemaClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentSchemaClusterTest.java index 68e722f45d3..fac6eeb3cc3 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentSchemaClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentSchemaClusterTest.java @@ -77,12 +77,6 @@ public class ContentSchemaClusterTest { return new ProtonConfig(builder); } - private static ContentCluster createClusterWithFeatureFlag(String clusterXml, boolean enableFeedBlockInDistributor) throws Exception { - var deployStateBuilder = new DeployState.Builder().properties( - new TestProperties().enableFeedBlockInDistributor(enableFeedBlockInDistributor)); - return createCluster(clusterXml, deployStateBuilder); - } - private static void assertProtonResourceLimits(double expDiskLimit, double expMemoryLimit, String clusterXml) throws Exception { assertProtonResourceLimits(expDiskLimit, expMemoryLimit, createCluster(clusterXml)); } @@ -142,15 +136,8 @@ public class ContentSchemaClusterTest { } @Test - public void default_resource_limits_when_feed_block_is_disabled_in_distributor() throws Exception { - var cluster = createClusterWithFeatureFlag(new ContentClusterBuilder().getXml(), false); - assertProtonResourceLimits(0.8, 0.8, cluster); - assertClusterControllerResourceLimits(0.8, 0.8, cluster); - } - - @Test - public void default_resource_limits_when_feed_block_is_enabled_in_distributor() throws Exception { - var cluster = createClusterWithFeatureFlag(new ContentClusterBuilder().getXml(), true); + public void default_resource_limits_with_feed_block_in_distributor() throws Exception { + var cluster = createCluster(new ContentClusterBuilder().getXml()); assertProtonResourceLimits(0.9, 0.9, cluster); assertClusterControllerResourceLimits(0.8, 0.8, cluster); } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/FleetControllerClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/FleetControllerClusterTest.java index b27a8a192bc..e074d493e68 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/FleetControllerClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/FleetControllerClusterTest.java @@ -19,14 +19,12 @@ public class FleetControllerClusterTest { private ClusterControllerConfig parse(String xml, TestProperties props) { Document doc = XML.getDocument(xml); var deployState = new DeployState.Builder().properties(props).build(); - boolean enableFeedBlockInDistributor = deployState.getProperties().featureFlags().enableFeedBlockInDistributor(); MockRoot root = new MockRoot("", deployState); var clusterElement = new ModelElement(doc.getDocumentElement()); ModelContext.FeatureFlags featureFlags = new TestProperties(); return new ClusterControllerConfig.Builder("storage", clusterElement, - new ClusterResourceLimits.Builder(enableFeedBlockInDistributor, - false, + new ClusterResourceLimits.Builder(false, featureFlags.resourceLimitDisk(), featureFlags.resourceLimitMemory()) .build(clusterElement).getClusterControllerLimits()) @@ -34,7 +32,7 @@ public class FleetControllerClusterTest { } private ClusterControllerConfig parse(String xml) { - return parse(xml, new TestProperties().enableFeedBlockInDistributor(true)); + return parse(xml, new TestProperties()); } @Test @@ -104,7 +102,6 @@ public class FleetControllerClusterTest { assertEquals(0.0, config.min_node_ratio_per_group(), 0.01); } - @Test public void default_cluster_feed_block_limits_are_set() { assertLimits(0.8, 0.8, getConfigForBasicCluster()); @@ -142,17 +139,6 @@ public class FleetControllerClusterTest { return new FleetcontrollerConfig(builder); } - @Test - public void feature_flag_controls_enable_cluster_feed_block() { - verifyThatFeatureFlagControlsEnableClusterFeedBlock(true); - verifyThatFeatureFlagControlsEnableClusterFeedBlock(false); - } - - private void verifyThatFeatureFlagControlsEnableClusterFeedBlock(boolean flag) { - var config = getConfigForBasicCluster(new TestProperties().enableFeedBlockInDistributor(flag)); - assertEquals(flag, config.enable_cluster_feed_block()); - } - @Test public void feature_flag_controls_min_node_ratio_per_group() { verifyFeatureFlagControlsMinNodeRatioPerGroup(0.0, new TestProperties()); @@ -175,6 +161,6 @@ public class FleetControllerClusterTest { } private FleetcontrollerConfig getConfigForBasicCluster() { - return getConfigForBasicCluster(new TestProperties().enableFeedBlockInDistributor(true)); + return getConfigForBasicCluster(new TestProperties()); } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java index 547494efcba..29791588a3e 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java @@ -177,7 +177,6 @@ public class ModelContextImpl implements ModelContext { private final boolean skipMbusReplyThread; private final boolean useAsyncMessageHandlingOnSchedule; private final double feedConcurrency; - private final boolean enableFeedBlockInDistributor; private final List allowedAthenzProxyIdentities; private final int maxActivationInhibitedOutOfSyncGroups; private final ToIntFunction jvmOmitStackTraceInFastThrow; @@ -221,7 +220,6 @@ public class ModelContextImpl implements ModelContext { this.skipMbusReplyThread = flagValue(source, appId, Flags.SKIP_MBUS_REPLY_THREAD); this.useAsyncMessageHandlingOnSchedule = flagValue(source, appId, Flags.USE_ASYNC_MESSAGE_HANDLING_ON_SCHEDULE); this.feedConcurrency = flagValue(source, appId, Flags.FEED_CONCURRENCY); - this.enableFeedBlockInDistributor = flagValue(source, appId, Flags.ENABLE_FEED_BLOCK_IN_DISTRIBUTOR); this.allowedAthenzProxyIdentities = flagValue(source, appId, Flags.ALLOWED_ATHENZ_PROXY_IDENTITIES); this.maxActivationInhibitedOutOfSyncGroups = flagValue(source, appId, Flags.MAX_ACTIVATION_INHIBITED_OUT_OF_SYNC_GROUPS); this.jvmOmitStackTraceInFastThrow = type -> flagValueAsInt(source, appId, type, PermanentFlags.JVM_OMIT_STACK_TRACE_IN_FAST_THROW); @@ -265,7 +263,6 @@ public class ModelContextImpl implements ModelContext { @Override public boolean skipMbusReplyThread() { return skipMbusReplyThread; } @Override public boolean useAsyncMessageHandlingOnSchedule() { return useAsyncMessageHandlingOnSchedule; } @Override public double feedConcurrency() { return feedConcurrency; } - @Override public boolean enableFeedBlockInDistributor() { return enableFeedBlockInDistributor; } @Override public List allowedAthenzProxyIdentities() { return allowedAthenzProxyIdentities; } @Override public int maxActivationInhibitedOutOfSyncGroups() { return maxActivationInhibitedOutOfSyncGroups; } @Override public String jvmOmitStackTraceInFastThrowOption(ClusterSpec.Type type) { diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index 65d4d76f5c3..6d852c65819 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -144,13 +144,6 @@ public class Flags { "Takes effect at redeployment", ZONE_ID, APPLICATION_ID); - public static final UnboundBooleanFlag ENABLE_FEED_BLOCK_IN_DISTRIBUTOR = defineFeatureFlag( - "enable-feed-block-in-distributor", true, - List.of("geirst"), "2021-01-27", "2022-06-01", - "Enables blocking of feed in the distributor if resource usage is above limit on at least one content node", - "Takes effect at redeployment", - ZONE_ID, APPLICATION_ID); - public static final UnboundBooleanFlag CONTAINER_DUMP_HEAP_ON_SHUTDOWN_TIMEOUT = defineFeatureFlag( "container-dump-heap-on-shutdown-timeout", false, List.of("baldersheim"), "2021-09-25", "2022-06-01", -- cgit v1.2.3 From 025df33c53f5ab6c6ae25f7e4ada4eda8bdf269c Mon Sep 17 00:00:00 2001 From: Geir Storli Date: Fri, 11 Feb 2022 13:13:08 +0000 Subject: Add "inhibit-default-merges-when-global-merges-pending" feature flag. --- .../java/com/yahoo/config/model/api/ModelContext.java | 1 + .../com/yahoo/config/model/deploy/TestProperties.java | 7 +++++++ .../yahoo/vespa/model/content/DistributorCluster.java | 9 +++++++-- .../yahoo/vespa/model/content/ContentClusterTest.java | 18 ++++++++++++++++++ .../vespa/config/server/deploy/ModelContextImpl.java | 3 +++ flags/src/main/java/com/yahoo/vespa/flags/Flags.java | 8 ++++++++ 6 files changed, 44 insertions(+), 2 deletions(-) (limited to 'config-model-api/src') diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java index 638a66db03f..19daa43de13 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java @@ -109,6 +109,7 @@ public interface ModelContext { @ModelFeatureFlag(owners = {"vekterli"}) default String mergeThrottlingPolicy() { throw new UnsupportedOperationException("TODO specify default value"); } @ModelFeatureFlag(owners = {"vekterli"}) default double persistenceThrottlingWsDecrementFactor() { throw new UnsupportedOperationException("TODO specify default value"); } @ModelFeatureFlag(owners = {"vekterli"}) default double persistenceThrottlingWsBackoff() { throw new UnsupportedOperationException("TODO specify default value"); } + @ModelFeatureFlag(owners = {"geirst", "vekterli"}) default boolean inhibitDefaultMergesWhenGlobalMergesPending() { return false; } @ModelFeatureFlag(owners = {"arnej"}) default boolean useQrserverServiceName() { return true; } @ModelFeatureFlag(owners = {"bjorncs", "baldersheim"}) default boolean enableJdiscPreshutdownCommand() { return true; } } diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java index c9ddcdf38eb..0cde8c361a7 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java @@ -72,6 +72,7 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea private String mergeThrottlingPolicy = "STATIC"; private double persistenceThrottlingWsDecrementFactor = 1.2; private double persistenceThrottlingWsBackoff = 0.95; + private boolean inhibitDefaultMergesWhenGlobalMergesPending = false; private boolean useV8GeoPositions = false; private List environmentVariables = List.of(); @@ -126,6 +127,7 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea @Override public String mergeThrottlingPolicy() { return mergeThrottlingPolicy; } @Override public double persistenceThrottlingWsDecrementFactor() { return persistenceThrottlingWsDecrementFactor; } @Override public double persistenceThrottlingWsBackoff() { return persistenceThrottlingWsBackoff; } + @Override public boolean inhibitDefaultMergesWhenGlobalMergesPending() { return inhibitDefaultMergesWhenGlobalMergesPending; } @Override public boolean useV8GeoPositions() { return useV8GeoPositions; } @Override public List environmentVariables() { return environmentVariables; } @@ -330,6 +332,11 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea return this; } + public TestProperties inhibitDefaultMergesWhenGlobalMergesPending(boolean value) { + this.inhibitDefaultMergesWhenGlobalMergesPending = value; + return this; + } + public TestProperties setUseV8GeoPositions(boolean value) { this.useV8GeoPositions = value; return this; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/DistributorCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/DistributorCluster.java index b8d2a4f91fe..3f01f5610f1 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/DistributorCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/DistributorCluster.java @@ -43,6 +43,7 @@ public class DistributorCluster extends AbstractConfigProducer impl private final boolean useThreePhaseUpdates; private final int maxActivationInhibitedOutOfSyncGroups; private final boolean unorderedMergeChaining; + private final boolean inhibitDefaultMergesWhenGlobalMergesPending; public static class Builder extends VespaDomBuilder.DomConfigProducerBuilder { @@ -106,11 +107,12 @@ public class DistributorCluster extends AbstractConfigProducer impl boolean useThreePhaseUpdates = deployState.getProperties().featureFlags().useThreePhaseUpdates(); int maxInhibitedGroups = deployState.getProperties().featureFlags().maxActivationInhibitedOutOfSyncGroups(); boolean unorderedMergeChaining = deployState.getProperties().featureFlags().unorderedMergeChaining(); + boolean inhibitDefaultMerges = deployState.getProperties().featureFlags().inhibitDefaultMergesWhenGlobalMergesPending(); return new DistributorCluster(parent, new BucketSplitting.Builder().build(new ModelElement(producerSpec)), gc, hasIndexedDocumentType, useThreePhaseUpdates, - maxInhibitedGroups, unorderedMergeChaining); + maxInhibitedGroups, unorderedMergeChaining, inhibitDefaultMerges); } } @@ -118,7 +120,8 @@ public class DistributorCluster extends AbstractConfigProducer impl GcOptions gc, boolean hasIndexedDocumentType, boolean useThreePhaseUpdates, int maxActivationInhibitedOutOfSyncGroups, - boolean unorderedMergeChaining) + boolean unorderedMergeChaining, + boolean inhibitDefaultMergesWhenGlobalMergesPending) { super(parent, "distributor"); this.parent = parent; @@ -128,6 +131,7 @@ public class DistributorCluster extends AbstractConfigProducer impl this.useThreePhaseUpdates = useThreePhaseUpdates; this.maxActivationInhibitedOutOfSyncGroups = maxActivationInhibitedOutOfSyncGroups; this.unorderedMergeChaining = unorderedMergeChaining; + this.inhibitDefaultMergesWhenGlobalMergesPending = inhibitDefaultMergesWhenGlobalMergesPending; } @Override @@ -142,6 +146,7 @@ public class DistributorCluster extends AbstractConfigProducer impl builder.enable_metadata_only_fetch_phase_for_inconsistent_updates(useThreePhaseUpdates); builder.max_activation_inhibited_out_of_sync_groups(maxActivationInhibitedOutOfSyncGroups); builder.use_unordered_merge_chaining(unorderedMergeChaining); + builder.inhibit_default_merges_when_global_merges_pending(inhibitDefaultMergesWhenGlobalMergesPending); bucketSplitting.getConfig(builder); } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java index 88073d281c5..9f571167d8c 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java @@ -1158,6 +1158,24 @@ public class ContentClusterTest extends ContentBaseTest { return (new StorDistributormanagerConfig(builder)).use_unordered_merge_chaining(); } + @Test + public void inhibit_default_merges_when_global_merges_pending_controlled_by_properties() throws Exception { + assertFalse(resolveInhibitDefaultMergesConfig(Optional.empty())); + assertFalse(resolveInhibitDefaultMergesConfig(Optional.of(false))); + assertTrue(resolveInhibitDefaultMergesConfig(Optional.of(true))); + } + + private boolean resolveInhibitDefaultMergesConfig(Optional inhibitDefaultMerges) throws Exception { + var props = new TestProperties(); + if (inhibitDefaultMerges.isPresent()) { + props.inhibitDefaultMergesWhenGlobalMergesPending(inhibitDefaultMerges.get()); + } + var cluster = createOneNodeCluster(props); + var builder = new StorDistributormanagerConfig.Builder(); + cluster.getDistributorNodes().getConfig(builder); + return (new StorDistributormanagerConfig(builder)).inhibit_default_merges_when_global_merges_pending(); + } + @Test public void testDedicatedClusterControllers() { VespaModel noContentModel = createEnd2EndOneNode(new TestProperties().setHostedVespa(true) diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java index 7cc5f5edd14..6028d559473 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java @@ -203,6 +203,7 @@ public class ModelContextImpl implements ModelContext { private final String mergeThrottlingPolicy; private final double persistenceThrottlingWsDecrementFactor; private final double persistenceThrottlingWsBackoff; + private final boolean inhibitDefaultMergesWhenGlobalMergesPending; private final boolean useQrserverServiceName; public FeatureFlags(FlagSource source, ApplicationId appId) { @@ -245,6 +246,7 @@ public class ModelContextImpl implements ModelContext { this.mergeThrottlingPolicy = flagValue(source, appId, Flags.MERGE_THROTTLING_POLICY); this.persistenceThrottlingWsDecrementFactor = flagValue(source, appId, Flags.PERSISTENCE_THROTTLING_WS_DECREMENT_FACTOR); this.persistenceThrottlingWsBackoff = flagValue(source, appId, Flags.PERSISTENCE_THROTTLING_WS_BACKOFF); + this.inhibitDefaultMergesWhenGlobalMergesPending = flagValue(source, appId, Flags.INHIBIT_DEFAULT_MERGES_WHEN_GLOBAL_MERGES_PENDING); this.useQrserverServiceName = flagValue(source, appId, Flags.USE_QRSERVER_SERVICE_NAME); } @@ -289,6 +291,7 @@ public class ModelContextImpl implements ModelContext { @Override public String mergeThrottlingPolicy() { return mergeThrottlingPolicy; } @Override public double persistenceThrottlingWsDecrementFactor() { return persistenceThrottlingWsDecrementFactor; } @Override public double persistenceThrottlingWsBackoff() { return persistenceThrottlingWsBackoff; } + @Override public boolean inhibitDefaultMergesWhenGlobalMergesPending() { return inhibitDefaultMergesWhenGlobalMergesPending; } @Override public boolean useQrserverServiceName() { return useQrserverServiceName; } private static V flagValue(FlagSource source, ApplicationId appId, UnboundFlag flag) { diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index 9c1e1776894..2e985a4fca0 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -328,6 +328,14 @@ public class Flags { "Takes effect on redeployment", ZONE_ID, APPLICATION_ID); + public static final UnboundBooleanFlag INHIBIT_DEFAULT_MERGES_WHEN_GLOBAL_MERGES_PENDING = defineFeatureFlag( + "inhibit-default-merges-when-global-merges-pending", false, + List.of("geirst", "vekterli"), "2022-02-11", "2022-06-01", + "Inhibits all merges to buckets in the default bucket space if the current " + + "cluster state bundle indicates that global merges are pending in the cluster", + "Takes effect on redeployment", + ZONE_ID, APPLICATION_ID); + public static final UnboundBooleanFlag CHECK_CONFIG_CONVERGENCE_BEFORE_RESTARTING = defineFeatureFlag( "check-config-convergence-before-restart", true, List.of("hmusum"), "2022-01-16", "2022-02-16", -- cgit v1.2.3 From d93989ba9fea718bd4e7b3db5cdd3bc6c9731721 Mon Sep 17 00:00:00 2001 From: Geir Storli Date: Mon, 14 Feb 2022 15:21:34 +0000 Subject: Reduce default resource limit for disk from 80% to 75%. After the changes in https://github.com/vespa-engine/vespa/pull/20818, the transient disk used as part of index fusion is no longer reported to the cluster controller. Transient disk usage should be covered by the resource headroom on content nodes, instead of leading to feed blocked due to natural fluctuations. We must therefore adjust the limit slightly down, to ensure the content nodes have enough headroom. --- .../src/main/java/com/yahoo/config/model/api/ModelContext.java | 2 +- .../java/com/yahoo/config/model/deploy/TestProperties.java | 2 +- .../yahoo/vespa/model/content/ClusterResourceLimitsTest.java | 10 +++++----- .../yahoo/vespa/model/content/ContentSchemaClusterTest.java | 6 +++--- .../yahoo/vespa/model/content/FleetControllerClusterTest.java | 4 ++-- flags/src/main/java/com/yahoo/vespa/flags/PermanentFlags.java | 6 +++--- 6 files changed, 15 insertions(+), 15 deletions(-) (limited to 'config-model-api/src') diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java index 19daa43de13..873d5539dff 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java @@ -93,7 +93,7 @@ public interface ModelContext { @ModelFeatureFlag(owners = {"bjorncs", "tokle"}) default List allowedAthenzProxyIdentities() { return List.of(); } @ModelFeatureFlag(owners = {"vekterli"}) default int maxActivationInhibitedOutOfSyncGroups() { return 0; } @ModelFeatureFlag(owners = {"hmusum"}) default String jvmOmitStackTraceInFastThrowOption(ClusterSpec.Type type) { return ""; } - @ModelFeatureFlag(owners = {"hmusum"}) default double resourceLimitDisk() { return 0.8; } + @ModelFeatureFlag(owners = {"hmusum"}) default double resourceLimitDisk() { return 0.75; } @ModelFeatureFlag(owners = {"hmusum"}) default double resourceLimitMemory() { return 0.8; } @ModelFeatureFlag(owners = {"geirst", "vekterli"}) default double minNodeRatioPerGroup() { return 0.0; } @ModelFeatureFlag(owners = {"arnej"}) default boolean forwardIssuesAsErrors() { return true; } diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java index 0cde8c361a7..e6a931e9474 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java @@ -58,7 +58,7 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea private int maxMergeQueueSize = 100; private boolean allowDisableMtls = true; private List operatorCertificates = Collections.emptyList(); - private double resourceLimitDisk = 0.8; + private double resourceLimitDisk = 0.75; private double resourceLimitMemory = 0.8; private double minNodeRatioPerGroup = 0.0; private boolean containerDumpHeapOnShutdownTimeout = false; diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/ClusterResourceLimitsTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/ClusterResourceLimitsTest.java index 86847dae11f..c9863cf2144 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/ClusterResourceLimitsTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/ClusterResourceLimitsTest.java @@ -66,7 +66,7 @@ public class ClusterResourceLimitsTest { new Fixture().ctrlDisk(0.4).ctrlMemory(0.7)); assertLimits(0.4, 0.8, 0.7, 0.9, new Fixture().ctrlDisk(0.4)); - assertLimits(0.8, 0.7, 0.9, 0.85, + assertLimits(0.75, 0.7, 0.875, 0.85, new Fixture().ctrlMemory(0.7)); } @@ -76,7 +76,7 @@ public class ClusterResourceLimitsTest { new Fixture().ctrlDisk(0.4).ctrlMemory(0.7).nodeDisk(0.9).nodeMemory(0.95)); assertLimits(0.4, 0.8, 0.95, 0.9, new Fixture().ctrlDisk(0.4).nodeDisk(0.95)); - assertLimits(0.8, 0.7, 0.9, 0.95, + assertLimits(0.75, 0.7, 0.875, 0.95, new Fixture().ctrlMemory(0.7).nodeMemory(0.95)); } @@ -86,9 +86,9 @@ public class ClusterResourceLimitsTest { new Fixture().nodeDisk(0.9).nodeMemory(0.95)); assertLimits(0.89, 0.8, 0.9, 0.9, new Fixture().nodeDisk(0.9)); - assertLimits(0.8, 0.94, 0.9, 0.95, + assertLimits(0.75, 0.94, 0.875, 0.95, new Fixture().nodeMemory(0.95)); - assertLimits(0.8, 0.0, 0.9, 0.005, + assertLimits(0.75, 0.0, 0.875, 0.005, new Fixture().nodeMemory(0.005)); } @@ -102,7 +102,7 @@ public class ClusterResourceLimitsTest { @Test public void default_resource_limits_when_feed_block_is_enabled_in_distributor() { - assertLimits(0.8, 0.8, 0.9, 0.9, + assertLimits(0.75, 0.8, 0.875, 0.9, new Fixture(true)); } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentSchemaClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentSchemaClusterTest.java index fac6eeb3cc3..ff399fd2294 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentSchemaClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentSchemaClusterTest.java @@ -118,7 +118,7 @@ public class ContentSchemaClusterTest { @Test public void requireThatOnlyMemoryLimitCanBeSet() throws Exception { - assertProtonResourceLimits(0.9, 0.77, + assertProtonResourceLimits(0.875, 0.77, new ContentClusterBuilder().protonMemoryLimit(0.77).getXml()); } @@ -138,8 +138,8 @@ public class ContentSchemaClusterTest { @Test public void default_resource_limits_with_feed_block_in_distributor() throws Exception { var cluster = createCluster(new ContentClusterBuilder().getXml()); - assertProtonResourceLimits(0.9, 0.9, cluster); - assertClusterControllerResourceLimits(0.8, 0.8, cluster); + assertProtonResourceLimits(0.875, 0.9, cluster); + assertClusterControllerResourceLimits(0.75, 0.8, cluster); } @Test diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/FleetControllerClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/FleetControllerClusterTest.java index e074d493e68..2e735e9c1b0 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/FleetControllerClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/FleetControllerClusterTest.java @@ -104,14 +104,14 @@ public class FleetControllerClusterTest { @Test public void default_cluster_feed_block_limits_are_set() { - assertLimits(0.8, 0.8, getConfigForBasicCluster()); + assertLimits(0.75, 0.8, getConfigForBasicCluster()); } @Test public void resource_limits_can_be_set_in_tuning() { assertLimits(0.6, 0.7, getConfigForResourceLimitsTuning(0.6, 0.7)); assertLimits(0.6, 0.8, getConfigForResourceLimitsTuning(0.6, null)); - assertLimits(0.8, 0.7, getConfigForResourceLimitsTuning(null, 0.7)); + assertLimits(0.75, 0.7, getConfigForResourceLimitsTuning(null, 0.7)); } private static final double DELTA = 0.00001; diff --git a/flags/src/main/java/com/yahoo/vespa/flags/PermanentFlags.java b/flags/src/main/java/com/yahoo/vespa/flags/PermanentFlags.java index ceb5e483934..23daf3c4138 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/PermanentFlags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/PermanentFlags.java @@ -177,15 +177,15 @@ public class PermanentFlags { APPLICATION_ID); public static final UnboundDoubleFlag RESOURCE_LIMIT_DISK = defineDoubleFlag( - "resource-limit-disk", 0.8, - "Resource limit (between 0.0 and 1.0) for disk used by cluster controller for when to block feed", + "resource-limit-disk", 0.75, + "Resource limit (between 0.0 and 1.0) for disk usage on content nodes, used by cluster controller for when to block feed", "Takes effect on next deployment", APPLICATION_ID ); public static final UnboundDoubleFlag RESOURCE_LIMIT_MEMORY = defineDoubleFlag( "resource-limit-memory", 0.8, - "Resource limit (between 0.0 and 1.0) for memory used by cluster controller for when to block feed", + "Resource limit (between 0.0 and 1.0) for memory usage on content nodes, used by cluster controller for when to block feed", "Takes effect on next deployment", APPLICATION_ID ); -- cgit v1.2.3 From 219b0323adfb4db1635814696c7dc855974d42b8 Mon Sep 17 00:00:00 2001 From: Martin Polden Date: Tue, 15 Feb 2022 09:23:11 +0100 Subject: Remove shared endpoints from config --- .../model/api/ApplicationClusterEndpoint.java | 11 +------ .../container/ApplicationContainerCluster.java | 31 ++++--------------- .../model/container/ContainerClusterTest.java | 36 ++++++++-------------- 3 files changed, 20 insertions(+), 58 deletions(-) (limited to 'config-model-api/src') diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ApplicationClusterEndpoint.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ApplicationClusterEndpoint.java index 08ec615e4c0..a879b335a34 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ApplicationClusterEndpoint.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ApplicationClusterEndpoint.java @@ -138,6 +138,7 @@ public class ApplicationClusterEndpoint { } public static class DnsName { + private static final int MAX_LABEL_LENGTH = 63; private final String name; @@ -150,11 +151,6 @@ public class ApplicationClusterEndpoint { return name; } - // TODO: remove when 7.508 is latest version - public static DnsName sharedNameFrom(ClusterSpec.Id cluster, ApplicationId applicationId, String suffix) { - return sharedNameFrom(SystemName.main, cluster, applicationId, suffix); - } - public static DnsName sharedNameFrom(SystemName systemName, ClusterSpec.Id cluster, ApplicationId applicationId, String suffix) { String name = dnsParts(systemName, cluster, applicationId) .filter(Objects::nonNull) // remove null values that were "default" @@ -162,11 +158,6 @@ public class ApplicationClusterEndpoint { return new DnsName(sanitize(name) + suffix); // Need to sanitize name since it is considered one label } - // TODO remove this method when 7.508 is latest version - public static DnsName sharedL4NameFrom(ClusterSpec.Id cluster, ApplicationId applicationId, String suffix) { - return sharedL4NameFrom(SystemName.main, cluster, applicationId, suffix); - } - public static DnsName sharedL4NameFrom(SystemName systemName, ClusterSpec.Id cluster, ApplicationId applicationId, String suffix) { String name = dnsParts(systemName, cluster, applicationId) .filter(Objects::nonNull) // remove null values that were "default" diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java index 6982f10f658..8f95e390b07 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java @@ -47,7 +47,6 @@ import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; -import static com.yahoo.config.model.api.ApplicationClusterEndpoint.RoutingMethod.shared; import static com.yahoo.config.model.api.ApplicationClusterEndpoint.RoutingMethod.sharedLayer4; /** @@ -180,21 +179,18 @@ public final class ApplicationContainerCluster extends ContainerCluster endpoints = new ArrayList<>(); - // Add zone local endpoints using zone dns suffixes, tenant, application and cluster id. - // For now support both L7 and L4 routing + // Add zone local endpoints using zone dns suffixes, tenant, application and cluster id. List hosts = getContainers().stream() .map(AbstractService::getHostName) .collect(Collectors.toList()); - for(String suffix : deployState.getProperties().zoneDnsSuffixes()) { - // L4 + + for (String suffix : deployState.getProperties().zoneDnsSuffixes()) { ApplicationClusterEndpoint.DnsName l4Name = ApplicationClusterEndpoint.DnsName.sharedL4NameFrom( deployState.zone().system(), ClusterSpec.Id.from(getName()), @@ -207,28 +203,13 @@ public final class ApplicationContainerCluster extends ContainerCluster supportedRoutingMethods = Set.of(shared, sharedLayer4); + // Include all endpoints provided by controller Set endpointsFromController = deployState.getEndpoints(); endpointsFromController.stream() .filter(ce -> ce.clusterId().equals(getName())) - .filter(ce -> supportedRoutingMethods.contains(ce.routingMethod())) + .filter(ce -> ce.routingMethod() == sharedLayer4) .forEach(ce -> ce.names().forEach( name -> endpoints.add(ApplicationClusterEndpoint.builder() .scope(ce.scope()) diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java index 36adee5ff15..198b5713876 100755 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java @@ -364,71 +364,61 @@ public class ContainerClusterTest { assertNames(main, ApplicationId.from("t1", "a1", "i1"), Set.of(), - List.of("search-cluster.i1.a1.t1.endpoint.suffix"), - List.of("search-cluster--i1--a1--t1.endpoint.suffix")); + List.of("search-cluster.i1.a1.t1.endpoint.suffix")); assertNames(main, ApplicationId.from("t1", "a1", "default"), Set.of(), - List.of("search-cluster.a1.t1.endpoint.suffix"), - List.of("search-cluster--a1--t1.endpoint.suffix")); + List.of("search-cluster.a1.t1.endpoint.suffix")); assertNames(main, ApplicationId.from("t1", "default", "default"), Set.of(), - List.of("search-cluster.default.t1.endpoint.suffix"), - List.of("search-cluster--default--t1.endpoint.suffix")); + List.of("search-cluster.default.t1.endpoint.suffix")); assertNames(main, ApplicationId.from("t1", "a1", "default"), Set.of(new ContainerEndpoint("not-in-this-cluster", global, List.of("foo", "bar"))), - List.of("search-cluster.a1.t1.endpoint.suffix"), - List.of("search-cluster--a1--t1.endpoint.suffix")); + List.of("search-cluster.a1.t1.endpoint.suffix")); assertNames(main, ApplicationId.from("t1", "a1", "default"), Set.of(new ContainerEndpoint("search-cluster", global, List.of("rotation-1.x.y.z", "rotation-2.x.y.z"), OptionalInt.empty(), sharedLayer4), new ContainerEndpoint("search-cluster", application, List.of("app-rotation.x.y.z"), OptionalInt.of(3), sharedLayer4)), - List.of("search-cluster.a1.t1.endpoint.suffix", "rotation-1.x.y.z", "rotation-2.x.y.z", "app-rotation.x.y.z"), - List.of("search-cluster--a1--t1.endpoint.suffix")); + List.of("search-cluster.a1.t1.endpoint.suffix", "rotation-1.x.y.z", "rotation-2.x.y.z", "app-rotation.x.y.z")); // cd system: assertNames(cd, ApplicationId.from("t1", "a1", "i1"), Set.of(), - List.of("search-cluster.cd.i1.a1.t1.endpoint.suffix"), - List.of("search-cluster--cd--i1--a1--t1.endpoint.suffix")); + List.of("search-cluster.cd.i1.a1.t1.endpoint.suffix")); assertNames(cd, ApplicationId.from("t1", "a1", "default"), Set.of(), - List.of("search-cluster.cd.a1.t1.endpoint.suffix"), - List.of("search-cluster--cd--a1--t1.endpoint.suffix")); + List.of("search-cluster.cd.a1.t1.endpoint.suffix")); assertNames(cd, ApplicationId.from("t1", "default", "default"), Set.of(), - List.of("search-cluster.cd.default.t1.endpoint.suffix"), - List.of("search-cluster--cd--default--t1.endpoint.suffix")); + List.of("search-cluster.cd.default.t1.endpoint.suffix")); assertNames(cd, ApplicationId.from("t1", "a1", "default"), Set.of(new ContainerEndpoint("not-in-this-cluster", global, List.of("foo", "bar"))), - List.of("search-cluster.cd.a1.t1.endpoint.suffix"), - List.of("search-cluster--cd--a1--t1.endpoint.suffix")); + List.of("search-cluster.cd.a1.t1.endpoint.suffix")); assertNames(cd, ApplicationId.from("t1", "a1", "default"), Set.of(new ContainerEndpoint("search-cluster", global, List.of("rotation-1.x.y.z", "rotation-2.x.y.z"), OptionalInt.empty(), sharedLayer4), - new ContainerEndpoint("search-cluster", global, List.of("a--b.x.y.z", "rotation-2.x.y.z"), OptionalInt.empty(), shared), + new ContainerEndpoint("search-cluster", global, List.of("a.b.x.y.z", "rotation-2.x.y.z"), OptionalInt.empty(), shared), new ContainerEndpoint("search-cluster", application, List.of("app-rotation.x.y.z"), OptionalInt.of(3), sharedLayer4), new ContainerEndpoint("not-supported", global, List.of("not.supported"), OptionalInt.empty(), exclusive)), - List.of("search-cluster.cd.a1.t1.endpoint.suffix", "rotation-1.x.y.z", "rotation-2.x.y.z", "app-rotation.x.y.z"), - List.of("search-cluster--cd--a1--t1.endpoint.suffix", "a--b.x.y.z", "rotation-2.x.y.z")); + List.of("search-cluster.cd.a1.t1.endpoint.suffix", "rotation-1.x.y.z", "rotation-2.x.y.z", "app-rotation.x.y.z")); } - private void assertNames(SystemName systemName, ApplicationId appId, Set globalEndpoints, List expectedSharedL4Names, List expectedSharedNames) { + private void assertNames(SystemName systemName, ApplicationId appId, Set globalEndpoints, List expectedSharedL4Names) { Zone zone = new Zone(systemName, Environment.defaultEnvironment(), RegionName.defaultName()); DeployState state = new DeployState.Builder() .zone(zone) @@ -444,7 +434,7 @@ public class ContainerClusterTest { cluster.doPrepare(state); List endpoints = cluster.endpoints(); - assertNames(expectedSharedNames, endpoints.stream().filter(e -> e.routingMethod() == shared).collect(Collectors.toList())); + assertNames(List.of(), endpoints.stream().filter(e -> e.routingMethod() == shared).collect(Collectors.toList())); assertNames(expectedSharedL4Names, endpoints.stream().filter(e -> e.routingMethod() == sharedLayer4).collect(Collectors.toList())); List endpointsWithWeight = -- cgit v1.2.3