diff options
13 files changed, 80 insertions, 19 deletions
diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java index 3df93f7d08d..1f78ad20e40 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java @@ -110,6 +110,7 @@ public interface ModelContext { @ModelFeatureFlag(owners = {"vekterli", "geirst"}) default boolean unorderedMergeChaining() { return false; } @ModelFeatureFlag(owners = {"arnej"}) default boolean useV8GeoPositions() { return false; } @ModelFeatureFlag(owners = {"arnej", "baldersheim"}) default boolean useV8DocManagerCfg() { return false; } + @ModelFeatureFlag(owners = {"baldersheim", "geirst", "toregge"}) default int maxCompactBuffers() { return 1; } } /** Warning: As elsewhere in this package, do not make backwards incompatible changes that will break old config models! */ diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java index ee9af73c554..cf649162c08 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java @@ -75,6 +75,7 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea private boolean asyncApplyBucketDiff = false; private boolean unorderedMergeChaining = false; private List<String> zoneDnsSuffixes = List.of(); + private int maxCompactBuffers = 1; @Override public ModelContext.FeatureFlags featureFlags() { return this; } @Override public boolean multitenant() { return multitenant; } @@ -130,6 +131,7 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea @Override public boolean asyncApplyBucketDiff() { return asyncApplyBucketDiff; } @Override public boolean unorderedMergeChaining() { return unorderedMergeChaining; } @Override public List<String> zoneDnsSuffixes() { return zoneDnsSuffixes; } + @Override public int maxCompactBuffers() { return maxCompactBuffers; } public TestProperties maxUnCommittedMemory(int maxUnCommittedMemory) { this.maxUnCommittedMemory = maxUnCommittedMemory; @@ -340,6 +342,11 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea return this; } + public TestProperties maxCompactBuffers(int maxCompactBuffers) { + this.maxCompactBuffers = maxCompactBuffers; + return this; + } + public static class Spec implements ConfigServerSpec { private final String hostName; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java index 2189625ef74..be67cbb9dd6 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java @@ -55,6 +55,7 @@ public class VespaMetricSet { metrics.add(new Metric("slobrok.heartbeats.failed.count")); metrics.add(new Metric("logd.processed.lines.count")); metrics.add(new Metric("worker.connections.max")); + metrics.add(new Metric("endpoint.certificate.expiry.seconds")); // Java (JRT) TLS metrics metrics.add(new Metric("jrt.transport.tls-certificate-verification-failures")); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java index 13c3c229acb..f981ed6228f 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java @@ -71,6 +71,7 @@ public class ContentSearchCluster extends AbstractConfigProducer<SearchCluster> private final double defaultDiskBloatFactor; private final int defaultDocStoreCompressionLevel; private final boolean forwardIssuesToQrs; + private final int defaultMaxCompactBuffers; /** Whether the nodes of this cluster also hosts a container cluster in a hosted system */ private final boolean combined; @@ -225,6 +226,7 @@ public class ContentSearchCluster extends AbstractConfigProducer<SearchCluster> this.defaultDiskBloatFactor = featureFlags.diskBloatFactor(); this.defaultDocStoreCompressionLevel = featureFlags.docstoreCompressionLevel(); this.forwardIssuesToQrs = featureFlags.forwardIssuesAsErrors(); + this.defaultMaxCompactBuffers = featureFlags.maxCompactBuffers(); } public void setVisibilityDelay(double delay) { @@ -387,6 +389,7 @@ public class ContentSearchCluster extends AbstractConfigProducer<SearchCluster> .configid(getConfigId()) .visibilitydelay(visibilityDelay) .global(globalDocType); + ddbB.allocation.max_compact_buffers(defaultMaxCompactBuffers); if (hasIndexingModeStreaming(type)) { hasAnyNonIndexedCluster = true; diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java index 729348a0e3a..357ae7e2656 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java @@ -47,6 +47,7 @@ import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.OptionalInt; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -1026,6 +1027,27 @@ public class ContentClusterTest extends ContentBaseTest { assertTrue(resolveThreePhaseUpdateConfigWithFeatureFlag(true)); } + private int resolveMaxCompactBuffers(OptionalInt maxCompactBuffers) { + TestProperties testProperties = new TestProperties(); + if (maxCompactBuffers.isPresent()) { + testProperties.maxCompactBuffers(maxCompactBuffers.getAsInt()); + } + VespaModel model = createEnd2EndOneNode(testProperties); + ContentCluster cc = model.getContentClusters().get("storage"); + ProtonConfig.Builder protonBuilder = new ProtonConfig.Builder(); + cc.getSearch().getConfig(protonBuilder); + ProtonConfig protonConfig = new ProtonConfig(protonBuilder); + assertEquals(1, protonConfig.documentdb().size()); + return protonConfig.documentdb(0).allocation().max_compact_buffers(); + } + + @Test + public void default_max_compact_buffers_config_controlled_by_properties() { + assertEquals(1, resolveMaxCompactBuffers(OptionalInt.empty())); + assertEquals(2, resolveMaxCompactBuffers(OptionalInt.of(2))); + assertEquals(7, resolveMaxCompactBuffers(OptionalInt.of(7))); + } + void assertZookeeperServerImplementation(String expectedClassName, ClusterControllerContainerCluster clusterControllerCluster) { for (ClusterControllerContainer c : clusterControllerCluster.getContainers()) { diff --git a/config/src/main/java/com/yahoo/config/subscription/impl/JRTConfigRequester.java b/config/src/main/java/com/yahoo/config/subscription/impl/JRTConfigRequester.java index fdfaf8b72fd..5f3445d1d8f 100644 --- a/config/src/main/java/com/yahoo/config/subscription/impl/JRTConfigRequester.java +++ b/config/src/main/java/com/yahoo/config/subscription/impl/JRTConfigRequester.java @@ -19,12 +19,10 @@ import java.time.Duration; import java.time.Instant; import java.util.concurrent.ScheduledThreadPoolExecutor; import java.util.concurrent.TimeUnit; -import java.util.logging.Level; import java.util.logging.Logger; import static java.util.logging.Level.FINE; import static java.util.logging.Level.FINEST; -import static java.util.logging.Level.INFO; import static java.util.logging.Level.SEVERE; import static java.util.logging.Level.WARNING; @@ -170,15 +168,11 @@ public class JRTConfigRequester implements RequestWaiter { private void handleFailedRequest(JRTClientConfigRequest jrtReq, JRTConfigSubscription<ConfigInstance> sub, Connection connection) { logError(jrtReq, connection); - log.log(INFO, "Failure of config subscription to " + connection.getAddress() + - ", clients will keep existing config until resolved: " + sub); connectionPool.switchConnection(connection); if (failures < 10) failures++; long delay = calculateFailedRequestDelay(failures, timingValues); - // The logging depends on whether we are configured or not. - Level logLevel = sub.getConfigState().getConfig() == null ? Level.FINE : Level.INFO; - log.log(logLevel, () -> "Request for config " + jrtReq.getShortDescription() + "' failed with error code " + + log.log(FINE, () -> "Request for config " + jrtReq.getShortDescription() + "' failed with error code " + jrtReq.errorCode() + " (" + jrtReq.errorMessage() + "), scheduling new request " + " in " + delay + " ms"); scheduleNextRequest(jrtReq, sub, delay, calculateErrorTimeout()); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java index 1b55d17fd36..978241339d2 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java @@ -202,6 +202,7 @@ public class ModelContextImpl implements ModelContext { private final boolean unorderedMergeChaining; private final boolean useV8GeoPositions; private final boolean useV8DocManagerCfg; + private final int maxCompactBuffers; public FeatureFlags(FlagSource source, ApplicationId appId) { this.defaultTermwiseLimit = flagValue(source, appId, Flags.DEFAULT_TERM_WISE_LIMIT); @@ -242,6 +243,7 @@ public class ModelContextImpl implements ModelContext { this.unorderedMergeChaining = flagValue(source, appId, Flags.UNORDERED_MERGE_CHAINING); this.useV8GeoPositions = flagValue(source, appId, Flags.USE_V8_GEO_POSITIONS); this.useV8DocManagerCfg = flagValue(source, appId, Flags.USE_V8_DOC_MANAGER_CFG); + this.maxCompactBuffers = flagValue(source, appId, Flags.MAX_COMPACT_BUFFERS); } @Override public double defaultTermwiseLimit() { return defaultTermwiseLimit; } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/ModelContextImplTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/ModelContextImplTest.java index b0046a201ab..0e8c69327e2 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/ModelContextImplTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/ModelContextImplTest.java @@ -102,6 +102,7 @@ public class ModelContextImplTest { assertEquals(1.0, context.properties().featureFlags().defaultTermwiseLimit(), 0.0); assertFalse(context.properties().featureFlags().useAsyncMessageHandlingOnSchedule()); assertEquals(0.5, context.properties().featureFlags().feedConcurrency(), 0.0); + assertEquals(1, context.properties().featureFlags().maxCompactBuffers()); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java index e28273870d7..0d56bc286eb 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java @@ -737,6 +737,9 @@ public class InternalStepRunner implements StepRunner { /** Sends a mail with a notification of a failed run, if one should be sent. */ private void sendEmailNotification(Run run, DualLogger logger) { + if ( ! isNewFailure(run)) + return; + Application application = controller.applications().requireApplication(TenantAndApplicationId.from(run.id().application())); Notifications notifications = application.deploymentSpec().requireInstance(run.id().application().instance()).notifications(); boolean newCommit = application.require(run.id().application().instance()).change().application() @@ -760,6 +763,12 @@ public class InternalStepRunner implements StepRunner { } } + private boolean isNewFailure(Run run) { + return controller.jobController().lastCompleted(run.id().job()) + .map(previous -> ! previous.hasFailed() || ! previous.versions().targetsMatch(run.versions())) + .orElse(true); + } + private void updateConsoleNotification(Run run) { NotificationSource source = NotificationSource.from(run.id()); Consumer<String> updater = msg -> controller.notificationsDb().setNotification(source, Notification.Type.deployment, Notification.Level.error, msg); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java index a1c25c1fb53..021b27456a4 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java @@ -195,7 +195,7 @@ public class MetricsReporter extends ControllerMaintainer { Optional<Instant> lastOpened = Optional.empty(); // When the upgrade window most recently opened Instant oneWeekAgo = upgradingAt.minus(Duration.ofDays(7)); Duration step = Duration.ofHours(1); - for (Instant instant = upgradingAt; !instanceSpec.canUpgradeAt(instant); instant = instant.minus(step).truncatedTo(ChronoUnit.HOURS)) { + for (Instant instant = upgradingAt.truncatedTo(ChronoUnit.HOURS); !instanceSpec.canUpgradeAt(instant); instant = instant.minus(step)) { if (!instant.isAfter(oneWeekAgo)) { // Wrapped around, the entire week is being blocked lastOpened = Optional.empty(); break; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java index ae92fd46f26..061cc69fc26 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java @@ -368,9 +368,7 @@ public class InternalStepRunnerTest { @Test public void notificationIsSent() { - app.startSystemTestTests(); - tester.cloud().set(TesterCloud.Status.NOT_STARTED); - tester.runner().run(); + app.submit().failDeployment(JobType.systemTest); MockMailer mailer = tester.controllerTester().serviceRegistry().mailer(); assertEquals(1, mailer.inbox("a@b").size()); assertEquals("Vespa application tenant.application: System test failing due to system error", @@ -378,6 +376,16 @@ public class InternalStepRunnerTest { assertEquals(1, mailer.inbox("b@a").size()); assertEquals("Vespa application tenant.application: System test failing due to system error", mailer.inbox("b@a").get(0).subject()); + + // Re-run failing causes no additional email to be sent. + app.failDeployment(JobType.systemTest); + assertEquals(1, mailer.inbox("a@b").size()); + assertEquals(1, mailer.inbox("b@a").size()); + + // Failure with new package causes new email to be sent. + app.submit().failDeployment(JobType.systemTest); + assertEquals(2, mailer.inbox("a@b").size()); + assertEquals(2, mailer.inbox("b@a").size()); } @Test diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java index 3c91fb66894..71a3ce262ad 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java @@ -510,7 +510,7 @@ public class MetricsReporterTest { .blockChange(true, false, "mon-sun", "0-7", "CET") .build(); - Instant mondayNight = Instant.parse("2021-12-13T23:00:00.00Z"); + Instant mondayNight = Instant.parse("2021-12-13T23:30:00.00Z"); DeploymentTester tester = new DeploymentTester().at(mondayNight); MetricsReporter reporter = createReporter(tester.controller()); DeploymentContext context = tester.newDeploymentContext(); @@ -535,17 +535,23 @@ public class MetricsReporterTest { assertEquals("Upgrade is not overdue yet", Duration.ZERO, metric.get()); // Upgrade continues into block window - tester.clock().advance(Duration.ofHours(3)); // Tuesday at 02:00 (03:00 CET) - assertEquals("Upgrade is overdue measured relative to window 2", Duration.ofHours(2), metric.get()); + tester.clock().advance(Duration.ofHours(1)); // Tuesday at 00:30 (01:30 CET) + assertEquals("Upgrade is overdue measured relative to window 2", Duration.ofHours(0).plusMinutes(30), metric.get()); + + tester.clock().advance(Duration.ofHours(1)); // Tuesday at 01:30 (02:30 CET) + assertEquals("Upgrade is overdue measured relative to window 2", Duration.ofHours(1).plusMinutes(30), metric.get()); + + tester.clock().advance(Duration.ofHours(1)); // Tuesday at 02:30 (03:30 CET) + assertEquals("Upgrade is overdue measured relative to window 2", Duration.ofHours(2).plusMinutes(30), metric.get()); - tester.clock().advance(Duration.ofHours(6)); // Tuesday at 08:00 (09:00 CET) - assertEquals("Upgrade is overdue measured relative to window 1", Duration.ofHours(8), metric.get()); + tester.clock().advance(Duration.ofHours(6)); // Tuesday at 08:30 (09:30 CET) + assertEquals("Upgrade is overdue measured relative to window 1", Duration.ofHours(8).plusMinutes(30), metric.get()); - tester.clock().advance(Duration.ofHours(1)); // Tuesday at 09:00 (10:00 CET) + tester.clock().advance(Duration.ofHours(1)); // Tuesday at 09:30 (10:30 CET) assertEquals("Upgrade is no longer overdue", Duration.ZERO, metric.get()); - tester.clock().advance(Duration.ofDays(2)); // Thursday at 10:00 (11:00 CET) - assertEquals("Upgrade is overdue measure relative to window 3", Duration.ofHours(34), metric.get()); + tester.clock().advance(Duration.ofDays(2)); // Thursday at 10:30 (11:30 CET) + assertEquals("Upgrade is overdue measure relative to window 3", Duration.ofHours(34).plusMinutes(30), metric.get()); } @Test diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index eb68c72eb6a..97b06c802dd 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -379,6 +379,13 @@ public class Flags { "Takes effect at redeployment", ZONE_ID, APPLICATION_ID); + public static final UnboundIntFlag MAX_COMPACT_BUFFERS = defineIntFlag( + "max-compact-buffers", 1, + List.of("baldersheim", "geirst", "toregge"), "2021-12-15", "2022-03-31", + "Upper limit of buffers to compact in a data store at the same time for each reason (memory usage, address space usage)", + "Takes effect at redeployment", + ZONE_ID, APPLICATION_ID); + /** WARNING: public for testing: All flags should be defined in {@link Flags}. */ public static UnboundBooleanFlag defineFeatureFlag(String flagId, boolean defaultValue, List<String> owners, String createdAt, String expiresAt, String description, |