diff options
Diffstat (limited to 'controller-server/src/main/java')
6 files changed, 73 insertions, 17 deletions
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java index e338bc17788..c293e00ae48 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java @@ -241,6 +241,8 @@ public class Controller extends AbstractComponent { return chefClient; } + public CuratorDb curator() { return curator; } + private String printableVersion(Optional<VespaVersion> vespaVersion) { return vespaVersion.map(v -> v.versionNumber().toFullString()).orElse("Unknown"); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java index 0722a58e18d..44b15053a8c 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java @@ -7,13 +7,11 @@ import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.application.ApplicationList; import com.yahoo.vespa.hosted.controller.application.Change; -import com.yahoo.vespa.hosted.controller.deployment.BuildSystem; import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; import com.yahoo.vespa.hosted.controller.versions.VespaVersion; import com.yahoo.yolean.Exceptions; import java.time.Duration; -import java.time.Instant; import java.util.logging.Level; import java.util.logging.Logger; @@ -111,4 +109,18 @@ public class Upgrader extends Maintainer { curator.writeUpgradesPerMinute(n); } + /** + * Returns whether to ignore confidence calculations when upgrading + */ + public boolean ignoreConfidence() { + return curator.readIgnoreConfidence(); + } + + /** + * Controls whether to ignore confidence calculations or not + */ + public void ignoreConfidence(boolean value) { + curator.writeIgnoreConfidence(value); + } + } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java index 03b4dd6efe3..0a288955055 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java @@ -212,6 +212,20 @@ public class CuratorDb { transaction.commit(); } + public boolean readIgnoreConfidence() { + Optional<byte[]> value = curator.getData(ignoreConfidencePath()); + if (! value.isPresent() || value.get().length == 0) { + return false; // Default if value has never been written + } + return ByteBuffer.wrap(value.get()).getInt() == 1; + } + + public void writeIgnoreConfidence(boolean value) { + NestedTransaction transaction = new NestedTransaction(); + curator.set(ignoreConfidencePath(), ByteBuffer.allocate(Integer.BYTES).putInt(value ? 1 : 0).array()); + transaction.commit(); + } + public void writeVersionStatus(VersionStatus status) { VersionStatusSerializer serializer = new VersionStatusSerializer(); NestedTransaction transaction = new NestedTransaction(); @@ -305,6 +319,10 @@ public class CuratorDb { return root.append("upgrader").append("upgradesPerMinute"); } + private Path ignoreConfidencePath() { + return root.append("upgrader").append("ignoreConfidence"); + } + private Path versionStatusPath() { return root.append("versionStatus"); } private Path provisionStatePath() { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiHandler.java index 03ac073a34a..162827cdb99 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiHandler.java @@ -7,9 +7,10 @@ import com.yahoo.container.jdisc.LoggingRequestHandler; import com.yahoo.container.logging.AccessLog; import com.yahoo.io.IOUtils; import com.yahoo.slime.Inspector; -import com.yahoo.slime.Slime; +import com.yahoo.text.Utf8; import com.yahoo.vespa.config.SlimeUtils; import com.yahoo.vespa.hosted.controller.maintenance.ControllerMaintenance; +import com.yahoo.vespa.hosted.controller.maintenance.Upgrader; import com.yahoo.vespa.hosted.controller.restapi.ErrorResponse; import com.yahoo.vespa.hosted.controller.restapi.MessageResponse; import com.yahoo.vespa.hosted.controller.restapi.Path; @@ -62,7 +63,7 @@ public class ControllerApiHandler extends LoggingRequestHandler { Path path = new Path(request.getUri().getPath()); if (path.matches("/controller/v1/")) return root(request); if (path.matches("/controller/v1/maintenance/")) return new JobsResponse(maintenance.jobControl()); - if (path.matches("/controller/v1/jobs/upgrader")) return new UpgraderResponse(maintenance.upgrader().upgradesPerMinute()); + if (path.matches("/controller/v1/jobs/upgrader")) return new UpgraderResponse(maintenance.upgrader()); return notFound(path); } @@ -101,18 +102,26 @@ public class ControllerApiHandler extends LoggingRequestHandler { private HttpResponse configureUpgrader(HttpRequest request) { String upgradesPerMinuteField = "upgradesPerMinute"; - Slime slime = toSlime(request.getData()); - Inspector inspect = slime.get(); + String ignoreConfidenceField = "ignoreConfidence"; + + byte[] jsonBytes = toJsonBytes(request.getData()); + Inspector inspect = SlimeUtils.jsonToSlime(jsonBytes).get(); + Upgrader upgrader = maintenance.upgrader(); if (inspect.field(upgradesPerMinuteField).valid()) { - maintenance.upgrader().setUpgradesPerMinute(inspect.field(upgradesPerMinuteField).asDouble()); + upgrader.setUpgradesPerMinute(inspect.field(upgradesPerMinuteField).asDouble()); + } else if (inspect.field(ignoreConfidenceField).valid()) { + upgrader.ignoreConfidence(inspect.field(ignoreConfidenceField).asBool()); + } else { + return ErrorResponse.badRequest("Unable to configure upgrader with data in request: '" + + Utf8.toString(jsonBytes) + "'"); } - return new UpgraderResponse(maintenance.upgrader().upgradesPerMinute()); + + return new UpgraderResponse(maintenance.upgrader()); } - private Slime toSlime(InputStream jsonStream) { + private byte[] toJsonBytes(InputStream jsonStream) { try { - byte[] jsonBytes = IOUtils.readBytes(jsonStream, 1000 * 1000); - return SlimeUtils.jsonToSlime(jsonBytes); + return IOUtils.readBytes(jsonStream, 1000 * 1000); } catch (IOException e) { throw new UncheckedIOException(e); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/UpgraderResponse.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/UpgraderResponse.java index fe88a0f1f22..3444f710d4c 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/UpgraderResponse.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/UpgraderResponse.java @@ -4,6 +4,7 @@ import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.slime.Cursor; import com.yahoo.slime.JsonFormat; import com.yahoo.slime.Slime; +import com.yahoo.vespa.hosted.controller.maintenance.Upgrader; import java.io.IOException; import java.io.OutputStream; @@ -13,18 +14,19 @@ import java.io.OutputStream; */ public class UpgraderResponse extends HttpResponse { - private final double upgradesPerMinute; + private final Upgrader upgrader; - public UpgraderResponse(double upgradesPerMinute) { + public UpgraderResponse(Upgrader upgrader) { super(200); - this.upgradesPerMinute = upgradesPerMinute; + this.upgrader = upgrader; } @Override public void render(OutputStream outputStream) throws IOException { Slime slime = new Slime(); Cursor root = slime.setObject(); - root.setDouble("upgradesPerMinute", upgradesPerMinute); + root.setDouble("upgradesPerMinute", upgrader.upgradesPerMinute()); + root.setBool("ignoreConfidence", upgrader.ignoreConfidence()); new JsonFormat(true).encode(outputStream, slime); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VespaVersion.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VespaVersion.java index c1b9c045fbe..1541e4d35f8 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VespaVersion.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VespaVersion.java @@ -6,6 +6,7 @@ import com.yahoo.component.Version; import com.yahoo.component.Vtag; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.application.ApplicationList; +import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; import java.time.Instant; import java.util.Collection; @@ -57,8 +58,7 @@ public class VespaVersion implements Comparable<VespaVersion> { return Confidence.broken; // 'broken' if 4 non-canary was broken by this, and that is at least 10% of all - int brokenByThisVersion = failingOnThis.without(UpgradePolicy.canary).startedFailingAfter(releasedAt).size(); - if (brokenByThisVersion >= 4 && brokenByThisVersion >= productionOnThis.size() * 0.1) + if (nonCanaryApplicationsBroken(failingOnThis, productionOnThis, releasedAt, controller.curator())) return Confidence.broken; // 'low' unless all canary applications are upgraded @@ -136,4 +136,17 @@ public class VespaVersion implements Comparable<VespaVersion> { } + private static boolean nonCanaryApplicationsBroken(ApplicationList failingOnThis, + ApplicationList productionOnThis, + Instant releasedAt, + CuratorDb curator) { + ApplicationList failingNonCanaries = failingOnThis.without(UpgradePolicy.canary).startedFailingAfter(releasedAt); + ApplicationList productionNonCanaries = productionOnThis.without(UpgradePolicy.canary); + + if (productionNonCanaries.size() + failingNonCanaries.size() == 0 || curator.readIgnoreConfidence()) return false; + + // 'broken' if 4 non-canary was broken by this, and that is at least 10% of all + int brokenByThisVersion = failingNonCanaries.size(); + return brokenByThisVersion >= 4 && brokenByThisVersion >= productionOnThis.size() * 0.1; + } } |