diff options
53 files changed, 464 insertions, 357 deletions
diff --git a/airlift-zstd/src/test/java/ai/vespa/airlift/zstd/TestCompressor.java b/airlift-zstd/src/test/java/ai/vespa/airlift/zstd/TestCompressor.java index d6f13b98c71..4aa00f91ffc 100644 --- a/airlift-zstd/src/test/java/ai/vespa/airlift/zstd/TestCompressor.java +++ b/airlift-zstd/src/test/java/ai/vespa/airlift/zstd/TestCompressor.java @@ -20,6 +20,7 @@ import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static sun.misc.Unsafe.ARRAY_BYTE_BASE_OFFSET; +@SuppressWarnings("proprietary") public class TestCompressor { @Test diff --git a/client/pom.xml b/client/pom.xml index a310e7d6feb..6da6dc74a82 100644 --- a/client/pom.xml +++ b/client/pom.xml @@ -35,6 +35,12 @@ <artifactId>maven-compiler-plugin</artifactId> <configuration> <release>${vespaClients.jdk.releaseVersion}</release> + <compilerArgs> <!-- Remove (to use default) when not compiling for 8 --> + <arg>-Xlint:all</arg> + <arg>-Xlint:-rawtypes</arg> + <arg>-Xlint:-unchecked</arg> + <arg>-Xlint:-serial</arg> + </compilerArgs> </configuration> </plugin> <plugin> diff --git a/container-search/src/main/java/com/yahoo/prelude/query/MultiRangeItem.java b/container-search/src/main/java/com/yahoo/prelude/query/MultiRangeItem.java index 7ba7a13936f..3dac4cb92c0 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/MultiRangeItem.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/MultiRangeItem.java @@ -271,7 +271,7 @@ public class MultiRangeItem<Type extends Number> extends MultiTermItem { if (endInclusive) metadata |= 0b00000100; encoder = type.encoderFor(sortedRanges()); - metadata |= encoder.id << 3; + metadata |= (byte)(encoder.id << 3); buffer.put(metadata); putString(startIndex, buffer); diff --git a/container-search/src/main/java/com/yahoo/prelude/query/MultiTermItem.java b/container-search/src/main/java/com/yahoo/prelude/query/MultiTermItem.java index a7ca62d153c..03a661499e0 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/MultiTermItem.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/MultiTermItem.java @@ -67,8 +67,8 @@ abstract class MultiTermItem extends SimpleTaggableItem { super.encodeThis(buffer); byte metadata = 0; - metadata |= (operatorType().code << 5) & 0b11100000; - metadata |= ( termType().code ) & 0b00011111; + metadata |= (byte)((byte)(operatorType().code << 5) & (byte)0b11100000); + metadata |= (byte)(termType().code & (byte)0b00011111); buffer.put(metadata); buffer.putInt(terms()); encodeBlueprint(buffer); diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/compiled/Binding.java b/container-search/src/main/java/com/yahoo/search/query/profile/compiled/Binding.java index 99c3477274d..01bbef13129 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profile/compiled/Binding.java +++ b/container-search/src/main/java/com/yahoo/search/query/profile/compiled/Binding.java @@ -48,7 +48,7 @@ public class Binding implements Comparable<Binding> { for (int i = 0; i <= maxDimensions; i++) { String value = i < dimensionBinding.getDimensions().size() ? dimensionBinding.getValues().get(i) : null; if (value == null) - generality += Math.pow(2, maxDimensions - i-1); + generality += (int)Math.pow(2, maxDimensions - i - 1); else context.put(dimensionBinding.getDimensions().get(i), value); } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/ApplicationVersion.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/ApplicationVersion.java index 04604ae7007..eb2005bf268 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/ApplicationVersion.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/ApplicationVersion.java @@ -1,11 +1,9 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.api.integration.deployment; -import ai.vespa.validation.Validation; import com.yahoo.component.Version; import java.time.Instant; -import java.util.Objects; import java.util.Optional; import java.util.OptionalLong; @@ -33,6 +31,7 @@ public class ApplicationVersion implements Comparable<ApplicationVersion> { private final Optional<String> sourceUrl; private final Optional<String> commit; private final Optional<String> bundleHash; + private final Optional<Instant> obsoleteAt; private final boolean hasPackage; private final boolean shouldSkip; private final Optional<String> description; @@ -41,7 +40,7 @@ public class ApplicationVersion implements Comparable<ApplicationVersion> { public ApplicationVersion(RevisionId id, Optional<SourceRevision> source, Optional<String> authorEmail, Optional<Version> compileVersion, Optional<Integer> allowedMajor, Optional<Instant> buildTime, Optional<String> sourceUrl, Optional<String> commit, Optional<String> bundleHash, - boolean hasPackage, boolean shouldSkip, Optional<String> description, int risk) { + Optional<Instant> obsoleteAt, boolean hasPackage, boolean shouldSkip, Optional<String> description, int risk) { if (commit.isPresent() && commit.get().length() > 128) throw new IllegalArgumentException("Commit may not be longer than 128 characters"); @@ -61,6 +60,7 @@ public class ApplicationVersion implements Comparable<ApplicationVersion> { this.sourceUrl = requireNonNull(sourceUrl, "sourceUrl cannot be null"); this.commit = requireNonNull(commit, "commit cannot be null"); this.bundleHash = bundleHash; + this.obsoleteAt = obsoleteAt; this.hasPackage = hasPackage; this.shouldSkip = shouldSkip; this.description = description; @@ -71,19 +71,9 @@ public class ApplicationVersion implements Comparable<ApplicationVersion> { return id; } - /** Create an application package version from a completed build, without an author email */ - public static ApplicationVersion from(RevisionId id, SourceRevision source) { - return new ApplicationVersion(id, Optional.of(source), Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty(), true, false, Optional.empty(), 0); - } - - /** Creates a version from a completed build, an author email, and build metadata. */ - public static ApplicationVersion from(RevisionId id, SourceRevision source, String authorEmail, Version compileVersion, Instant buildTime) { - return new ApplicationVersion(id, Optional.of(source), Optional.of(authorEmail), Optional.of(compileVersion), Optional.empty(), Optional.of(buildTime), Optional.empty(), Optional.empty(), Optional.empty(), true, false, Optional.empty(), 0); - } - /** Creates a minimal version for a development build. */ public static ApplicationVersion forDevelopment(RevisionId id, Optional<Version> compileVersion, Optional<Integer> allowedMajor) { - return new ApplicationVersion(id, Optional.empty(), Optional.empty(), compileVersion, allowedMajor, Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty(), true, false, Optional.empty(), 0); + return new ApplicationVersion(id, Optional.empty(), Optional.empty(), compileVersion, allowedMajor, Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty(), true, false, Optional.empty(), 0); } /** Creates a version from a completed build, an author email, and build metadata. */ @@ -91,7 +81,7 @@ public class ApplicationVersion implements Comparable<ApplicationVersion> { Optional<Version> compileVersion, Optional<Integer> allowedMajor, Optional<Instant> buildTime, Optional<String> sourceUrl, Optional<String> commit, Optional<String> bundleHash, Optional<String> description, int risk) { return new ApplicationVersion(id, source, authorEmail, compileVersion, allowedMajor, buildTime, - sourceUrl, commit, bundleHash, true, false, description, risk); + sourceUrl, commit, bundleHash, Optional.empty(), true, false, description, risk); } /** Returns a unique identifier for this version or "unknown" if version is not known */ @@ -150,7 +140,17 @@ public class ApplicationVersion implements Comparable<ApplicationVersion> { /** Returns a copy of this without a package stored. */ public ApplicationVersion withoutPackage() { - return new ApplicationVersion(id, source, authorEmail, compileVersion, allowedMajor, buildTime, sourceUrl, commit, bundleHash, false, shouldSkip, description, risk); + return new ApplicationVersion(id, source, authorEmail, compileVersion, allowedMajor, buildTime, sourceUrl, commit, bundleHash, obsoleteAt, false, shouldSkip, description, risk); + } + + /** Returns a copy of this which is obsolete now. */ + public ApplicationVersion obsoleteAt(Instant now) { + return new ApplicationVersion(id, source, authorEmail, compileVersion, allowedMajor, buildTime, sourceUrl, commit, bundleHash, Optional.of(now), hasPackage, shouldSkip, description, risk); + } + + /** Returns the instant at which this became obsolete, i.e., no longer relevant for automated deployments. */ + public Optional<Instant> obsoleteAt() { + return obsoleteAt; } /** Whether we still have the package for this revision. */ @@ -160,7 +160,7 @@ public class ApplicationVersion implements Comparable<ApplicationVersion> { /** Returns a copy of this which will not be rolled out to production. */ public ApplicationVersion skipped() { - return new ApplicationVersion(id, source, authorEmail, compileVersion, allowedMajor, buildTime, sourceUrl, commit, bundleHash, hasPackage, true, description, risk); + return new ApplicationVersion(id, source, authorEmail, compileVersion, allowedMajor, buildTime, sourceUrl, commit, bundleHash, obsoleteAt, hasPackage, true, description, risk); } /** Whether we still have the package for this revision. */ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Change.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Change.java index 64cad599168..5ebb3d53529 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Change.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Change.java @@ -2,7 +2,6 @@ package com.yahoo.vespa.hosted.controller.application; import com.yahoo.component.Version; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RevisionId; import java.util.Objects; @@ -23,7 +22,7 @@ import static java.util.Objects.requireNonNull; */ public final class Change { - private static final Change empty = new Change(Optional.empty(), Optional.empty(), false); + private static final Change empty = new Change(Optional.empty(), Optional.empty(), false, false); /** The platform version we are upgrading to, or empty if none */ private final Optional<Version> platform; @@ -32,23 +31,27 @@ public final class Change { private final Optional<RevisionId> revision; /** Whether this change is a pin to its contained Vespa version, or to the application's current. */ - private final boolean pinned; + private final boolean platformPinned; - private Change(Optional<Version> platform, Optional<RevisionId> revision, boolean pinned) { + /** Whether this change is a pin to its contained application revision, or to the application's current. */ + private final boolean revisionPinned; + + private Change(Optional<Version> platform, Optional<RevisionId> revision, boolean platformPinned, boolean revisionPinned) { this.platform = requireNonNull(platform, "platform cannot be null"); this.revision = requireNonNull(revision, "revision cannot be null"); if (revision.isPresent() && ( ! revision.get().isProduction())) { throw new IllegalArgumentException("Application version to deploy must be a known version"); } - this.pinned = pinned; + this.platformPinned = platformPinned; + this.revisionPinned = revisionPinned; } public Change withoutPlatform() { - return new Change(Optional.empty(), revision, pinned); + return new Change(Optional.empty(), revision, platformPinned, revisionPinned); } public Change withoutApplication() { - return new Change(platform, Optional.empty(), pinned); + return new Change(platform, Optional.empty(), platformPinned, revisionPinned); } /** Returns whether a change should currently be deployed */ @@ -58,7 +61,7 @@ public final class Change { /** Returns whether this is the empty change. */ public boolean isEmpty() { - return ! hasTargets() && ! pinned; + return ! hasTargets() && ! platformPinned && ! revisionPinned; } /** Returns the platform version carried by this. */ @@ -67,42 +70,55 @@ public final class Change { /** Returns the application version carried by this. */ public Optional<RevisionId> revision() { return revision; } - public boolean isPinned() { return pinned; } + public boolean isPlatformPinned() { return platformPinned; } + + public boolean isRevisionPinned() { return revisionPinned; } /** Returns an instance representing no change */ public static Change empty() { return empty; } /** Returns a version of this change which replaces or adds this platform change */ public Change with(Version platformVersion) { - if (pinned) + if (platformPinned) throw new IllegalArgumentException("Not allowed to set a platform version when pinned."); - return new Change(Optional.of(platformVersion), revision, pinned); + return new Change(Optional.of(platformVersion), revision, platformPinned, revisionPinned); } /** Returns a version of this change which replaces or adds this revision change */ public Change with(RevisionId revision) { - return new Change(platform, Optional.of(revision), pinned); + if (revisionPinned) + throw new IllegalArgumentException("Not allowed to set a revision when pinned."); + + return new Change(platform, Optional.of(revision), platformPinned, revisionPinned); + } + + /** Returns a change with the versions of this, and with the platform version pinned. */ + public Change withPlatformPin() { + return new Change(platform, revision, true, revisionPinned); + } + + /** Returns a change with the versions of this, and with the platform version unpinned. */ + public Change withoutPlatformPin() { + return new Change(platform, revision, false, revisionPinned); } /** Returns a change with the versions of this, and with the platform version pinned. */ - public Change withPin() { - return new Change(platform, revision, true); + public Change withRevisionPin() { + return new Change(platform, revision, platformPinned, true); } /** Returns a change with the versions of this, and with the platform version unpinned. */ - public Change withoutPin() { - return new Change(platform, revision, false); + public Change withoutRevisionPin() { + return new Change(platform, revision, platformPinned, false); } /** Returns the change obtained when overwriting elements of the given change with any present in this */ public Change onTopOf(Change other) { - if (platform.isPresent()) - other = other.with(platform.get()); - if (revision.isPresent()) - other = other.with(revision.get()); - if (pinned) - other = other.withPin(); + if (platform.isPresent()) other = other.with(platform.get()); + if (revision.isPresent()) other = other.with(revision.get()); + if (platformPinned) other = other.withPlatformPin(); + if (revisionPinned) other = other.withRevisionPin(); return other; } @@ -111,34 +127,38 @@ public final class Change { if (this == o) return true; if (!(o instanceof Change)) return false; Change change = (Change) o; - return pinned == change.pinned && + return platformPinned == change.platformPinned && + revisionPinned == change.revisionPinned && Objects.equals(platform, change.platform) && Objects.equals(revision, change.revision); } @Override public int hashCode() { - return Objects.hash(platform, revision, pinned); + return Objects.hash(platform, revision, platformPinned, revisionPinned); } @Override public String toString() { StringJoiner changes = new StringJoiner(" and "); - if (pinned) + if (platformPinned) changes.add("pin to " + platform.map(Version::toString).orElse("current platform")); else platform.ifPresent(version -> changes.add("upgrade to " + version)); - revision.ifPresent(revision -> changes.add("revision change to " + revision)); + if (revisionPinned) + changes.add("pin to " + revision.map(RevisionId::toString).orElse("current revision")); + else + revision.ifPresent(revision -> changes.add("revision change to " + revision)); changes.setEmptyValue("no change"); return changes.toString(); } public static Change of(RevisionId revision) { - return new Change(Optional.empty(), Optional.of(revision), false); + return new Change(Optional.empty(), Optional.of(revision), false, false); } public static Change of(Version platformChange) { - return new Change(Optional.of(platformChange), Optional.empty(), false); + return new Change(Optional.of(platformChange), Optional.empty(), false, false); } /** Returns whether this change carries a revision downgrade relative to the given revision. */ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/InstanceList.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/InstanceList.java index c1bf083b26c..b94779994e4 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/InstanceList.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/InstanceList.java @@ -125,7 +125,7 @@ public class InstanceList extends AbstractFilteringList<ApplicationId, InstanceL /** Returns the subset of instances which are not pinned to a certain Vespa version. */ public InstanceList unpinned() { - return matching(id -> ! instance(id).change().isPinned()); + return matching(id -> ! instance(id).change().isPlatformPinned()); } /** Returns the subset of instances which are currently failing a job. */ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatus.java index 00da34fe2e4..0f1bbfeb25e 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatus.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatus.java @@ -229,7 +229,7 @@ public class DeploymentStatus { .anyMatch(deployment -> ! compatibleWithCompileVersion.test(deployment.version()))) { for (Version platform : targetsForPolicy(versionStatus, systemVersion, application.deploymentSpec().requireInstance(instance).upgradePolicy())) if (compatibleWithCompileVersion.test(platform)) - return change.withoutPin().with(platform); + return change.withoutPlatformPin().with(platform); } return change; } @@ -265,7 +265,7 @@ public class DeploymentStatus { for (InstanceName instance : application.deploymentSpec().instanceNames()) { Change outstanding = outstandingChange(instance); if (outstanding.hasTargets()) - outstandingChanges.put(instance, outstanding.onTopOf(application.require(instance).change())); + outstandingChanges.put(instance, outstanding.onTopOf(application.require(instance).change().withoutRevisionPin())); } var testJobs = jobsToRun(outstandingChanges, true).entrySet().stream() .filter(entry -> ! entry.getKey().type().isProduction()); @@ -596,7 +596,8 @@ public class DeploymentStatus { /** Changes to deploy with the given job, possibly split in two steps. */ private List<Change> changes(JobId job, StepStatus step, Change change) { - if (change.platform().isEmpty() || change.revision().isEmpty() || change.isPinned()) + if ( change.platform().isEmpty() || change.revision().isEmpty() + || change.isPlatformPinned() || change.isRevisionPinned()) return List.of(change); if ( step.completedAt(change.withoutApplication(), Optional.of(job)).isPresent() @@ -1090,14 +1091,14 @@ public class DeploymentStatus { /** Complete if deployment is on pinned version, and last successful deployment, or if given versions is strictly a downgrade, and this isn't forced by a pin. */ @Override Optional<Instant> completedAt(Change change, Optional<JobId> dependent) { - if ( change.isPinned() + if ( change.isPlatformPinned() && change.platform().isPresent() && ! existingDeployment.map(Deployment::version).equals(change.platform())) return Optional.empty(); if ( change.revision().isPresent() - && ! existingDeployment.map(Deployment::revision).equals(change.revision()) - && dependent.equals(job())) // Job should (re-)run in this case, but other dependents need not wait. + && change.isRevisionPinned() + && ! existingDeployment.map(Deployment::revision).equals(change.revision())) return Optional.empty(); Change fullChange = status.application().require(job.id().application().instance()).change(); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java index 00a0e22f87d..4e699f2c28f 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java @@ -41,7 +41,6 @@ import java.util.logging.Logger; import java.util.stream.Collectors; import static java.util.Comparator.comparing; -import static java.util.Comparator.comparingDouble; import static java.util.stream.Collectors.groupingBy; import static java.util.stream.Collectors.toMap; @@ -331,15 +330,14 @@ public class DeploymentTrigger { /** Cancels the indicated part of the given application's change. */ public void cancelChange(ApplicationId instanceId, ChangesToCancel cancellation) { applications().lockApplicationOrThrow(TenantAndApplicationId.from(instanceId), application -> { - Change change; - switch (cancellation) { - case ALL: change = Change.empty(); break; - case VERSIONS: change = Change.empty().withPin(); break; - case PLATFORM: change = application.get().require(instanceId.instance()).change().withoutPlatform(); break; - case APPLICATION: change = application.get().require(instanceId.instance()).change().withoutApplication(); break; - case PIN: change = application.get().require(instanceId.instance()).change().withoutPin(); break; - default: throw new IllegalArgumentException("Unknown cancellation choice '" + cancellation + "'!"); - } + Change change = switch (cancellation) { + case ALL -> Change.empty(); + case PLATFORM -> application.get().require(instanceId.instance()).change().withoutPlatform(); + case APPLICATION -> application.get().require(instanceId.instance()).change().withoutApplication(); + case PIN -> application.get().require(instanceId.instance()).change().withoutPlatformPin(); + case PLATFORM_PIN -> application.get().require(instanceId.instance()).change().withoutPlatformPin(); + case APPLICATION_PIN -> application.get().require(instanceId.instance()).change().withoutRevisionPin(); + }; applications().store(application.with(instanceId.instance(), instance -> withRemainingChange(instance, change, @@ -348,7 +346,7 @@ public class DeploymentTrigger { }); } - public enum ChangesToCancel { ALL, PLATFORM, APPLICATION, VERSIONS, PIN } + public enum ChangesToCancel { ALL, PLATFORM, APPLICATION, PIN, PLATFORM_PIN, APPLICATION_PIN } // ---------- Conveniences ---------- diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java index 7b1a1e879d6..52ddcfd5171 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java @@ -437,7 +437,7 @@ public class InternalStepRunner implements StepRunner { Version targetPlatform = controller.jobController().run(id).versions().targetPlatform(); Version systemVersion = controller.readSystemVersion(); boolean incompatible = controller.applications().versionCompatibility(id.application()).refuse(targetPlatform, systemVersion); - return incompatible || application(id.application()).change().isPinned() ? targetPlatform : systemVersion; + return incompatible || application(id.application()).change().isPlatformPinned() ? targetPlatform : systemVersion; } private Optional<RunStatus> installTester(RunId id, DualLogger logger) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java index 10e4052f067..318a6ffe820 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java @@ -111,6 +111,7 @@ import static java.util.logging.Level.WARNING; public class JobController { public static final Duration maxHistoryAge = Duration.ofDays(60); + public static final Duration obsoletePackageExpiry = Duration.ofDays(7); private static final Logger log = Logger.getLogger(JobController.class.getName()); @@ -165,8 +166,8 @@ public class JobController { return Optional.empty(); return active(id).isPresent() - ? Optional.of(logs.readActive(id.application(), id.type(), after)) - : logs.readFinished(id, after); + ? Optional.of(logs.readActive(id.application(), id.type(), after)) + : logs.readFinished(id, after); } } @@ -284,10 +285,10 @@ public class JobController { private Optional<InputStream> getVespaLogsFromLogserver(Run run, long fromMillis, boolean tester) { return deploymentCompletedAt(run, tester).map(at -> - controller.serviceRegistry().configServer().getLogs(new DeploymentId(tester ? run.id().tester().id() : run.id().application(), - run.id().type().zone()), - Map.of("from", Long.toString(Math.max(fromMillis, at.toEpochMilli())), - "to", Long.toString(run.end().orElse(controller.clock().instant()).toEpochMilli())))); + controller.serviceRegistry().configServer().getLogs(new DeploymentId(tester ? run.id().tester().id() : run.id().application(), + run.id().type().zone()), + Map.of("from", Long.toString(Math.max(fromMillis, at.toEpochMilli())), + "to", Long.toString(run.end().orElse(controller.clock().instant()).toEpochMilli())))); } /** Fetches any new test log entries, and records the id of the last of these, for continuation. */ @@ -509,14 +510,14 @@ public class JobController { long successes = runs.values().stream().filter(Run::hasSucceeded).count(); var oldEntries = runs.entrySet().iterator(); for (var old = oldEntries.next(); - old.getKey().number() <= last - historyLength + old.getKey().number() <= last - historyLength || old.getValue().start().isBefore(controller.clock().instant().minus(maxHistoryAge)); old = oldEntries.next()) { // Make sure we keep the last success and the first failing if ( successes == 1 - && old.getValue().hasSucceeded() - && ! old.getValue().start().isBefore(controller.clock().instant().minus(maxHistoryAge))) { + && old.getValue().hasSucceeded() + && ! old.getValue().start().isBefore(controller.clock().instant().minus(maxHistoryAge))) { oldEntries.next(); continue; } @@ -624,7 +625,7 @@ public class JobController { }); } - private LockedApplication withPrunedPackages(LockedApplication application, RevisionId latest){ + private LockedApplication withPrunedPackages(LockedApplication application, RevisionId latest) { TenantAndApplicationId id = application.get().id(); Application wrapped = application.get(); RevisionId oldestDeployed = application.get().oldestDeployedRevision() @@ -632,11 +633,28 @@ public class JobController { .flatMap(instance -> instance.change().revision().stream()) .min(naturalOrder())) .orElse(latest); - controller.applications().applicationStore().prune(id.tenant(), id.application(), oldestDeployed); + RevisionId oldestToKeep = null; + Instant now = controller.clock().instant(); + for (ApplicationVersion version : application.get().revisions().withPackage()) { + if (version.id().compareTo(oldestDeployed) < 0) { + if (version.obsoleteAt().isEmpty()) { + application = application.withRevisions(revisions -> revisions.with(version.obsoleteAt(now))); + if (oldestToKeep == null) + oldestToKeep = version.id(); + } + else { + if (oldestToKeep == null && !version.obsoleteAt().get().isBefore(now.minus(obsoletePackageExpiry))) + oldestToKeep = version.id(); + } + } + } - for (ApplicationVersion version : application.get().revisions().withPackage()) - if (version.id().compareTo(oldestDeployed) < 0) - application = application.withRevisions(revisions -> revisions.with(version.withoutPackage())); + if (oldestToKeep != null) { + controller.applications().applicationStore().prune(id.tenant(), id.application(), oldestToKeep); + for (ApplicationVersion version : application.get().revisions().withPackage()) + if (version.id().compareTo(oldestToKeep) < 0) + application = application.withRevisions(revisions -> revisions.with(version.withoutPackage())); + } return application; } @@ -703,8 +721,8 @@ public class JobController { VersionStatus versionStatus = controller.readVersionStatus(); if ( ! controller.system().isCd() - && platform.isPresent() - && versionStatus.deployableVersions().stream().map(VespaVersion::versionNumber).noneMatch(platform.get()::equals)) + && platform.isPresent() + && versionStatus.deployableVersions().stream().map(VespaVersion::versionNumber).noneMatch(platform.get()::equals)) throw new IllegalArgumentException("platform version " + platform.get() + " is not present in this system"); controller.applications().lockApplicationOrThrow(TenantAndApplicationId.from(id), application -> { @@ -731,8 +749,8 @@ public class JobController { controller.applications().lockApplicationOrThrow(TenantAndApplicationId.from(id), application -> { Version targetPlatform = platform.orElseGet(() -> findTargetPlatform(applicationPackage, deploymentId, application.get().get(id.instance()), versionStatus)); if ( ! allowOutdatedPlatform - && ! controller.readVersionStatus().isOnCurrentMajor(targetPlatform) - && runs(id, type).values().stream().noneMatch(run -> run.versions().targetPlatform().getMajor() == targetPlatform.getMajor())) + && ! controller.readVersionStatus().isOnCurrentMajor(targetPlatform) + && runs(id, type).values().stream().noneMatch(run -> run.versions().targetPlatform().getMajor() == targetPlatform.getMajor())) throw new IllegalArgumentException("platform version " + targetPlatform + " is not on a current major version in this system"); controller.applications().applicationStore().putDev(deploymentId, version.id(), applicationPackage.zippedContent(), diff); @@ -872,7 +890,7 @@ public class JobController { /** Locks all runs and modifies the list of historic runs for the given application and job type. */ private void locked(ApplicationId id, JobType type, Consumer<SortedMap<RunId, Run>> modifications) { - try (Mutex __ = curator.lock(id, type)) { + try (Mutex __ = curator.lock(id, type)) { SortedMap<RunId, Run> runs = new TreeMap<>(curator.readHistoricRuns(id, type)); modifications.accept(runs); curator.writeHistoricRuns(id, type, runs.values()); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RevisionHistory.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RevisionHistory.java index bbab9487ea2..272417ba0ac 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RevisionHistory.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RevisionHistory.java @@ -93,7 +93,7 @@ public class RevisionHistory { // Fallback for when an application version isn't known for the given key. private static ApplicationVersion revisionOf(RevisionId id) { - return new ApplicationVersion(id, Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty(), false, false, Optional.empty(), 0); + return new ApplicationVersion(id, Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty(), false, false, Optional.empty(), 0); } /** Returns the production {@link ApplicationVersion} with this revision ID. */ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Versions.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Versions.java index e7371561636..f752e396c09 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Versions.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Versions.java @@ -126,7 +126,7 @@ public class Versions { private static Version targetPlatform(Application application, Change change, Optional<Version> existing, Supplier<Version> defaultVersion) { - if (change.isPinned() && change.platform().isPresent()) + if (change.isPlatformPinned() && change.platform().isPresent()) return change.platform().get(); return max(change.platform(), existing) @@ -135,6 +135,9 @@ public class Versions { private static RevisionId targetRevision(Application application, Change change, Optional<RevisionId> existing) { + if (change.isRevisionPinned() && change.revision().isPresent()) + return change.revision().get(); + return change.revision() .or(() -> existing) .orElseGet(() -> defaultRevision(application)); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java index ee12c9957b1..e5006ab9785 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java @@ -82,7 +82,8 @@ public class ApplicationSerializer { private static final String versionsField = "versions"; private static final String prodVersionsField = "prodVersions"; private static final String devVersionsField = "devVersions"; - private static final String pinnedField = "pinned"; + private static final String platformPinnedField = "pinned"; + private static final String revisionPinnedField = "revisionPinned"; private static final String deploymentIssueField = "deploymentIssueId"; private static final String ownershipIssueIdField = "ownershipIssueId"; private static final String ownerField = "confirmedOwner"; @@ -118,6 +119,7 @@ public class ApplicationSerializer { private static final String riskField = "risk"; private static final String authorEmailField = "authorEmailField"; private static final String deployedDirectlyField = "deployedDirectly"; + private static final String obsoleteAtField = "obsoleteAt"; private static final String hasPackageField = "hasPackage"; private static final String shouldSkipField = "shouldSkip"; private static final String compileVersionField = "compileVersion"; @@ -265,6 +267,7 @@ public class ApplicationSerializer { applicationVersion.sourceUrl().ifPresent(url -> object.setString(sourceUrlField, url)); applicationVersion.commit().ifPresent(commit -> object.setString(commitField, commit)); object.setBool(deployedDirectlyField, applicationVersion.isDeployedDirectly()); + applicationVersion.obsoleteAt().ifPresent(at -> object.setLong(obsoleteAtField, at.toEpochMilli())); object.setBool(hasPackageField, applicationVersion.hasPackage()); object.setBool(shouldSkipField, applicationVersion.shouldSkip()); applicationVersion.description().ifPresent(description -> object.setString(descriptionField, description)); @@ -295,8 +298,10 @@ public class ApplicationSerializer { object.setString(versionField, deploying.platform().get().toString()); if (deploying.revision().isPresent()) toSlime(deploying.revision().get(), object); - if (deploying.isPinned()) - object.setBool(pinnedField, true); + if (deploying.isPlatformPinned()) + object.setBool(platformPinnedField, true); + if (deploying.isRevisionPinned()) + object.setBool(revisionPinnedField, true); } private void toSlime(RotationStatus status, Cursor array) { @@ -487,6 +492,7 @@ public class ApplicationSerializer { Optional<Instant> buildTime = SlimeUtils.optionalInstant(object.field(buildTimeField)); Optional<String> sourceUrl = SlimeUtils.optionalString(object.field(sourceUrlField)); Optional<String> commit = SlimeUtils.optionalString(object.field(commitField)); + Optional<Instant> obsoleteAt = SlimeUtils.optionalInstant(object.field(obsoleteAtField)); boolean hasPackage = object.field(hasPackageField).asBool(); boolean shouldSkip = object.field(shouldSkipField).asBool(); Optional<String> description = SlimeUtils.optionalString(object.field(descriptionField)); @@ -494,7 +500,7 @@ public class ApplicationSerializer { Optional<String> bundleHash = SlimeUtils.optionalString(object.field(bundleHashField)); return new ApplicationVersion(id, sourceRevision, authorEmail, compileVersion, allowedMajor, buildTime, - sourceUrl, commit, bundleHash, hasPackage, shouldSkip, description, risk); + sourceUrl, commit, bundleHash, obsoleteAt, hasPackage, shouldSkip, description, risk); } private Optional<SourceRevision> sourceRevisionFromSlime(Inspector object) { @@ -520,8 +526,10 @@ public class ApplicationSerializer { change = Change.of(Version.fromString(versionFieldValue.asString())); if (object.field(applicationBuildNumberField).valid()) change = change.with(revisionFromSlime(object, null)); - if (object.field(pinnedField).asBool()) - change = change.withPin(); + if (object.field(platformPinnedField).asBool()) + change = change.withPlatformPin(); + if (object.field(revisionPinnedField).asBool()) + change = change.withRevisionPin(); return change; } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index 81988753621..ded27ee1060 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -260,11 +260,9 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { if (path.matches("/application/v4/tenant/{tenant}/application/{application}/package")) return applicationPackage(path.get("tenant"), path.get("application"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/diff/{number}")) return applicationPackageDiff(path.get("tenant"), path.get("application"), path.get("number")); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/deploying")) return deploying(path.get("tenant"), path.get("application"), "default", request); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/deploying/pin")) return deploying(path.get("tenant"), path.get("application"), "default", request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance")) return applications(path.get("tenant"), Optional.of(path.get("application")), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}")) return instance(path.get("tenant"), path.get("application"), path.get("instance"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/deploying")) return deploying(path.get("tenant"), path.get("application"), path.get("instance"), request); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/deploying/pin")) return deploying(path.get("tenant"), path.get("application"), path.get("instance"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job")) return JobControllerApiHandlerHelper.jobTypeResponse(controller, appIdFromPath(path), request.getUri()); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job/{jobtype}")) return JobControllerApiHandlerHelper.runResponse(controller.applications().requireApplication(TenantAndApplicationId.from(path.get("tenant"), path.get("application"))), controller.jobController().runs(appIdFromPath(path), jobTypeFromPath(path)).descendingMap(), Optional.ofNullable(request.getProperty("limit")), request.getUri()); // (((＼（✘෴✘）／))) if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job/{jobtype}/package")) return devApplicationPackage(appIdFromPath(path), jobTypeFromPath(path)); @@ -327,14 +325,18 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { if (path.matches("/application/v4/tenant/{tenant}/application/{application}")) return createApplication(path.get("tenant"), path.get("application"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/deploying/platform")) return deployPlatform(path.get("tenant"), path.get("application"), "default", false, request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/deploying/pin")) return deployPlatform(path.get("tenant"), path.get("application"), "default", true, request); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/deploying/application")) return deployApplication(path.get("tenant"), path.get("application"), "default", request); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/deploying/platform-pin")) return deployPlatform(path.get("tenant"), path.get("application"), "default", true, request); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/deploying/application-pin")) return deployApplication(path.get("tenant"), path.get("application"), "default", true, request); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/deploying/application")) return deployApplication(path.get("tenant"), path.get("application"), "default", false, request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/key")) return addDeployKey(path.get("tenant"), path.get("application"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/submit")) return submit(path.get("tenant"), path.get("application"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}")) return createInstance(path.get("tenant"), path.get("application"), path.get("instance"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/deploy/{jobtype}")) return jobDeploy(appIdFromPath(path), jobTypeFromPath(path), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/deploying/platform")) return deployPlatform(path.get("tenant"), path.get("application"), path.get("instance"), false, request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/deploying/pin")) return deployPlatform(path.get("tenant"), path.get("application"), path.get("instance"), true, request); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/deploying/application")) return deployApplication(path.get("tenant"), path.get("application"), path.get("instance"), request); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/deploying/platform-pin")) return deployPlatform(path.get("tenant"), path.get("application"), path.get("instance"), true, request); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/deploying/application-pin")) return deployApplication(path.get("tenant"), path.get("application"), path.get("instance"), true, request); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/deploying/application")) return deployApplication(path.get("tenant"), path.get("application"), path.get("instance"), false, request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/submit")) return submit(path.get("tenant"), path.get("application"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job/{jobtype}")) return trigger(appIdFromPath(path), jobTypeFromPath(path), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job/{jobtype}/pause")) return pause(appIdFromPath(path), jobTypeFromPath(path)); @@ -2059,7 +2061,9 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { if ( ! instance.change().isEmpty()) { instance.change().platform().ifPresent(version -> root.setString("platform", version.toString())); instance.change().revision().ifPresent(revision -> root.setString("application", revision.toString())); - root.setBool("pinned", instance.change().isPinned()); + root.setBool("pinned", instance.change().isPlatformPinned()); + root.setBool("platform-pinned", instance.change().isPlatformPinned()); + root.setBool("application-pinned", instance.change().isRevisionPinned()); } return new SlimeJsonResponse(slime); } @@ -2172,7 +2176,7 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { .collect(joining(", "))); Change change = Change.of(version); if (pin) - change = change.withPin(); + change = change.withPlatformPin(); controller.applications().deploymentTrigger().forceChange(id, change, isOperator(request)); response.append("Triggered ").append(change).append(" for ").append(id); @@ -2181,7 +2185,7 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { } /** Trigger deployment to the last known application package for the given application. */ - private HttpResponse deployApplication(String tenantName, String applicationName, String instanceName, HttpRequest request) { + private HttpResponse deployApplication(String tenantName, String applicationName, String instanceName, boolean pin, HttpRequest request) { ApplicationId id = ApplicationId.from(tenantName, applicationName, instanceName); Inspector buildField = toSlime(request.getData()).get().field("build"); long build = buildField.valid() ? buildField.asLong() : -1; @@ -2191,6 +2195,8 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { RevisionId revision = build == -1 ? application.get().revisions().last().get().id() : getRevision(application.get(), build); Change change = Change.of(revision); + if (pin) + change = change.withRevisionPin(); controller.applications().deploymentTrigger().forceChange(id, change, isOperator(request)); response.append("Triggered ").append(change).append(" for ").append(id); }); @@ -2231,7 +2237,7 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { return; } - ChangesToCancel cancel = ChangesToCancel.valueOf(choice.toUpperCase()); + ChangesToCancel cancel = ChangesToCancel.valueOf(choice.replaceAll("-", "_").toUpperCase()); controller.applications().deploymentTrigger().cancelChange(id, cancel); response.append("Changed deployment from '").append(change).append("' to '").append(controller.applications().requireInstance(id).change()).append("' for ").append(id); }); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java index 804ae7b7805..9ff8c7df18b 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java @@ -312,7 +312,9 @@ class JobControllerApiHandlerHelper { if ( ! change.isEmpty()) { change.platform().ifPresent(version -> deployingObject.setString("platform", version.toFullString())); change.revision().ifPresent(revision -> toSlime(deployingObject.setObject("application"), application.revisions().get(revision))); - if (change.isPinned()) deployingObject.setBool("pinned", true); + if (change.isPlatformPinned()) deployingObject.setBool("pinned", true); + if (change.isPlatformPinned()) deployingObject.setBool("platformPinned", true); + if (change.isRevisionPinned()) deployingObject.setBool("revisionPinned", true); } Cursor latestVersionsObject = stepObject.setObject("latestVersions"); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java index 069ee58e9c5..6e5635e8c8c 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java @@ -171,7 +171,9 @@ public class DeploymentApiHandler extends ThreadedHttpRequestHandler { instanceObject.setString("application", instance.application().value()); instanceObject.setString("instance", instance.instance().value()); instanceObject.setBool("upgrading", status.application().require(instance.instance()).change().platform().equals(Optional.of(statistics.version()))); - instanceObject.setBool("pinned", status.application().require(instance.instance()).change().isPinned()); + instanceObject.setBool("pinned", status.application().require(instance.instance()).change().isPlatformPinned()); + instanceObject.setBool("platformPinned", status.application().require(instance.instance()).change().isPlatformPinned()); + instanceObject.setBool("revisionPinned", status.application().require(instance.instance()).change().isRevisionPinned()); DeploymentStatus.StepStatus stepStatus = status.instanceSteps().get(instance.instance()); if (stepStatus != null) { // Instance may not have any steps, i.e. an empty deployment spec has been submitted Readiness platformReadiness = stepStatus.blockedUntil(Change.of(statistics.version())); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java index a9a6fe602b6..04c8c46e1ef 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java @@ -41,6 +41,7 @@ import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; import com.yahoo.vespa.hosted.controller.deployment.DeploymentContext; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; +import com.yahoo.vespa.hosted.controller.deployment.JobController; import com.yahoo.vespa.hosted.controller.deployment.Submission; import com.yahoo.vespa.hosted.controller.integration.ZoneApiMock; import com.yahoo.vespa.hosted.controller.notification.Notification; @@ -106,9 +107,11 @@ public class ControllerTest { Version version1 = tester.configServer().initialVersion(); var context = tester.newDeploymentContext(); context.submit(applicationPackage); - assertEquals(ApplicationVersion.from(RevisionId.forProduction(1), DeploymentContext.defaultSourceRevision, "a@b", new Version("6.1"), Instant.ofEpochSecond(1)), - context.application().revisions().get(context.instance().change().revision().get()), - "Application version is known from completion of initial job"); + RevisionId id = RevisionId.forProduction(1); + Version compileVersion = new Version("6.1"); + assertEquals(new ApplicationVersion(id, Optional.of(DeploymentContext.defaultSourceRevision), Optional.of("a@b"), Optional.of(compileVersion), Optional.empty(), Optional.of(Instant.ofEpochSecond(1)), Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty(), true, false, Optional.empty(), 0), + context.application().revisions().get(context.instance().change().revision().get()), + "Application version is known from completion of initial job"); context.runJob(systemTest); context.runJob(stagingTest); @@ -220,6 +223,59 @@ public class ControllerTest { } @Test + void testPackagePruning() { + DeploymentContext app = tester.newDeploymentContext().submit().deploy(); + RevisionId revision1 = app.lastSubmission().get(); + assertTrue(tester.controllerTester().serviceRegistry().applicationStore() + .hasBuild(app.instanceId().tenant(), app.instanceId().application(), revision1.number())); + + app.submit().deploy(); + RevisionId revision2 = app.lastSubmission().get(); + assertTrue(tester.controllerTester().serviceRegistry().applicationStore() + .hasBuild(app.instanceId().tenant(), app.instanceId().application(), revision1.number())); + assertTrue(tester.controllerTester().serviceRegistry().applicationStore() + .hasBuild(app.instanceId().tenant(), app.instanceId().application(), revision2.number())); + + // Revision 1 is marked as obsolete now + app.submit().deploy(); + RevisionId revision3 = app.lastSubmission().get(); + assertTrue(tester.controllerTester().serviceRegistry().applicationStore() + .hasBuild(app.instanceId().tenant(), app.instanceId().application(), revision1.number())); + assertTrue(tester.controllerTester().serviceRegistry().applicationStore() + .hasBuild(app.instanceId().tenant(), app.instanceId().application(), revision2.number())); + assertTrue(tester.controllerTester().serviceRegistry().applicationStore() + .hasBuild(app.instanceId().tenant(), app.instanceId().application(), revision3.number())); + + // Time advances, and revision 2 is marked as obsolete now + tester.clock().advance(JobController.obsoletePackageExpiry); + app.submit().deploy(); + RevisionId revision4 = app.lastSubmission().get(); + assertTrue(tester.controllerTester().serviceRegistry().applicationStore() + .hasBuild(app.instanceId().tenant(), app.instanceId().application(), revision1.number())); + assertTrue(tester.controllerTester().serviceRegistry().applicationStore() + .hasBuild(app.instanceId().tenant(), app.instanceId().application(), revision2.number())); + assertTrue(tester.controllerTester().serviceRegistry().applicationStore() + .hasBuild(app.instanceId().tenant(), app.instanceId().application(), revision3.number())); + assertTrue(tester.controllerTester().serviceRegistry().applicationStore() + .hasBuild(app.instanceId().tenant(), app.instanceId().application(), revision4.number())); + + // Time advances, and revision is now old enough to be pruned + tester.clock().advance(Duration.ofMillis(1)); + app.submit().deploy(); + RevisionId revision5 = app.lastSubmission().get(); + assertFalse(tester.controllerTester().serviceRegistry().applicationStore() + .hasBuild(app.instanceId().tenant(), app.instanceId().application(), revision1.number())); + assertTrue(tester.controllerTester().serviceRegistry().applicationStore() + .hasBuild(app.instanceId().tenant(), app.instanceId().application(), revision2.number())); + assertTrue(tester.controllerTester().serviceRegistry().applicationStore() + .hasBuild(app.instanceId().tenant(), app.instanceId().application(), revision3.number())); + assertTrue(tester.controllerTester().serviceRegistry().applicationStore() + .hasBuild(app.instanceId().tenant(), app.instanceId().application(), revision4.number())); + assertTrue(tester.controllerTester().serviceRegistry().applicationStore() + .hasBuild(app.instanceId().tenant(), app.instanceId().application(), revision5.number())); + } + + @Test void testGlobalRotationStatus() { var context = tester.newDeploymentContext(); var zone1 = ZoneId.from("prod", "us-west-1"); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java index afb92d84f3b..6e5c2458c92 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java @@ -653,15 +653,21 @@ public class DeploymentTriggerTest { assertEquals(appVersion1, latestDeployed(app.instance())); // Downgrading application version. - tester.deploymentTrigger().forceChange(app.instanceId(), Change.of(appVersion0)); - assertEquals(Change.of(appVersion0), app.instance().change()); + tester.deploymentTrigger().forceChange(app.instanceId(), Change.of(appVersion0).withRevisionPin()); + assertEquals(Change.of(appVersion0).withRevisionPin(), app.instance().change()); app.runJob(stagingTest) - .runJob(productionUsCentral1) - .runJob(productionUsEast3) - .runJob(productionUsWest1); - assertEquals(Change.empty(), app.instance().change()); + .runJob(productionUsCentral1) + .runJob(productionUsEast3) + .runJob(productionUsWest1); + assertEquals(Change.empty().withRevisionPin(), app.instance().change()); assertEquals(appVersion0, app.instance().deployments().get(productionUsEast3.zone()).revision()); assertEquals(appVersion0, latestDeployed(app.instance())); + + tester.outstandingChangeDeployer().run(); + assertEquals(Change.empty().withRevisionPin(), app.instance().change()); + tester.deploymentTrigger().cancelChange(app.instanceId(), ALL); + tester.outstandingChangeDeployer().run(); + assertEquals(Change.of(appVersion1), app.instance().change()); } @Test @@ -1239,13 +1245,13 @@ public class DeploymentTriggerTest { assertEquals(Change.empty(), app.instance().change()); // Application is pinned to previous version, and downgrades to that. Tests are re-run. - tester.deploymentTrigger().forceChange(app.instanceId(), Change.of(version0).withPin()); + tester.deploymentTrigger().forceChange(app.instanceId(), Change.of(version0).withPlatformPin()); app.runJob(stagingTest).runJob(productionUsEast3); tester.clock().advance(Duration.ofMinutes(1)); app.failDeployment(testUsEast3); tester.clock().advance(Duration.ofMinutes(11)); // Job is cooling down after consecutive failures. app.runJob(testUsEast3); - assertEquals(Change.empty().withPin(), app.instance().change()); + assertEquals(Change.empty().withPlatformPin(), app.instance().change()); // A new upgrade is attempted, and production tests wait for redeployment. tester.controllerTester().upgradeSystem(version2); @@ -2234,7 +2240,7 @@ public class DeploymentTriggerTest { .majorVersion(7) .compileVersion(version1) .build()); - tester.deploymentTrigger().forceChange(app.instanceId(), app.instance().change().withPin()); + tester.deploymentTrigger().forceChange(app.instanceId(), app.instance().change().withPlatformPin()); app.deploy(); assertEquals(version1, tester.jobs().last(app.instanceId(), productionUsEast3).get().versions().targetPlatform()); assertEquals(version1, app.application().revisions().get(tester.jobs().last(app.instanceId(), productionUsEast3).get().versions().targetRevision()).compileVersion().get()); @@ -2251,7 +2257,7 @@ public class DeploymentTriggerTest { // The new app enters a platform block window, and is pinned to the old platform; // the new submission overrides both those settings, as the new revision should roll out regardless. tester.atMondayMorning(); - tester.deploymentTrigger().forceChange(newApp.instanceId(), Change.empty().withPin()); + tester.deploymentTrigger().forceChange(newApp.instanceId(), Change.empty().withPlatformPin()); newApp.submit(new ApplicationPackageBuilder().compileVersion(version2) .systemTest() .blockChange(false, true, "mon", "0-23", "UTC") @@ -2280,11 +2286,11 @@ public class DeploymentTriggerTest { tester.upgrader().run(); assertEquals(Change.of(newRevision).with(version1), newApp.instance().change()); - tester.deploymentTrigger().forceChange(newApp.instanceId(), newApp.instance().change().withPin()); + tester.deploymentTrigger().forceChange(newApp.instanceId(), newApp.instance().change().withPlatformPin()); tester.outstandingChangeDeployer().run(); - assertEquals(Change.of(newRevision).with(version1).withPin(), newApp.instance().change()); + assertEquals(Change.of(newRevision).with(version1).withPlatformPin(), newApp.instance().change()); tester.upgrader().run(); - assertEquals(Change.of(newRevision).with(version1).withPin(), newApp.instance().change()); + assertEquals(Change.of(newRevision).with(version1).withPlatformPin(), newApp.instance().change()); newApp.deploy(); assertEquals(version1, tester.jobs().last(newApp.instanceId(), productionUsEast3).get().versions().targetPlatform()); @@ -2381,7 +2387,7 @@ public class DeploymentTriggerTest { .build())) .getMessage()); - tester.deploymentTrigger().forceChange(app.instanceId(), Change.of(oldVersion).with(app.application().revisions().last().get().id()).withPin()); + tester.deploymentTrigger().forceChange(app.instanceId(), Change.of(oldVersion).with(app.application().revisions().last().get().id()).withPlatformPin()); app.deploy(); assertEquals(oldVersion, app.deployment(ZoneId.from("prod", "us-east-3")).version()); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java index 11110d6edaa..96c1d7c545d 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java @@ -5,7 +5,6 @@ import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.test.ManualClock; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RevisionId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; import com.yahoo.vespa.hosted.controller.application.Change; @@ -27,7 +26,6 @@ import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Optional; -import java.util.OptionalInt; import java.util.Set; import java.util.stream.Collectors; @@ -856,10 +854,10 @@ public class UpgraderTest { // Create an application with pinned platform version. var context = tester.newDeploymentContext().submit().deploy(); - tester.deploymentTrigger().forceChange(context.instanceId(), Change.empty().withPin()); + tester.deploymentTrigger().forceChange(context.instanceId(), Change.empty().withPlatformPin()); assertFalse(context.instance().change().hasTargets()); - assertTrue(context.instance().change().isPinned()); + assertTrue(context.instance().change().isPlatformPinned()); assertEquals(3, context.instance().deployments().size()); // Application does not upgrade. @@ -867,21 +865,21 @@ public class UpgraderTest { tester.controllerTester().upgradeSystem(version1); tester.upgrader().maintain(); assertFalse(context.instance().change().hasTargets()); - assertTrue(context.instance().change().isPinned()); + assertTrue(context.instance().change().isPlatformPinned()); // New application package is deployed. context.submit().deploy(); assertFalse(context.instance().change().hasTargets()); - assertTrue(context.instance().change().isPinned()); + assertTrue(context.instance().change().isPlatformPinned()); // Application upgrades to new version when pin is removed. tester.deploymentTrigger().cancelChange(context.instanceId(), PIN); tester.upgrader().maintain(); assertTrue(context.instance().change().hasTargets()); - assertFalse(context.instance().change().isPinned()); + assertFalse(context.instance().change().isPlatformPinned()); // Application is pinned to new version, and upgrade is therefore not cancelled, even though confidence is broken. - tester.deploymentTrigger().forceChange(context.instanceId(), Change.empty().withPin()); + tester.deploymentTrigger().forceChange(context.instanceId(), Change.empty().withPlatformPin()); tester.upgrader().maintain(); tester.triggerJobs(); assertEquals(version1, context.instance().change().platform().get()); @@ -890,7 +888,7 @@ public class UpgraderTest { context.runJob(systemTest).runJob(stagingTest).runJob(productionUsCentral1) .timeOutUpgrade(productionUsWest1); tester.deploymentTrigger().cancelChange(context.instanceId(), ALL); - tester.deploymentTrigger().forceChange(context.instanceId(), Change.of(version0).withPin()); + tester.deploymentTrigger().forceChange(context.instanceId(), Change.of(version0).withPlatformPin()); assertEquals(version0, context.instance().change().platform().get()); // Application downgrades to pinned version. @@ -913,7 +911,7 @@ public class UpgraderTest { // Keep app 1 on current version tester.controller().applications().lockApplicationIfPresent(app1.application().id(), app -> tester.controller().applications().store(app.with(app1.instance().name(), - instance -> instance.withChange(instance.change().withPin())))); + instance -> instance.withChange(instance.change().withPlatformPin())))); // New version is released Version version1 = Version.fromString("6.2"); @@ -935,7 +933,7 @@ public class UpgraderTest { // App 1 is unpinned and upgrades to latest 6 tester.controller().applications().lockApplicationIfPresent(app1.application().id(), app -> tester.controller().applications().store(app.with(app1.instance().name(), - instance -> instance.withChange(instance.change().withoutPin())))); + instance -> instance.withChange(instance.change().withoutPlatformPin())))); tester.upgrader().maintain(); assertEquals(version1, app1.instance().change().platform().orElseThrow(), diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java index 589fc25700f..b71d3cf838b 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java @@ -101,16 +101,17 @@ public class ApplicationSerializerTest { Optional.empty(), Optional.of("best commit"), Optional.of("hash1"), + Optional.of(Instant.ofEpochMilli(777)), true, false, Optional.of("~(˘▾˘)~"), 3); assertEquals("https://github/org/repo/tree/commit1", applicationVersion1.sourceUrl().get()); - ApplicationVersion applicationVersion2 = ApplicationVersion.from(RevisionId.forDevelopment(31, new JobId(id1, DeploymentContext.productionUsEast3)), - new SourceRevision("repo1", "branch1", "commit1"), "a@b", - Version.fromString("6.3.1"), - Instant.ofEpochMilli(496)); + RevisionId id = RevisionId.forDevelopment(31, new JobId(id1, DeploymentContext.productionUsEast3)); + SourceRevision source = new SourceRevision("repo1", "branch1", "commit1"); + Version compileVersion = Version.fromString("6.3.1"); + ApplicationVersion applicationVersion2 = new ApplicationVersion(id, Optional.of(source), Optional.of("a@b"), Optional.of(compileVersion), Optional.empty(), Optional.of(Instant.ofEpochMilli(496)), Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty(), true, false, Optional.empty(), 0); Instant activityAt = Instant.parse("2018-06-01T10:15:30.00Z"); deployments.add(new Deployment(zone1, CloudAccount.empty, applicationVersion1.id(), Version.fromString("1.2.3"), Instant.ofEpochMilli(3), DeploymentMetrics.none, DeploymentActivity.none, QuotaUsage.none, OptionalDouble.empty())); @@ -143,7 +144,7 @@ public class ApplicationSerializerTest { Map.of(), List.of(), RotationStatus.EMPTY, - Change.of(Version.fromString("6.7")).withPin())); + Change.of(Version.fromString("6.7")).withPlatformPin().withRevisionPin())); Application original = new Application(TenantAndApplicationId.from(id1), Instant.now().truncatedTo(ChronoUnit.MILLIS), @@ -174,6 +175,7 @@ public class ApplicationSerializerTest { assertEquals(original.revisions().last().get().sourceUrl(), serialized.revisions().last().get().sourceUrl()); assertEquals(original.revisions().last().get().commit(), serialized.revisions().last().get().commit()); assertEquals(original.revisions().last().get().bundleHash(), serialized.revisions().last().get().bundleHash()); + assertEquals(original.revisions().last().get().obsoleteAt(), serialized.revisions().last().get().obsoleteAt()); assertEquals(original.revisions().last().get().hasPackage(), serialized.revisions().last().get().hasPackage()); assertEquals(original.revisions().last().get().shouldSkip(), serialized.revisions().last().get().shouldSkip()); assertEquals(original.revisions().last().get().description(), serialized.revisions().last().get().description()); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java index 9a34989aeff..76bcbe078ff 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java @@ -541,24 +541,22 @@ public class ApplicationApiTest extends ControllerContainerTest { "{\"message\":\"No deployment in progress for tenant1.application1.instance1 at this time\"}"); // POST pinning to a given version to an application - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/deploying/pin", POST) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/deploying/platform-pin", POST) .userIdentity(USER_ID) .data("6.1.0"), "{\"message\":\"Triggered pin to 6.1 for tenant1.application1.instance1\"}"); assertTrue(tester.controller().auditLogger().readLog().entries().stream() - .anyMatch(entry -> entry.resource().equals("/application/v4/tenant/tenant1/application/application1/instance/instance1/deploying/pin?")), + .anyMatch(entry -> entry.resource().equals("/application/v4/tenant/tenant1/application/application1/instance/instance1/deploying/platform-pin?")), "Action is logged to audit log"); tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/deploying", GET) - .userIdentity(USER_ID), "{\"platform\":\"6.1\",\"pinned\":true}"); - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/deploying/pin", GET) - .userIdentity(USER_ID), "{\"platform\":\"6.1\",\"pinned\":true}"); + .userIdentity(USER_ID), "{\"platform\":\"6.1\",\"pinned\":true,\"platform-pinned\":true,\"application-pinned\":false}"); // DELETE only the pin to a given version - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/deploying/pin", DELETE) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/deploying/platform-pin", DELETE) .userIdentity(USER_ID), "{\"message\":\"Changed deployment from 'pin to 6.1' to 'upgrade to 6.1' for tenant1.application1.instance1\"}"); tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/deploying", GET) - .userIdentity(USER_ID), "{\"platform\":\"6.1\",\"pinned\":false}"); + .userIdentity(USER_ID), "{\"platform\":\"6.1\",\"pinned\":false,\"platform-pinned\":false,\"application-pinned\":false}"); // POST pinning again tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/deploying/pin", POST) @@ -566,14 +564,14 @@ public class ApplicationApiTest extends ControllerContainerTest { .data("6.1"), "{\"message\":\"Triggered pin to 6.1 for tenant1.application1.instance1\"}"); tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/deploying", GET) - .userIdentity(USER_ID), "{\"platform\":\"6.1\",\"pinned\":true}"); + .userIdentity(USER_ID), "{\"platform\":\"6.1\",\"pinned\":true,\"platform-pinned\":true,\"application-pinned\":false}"); // DELETE only the version, but leave the pin tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/deploying/platform", DELETE) .userIdentity(USER_ID), "{\"message\":\"Changed deployment from 'pin to 6.1' to 'pin to current platform' for tenant1.application1.instance1\"}"); tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/deploying", GET) - .userIdentity(USER_ID), "{\"pinned\":true}"); + .userIdentity(USER_ID), "{\"pinned\":true,\"platform-pinned\":true,\"application-pinned\":false}"); // DELETE also the pin to a given version tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/deploying/pin", DELETE) @@ -582,6 +580,32 @@ public class ApplicationApiTest extends ControllerContainerTest { tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/deploying", GET) .userIdentity(USER_ID), "{}"); + // POST pinning to a given revision to an application + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/deploying/application-pin", POST) + .userIdentity(USER_ID) + .data(""), + "{\"message\":\"Triggered pin to build 1 for tenant1.application1.instance1\"}"); + assertTrue(tester.controller().auditLogger().readLog().entries().stream() + .anyMatch(entry -> entry.resource().equals("/application/v4/tenant/tenant1/application/application1/instance/instance1/deploying/application-pin?")), + "Action is logged to audit log"); + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/deploying", GET) + .userIdentity(USER_ID), "{\"application\":\"build 1\",\"pinned\":false,\"platform-pinned\":false,\"application-pinned\":true}"); + + // DELETE only the pin to a given revision + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/deploying/application-pin", DELETE) + .userIdentity(USER_ID), + "{\"message\":\"Changed deployment from 'pin to build 1' to 'revision change to build 1' for tenant1.application1.instance1\"}"); + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/deploying", GET) + .userIdentity(USER_ID), "{\"application\":\"build 1\",\"pinned\":false,\"platform-pinned\":false,\"application-pinned\":false}"); + + // DELETE deploying to a given revision + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/deploying/application", DELETE) + .userIdentity(USER_ID), + "{\"message\":\"Changed deployment from 'revision change to build 1' to 'no change' for tenant1.application1.instance1\"}"); + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/deploying", GET) + .userIdentity(USER_ID), "{}"); + + // POST a pause to a production job tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/job/production-us-west-1/pause", POST) .userIdentity(USER_ID), diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview.json index ec6ccf3ecf2..0b7c64c72a5 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview.json @@ -48,6 +48,14 @@ "sourceUrl": "repository1/tree/commit1", "commit": "commit1" } + }, + { + "application": { + "build": 1, + "compileVersion": "6.1.0", + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" + } } ], "blockers": [ ] @@ -594,6 +602,14 @@ "sourceUrl": "repository1/tree/commit1", "commit": "commit1" } + }, + { + "application": { + "build": 1, + "compileVersion": "6.1.0", + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" + } } ], "blockers": [ ] @@ -709,6 +725,15 @@ "description": "my best commit yet", "risk": 9001, "deployable": false + }, + { + "build": 1, + "compileVersion": "6.1.0", + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1", + "description": "my best commit yet", + "risk": 9001, + "deployable": true } ] } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/root.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/root.json index a1f386d51a7..ac43fbf2a80 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/root.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/root.json @@ -37,6 +37,8 @@ "instance": "default", "upgrading": false, "pinned": false, + "platformPinned": false, + "revisionPinned": false, "upgradePolicy": "default", "compileVersion": "6.1.0", "jobs": [ @@ -78,6 +80,8 @@ "instance": "i2", "upgrading": false, "pinned": false, + "platformPinned": false, + "revisionPinned": false, "upgradePolicy": "default", "compileVersion": "6.1.0", "jobs": [ @@ -179,6 +183,8 @@ "instance": "default", "upgrading": true, "pinned": false, + "platformPinned": false, + "revisionPinned": false, "upgradePolicy": "default", "compileVersion": "6.1.0", "jobs": [ @@ -249,6 +255,8 @@ "instance": "i1", "upgrading": false, "pinned": false, + "platformPinned": false, + "revisionPinned": false, "upgradePolicy": "default", "compileVersion": "6.1.0", "jobs": [ @@ -309,6 +317,8 @@ "instance": "i2", "upgrading": true, "pinned": false, + "platformPinned": false, + "revisionPinned": false, "upgradePolicy": "default", "compileVersion": "6.1.0", "jobs": [ diff --git a/document/src/main/java/com/yahoo/document/serialization/DocumentUpdateFlags.java b/document/src/main/java/com/yahoo/document/serialization/DocumentUpdateFlags.java index e3510676148..11ded80ed2a 100644 --- a/document/src/main/java/com/yahoo/document/serialization/DocumentUpdateFlags.java +++ b/document/src/main/java/com/yahoo/document/serialization/DocumentUpdateFlags.java @@ -23,7 +23,7 @@ public class DocumentUpdateFlags { } public void setCreateIfNonExistent(boolean value) { flags &= ~1; // clear flag - flags |= value ? 1 : 0; // set flag + flags |= value ? (byte)1 : (byte)0; // set flag } public int injectInto(int value) { return extractValue(value) | (flags << 28); diff --git a/jdisc_core/src/test/java/com/yahoo/jdisc/core/ExportPackagesIT.java b/jdisc_core/src/test/java/com/yahoo/jdisc/core/ExportPackagesIT.java index e9aba0893f9..7ec3406be1f 100644 --- a/jdisc_core/src/test/java/com/yahoo/jdisc/core/ExportPackagesIT.java +++ b/jdisc_core/src/test/java/com/yahoo/jdisc/core/ExportPackagesIT.java @@ -62,8 +62,8 @@ public class ExportPackagesIT { String expectedValue = expectedProperties.getProperty(ExportPackages.EXPORT_PACKAGES); assertNotNull(expectedValue, "Missing exportPackages property in file."); - Set<String> actualPackages = getPackages(actualValue); - Set<String> expectedPackages = getPackages(expectedValue); + Set<String> actualPackages = removeNewPackageOnJava20(removeJavaVersion(getPackages(actualValue))); + Set<String> expectedPackages = removeNewPackageOnJava20(removeJavaVersion(getPackages(expectedValue))); if (!actualPackages.equals(expectedPackages)) { StringBuilder message = getDiff(actualPackages, expectedPackages); message.append("\n\nIf this test fails due to an intentional change in exported packages, run the following command:\n") @@ -73,6 +73,14 @@ public class ExportPackagesIT { } } + private static Set<String> removeJavaVersion(Set<String> packages) { + return packages.stream().map(p -> p.replaceAll(".JavaSE_\\d+", "")).collect(Collectors.toSet()); + } + + private static Set<String> removeNewPackageOnJava20(Set<String> packages) { + return packages.stream().filter(p -> ! p.contains("java.lang.foreign")).collect(Collectors.toSet()); + } + private static StringBuilder getDiff(Set<String> actual, Set<String> expected) { StringBuilder sb = new StringBuilder(); Set<String> onlyInActual = onlyInSet1(actual, expected); diff --git a/model-integration/src/main/java/ai/vespa/embedding/BertBaseEmbedder.java b/model-integration/src/main/java/ai/vespa/embedding/BertBaseEmbedder.java index b40e2b5be72..bf56d233f89 100644 --- a/model-integration/src/main/java/ai/vespa/embedding/BertBaseEmbedder.java +++ b/model-integration/src/main/java/ai/vespa/embedding/BertBaseEmbedder.java @@ -32,10 +32,9 @@ import java.util.Map; */ public class BertBaseEmbedder extends AbstractComponent implements Embedder { - private final static int TOKEN_CLS = 101; // [CLS] - private final static int TOKEN_SEP = 102; // [SEP] - private final int maxTokens; + private final int startSequenceToken; + private final int endSequenceToken; private final String inputIdsName; private final String attentionMaskName; private final String tokenTypeIdsName; @@ -48,6 +47,8 @@ public class BertBaseEmbedder extends AbstractComponent implements Embedder { @Inject public BertBaseEmbedder(OnnxRuntime onnx, BertBaseEmbedderConfig config) { maxTokens = config.transformerMaxTokens(); + startSequenceToken = config.transformerStartSequenceToken(); + endSequenceToken = config.transformerEndSequenceToken(); inputIdsName = config.transformerInputIds(); attentionMaskName = config.transformerAttentionMask(); tokenTypeIdsName = config.transformerTokenTypeIds(); @@ -98,7 +99,7 @@ public class BertBaseEmbedder extends AbstractComponent implements Embedder { if (!type.dimensions().get(0).isIndexed()) { throw new IllegalArgumentException("Error in embedding to type '" + type + "': dimension should be indexed."); } - List<Integer> tokens = embedWithSeperatorTokens(text, context, maxTokens); + List<Integer> tokens = embedWithSeparatorTokens(text, context, maxTokens); return embedTokens(tokens, type); } @@ -109,6 +110,7 @@ public class BertBaseEmbedder extends AbstractComponent implements Embedder { Tensor attentionMask = createAttentionMask(inputSequence); Tensor tokenTypeIds = createTokenTypeIds(inputSequence); + Map<String, Tensor> inputs; if (!"".equals(tokenTypeIdsName)) { inputs = Map.of(inputIdsName, inputSequence.expand("d0"), @@ -138,14 +140,14 @@ public class BertBaseEmbedder extends AbstractComponent implements Embedder { return builder.build(); } - private List<Integer> embedWithSeperatorTokens(String text, Context context, int maxLength) { + private List<Integer> embedWithSeparatorTokens(String text, Context context, int maxLength) { List<Integer> tokens = new ArrayList<>(); - tokens.add(TOKEN_CLS); + tokens.add(startSequenceToken); tokens.addAll(embed(text, context)); - tokens.add(TOKEN_SEP); + tokens.add(endSequenceToken); if (tokens.size() > maxLength) { tokens = tokens.subList(0, maxLength-1); - tokens.add(TOKEN_SEP); + tokens.add(endSequenceToken); } return tokens; } diff --git a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/OrderedTensorType.java b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/OrderedTensorType.java index ceb9a27924d..eee60d56c55 100644 --- a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/OrderedTensorType.java +++ b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/OrderedTensorType.java @@ -96,14 +96,14 @@ public class OrderedTensorType { * so that they are correctly laid out in memory for Vespa. * Used when importing tensors. */ - public int toDirectIndex(int index) { + public long toDirectIndex(int index) { if (dimensions.size() == 0) { return 0; } if (dimensionMap == null) { throw new IllegalArgumentException("Dimension map is not available"); } - int directIndex = 0; + long directIndex = 0; long rest = index; for (int i = 0; i < dimensions.size(); ++i) { long address = rest / innerSizesOriginal[i]; diff --git a/model-integration/src/main/resources/configdefinitions/embedding.bert-base-embedder.def b/model-integration/src/main/resources/configdefinitions/embedding.bert-base-embedder.def index 14d953eeef9..ef42d81e1fe 100644 --- a/model-integration/src/main/resources/configdefinitions/embedding.bert-base-embedder.def +++ b/model-integration/src/main/resources/configdefinitions/embedding.bert-base-embedder.def @@ -17,6 +17,10 @@ transformerInputIds string default=input_ids transformerAttentionMask string default=attention_mask transformerTokenTypeIds string default=token_type_ids +# special token ids +transformerStartSequenceToken int default=101 +transformerEndSequenceToken int default=102 + # Output name transformerOutput string default=output_0 diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityChecker.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityChecker.java index c8b736cb25b..f3ea326a3c0 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityChecker.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityChecker.java @@ -130,7 +130,7 @@ public class CapacityChecker { Set<String> ipPool = host.ipConfig().pool().asSet(); for (var child : nodeChildren.get(host)) { hostResources = hostResources.subtract(child.resources().justNumbers()); - occupiedIps += child.ipConfig().primary().stream().filter(ipPool::contains).count(); + occupiedIps += (int)child.ipConfig().primary().stream().filter(ipPool::contains).count(); } availableResources.put(host, new AllocationResources(hostResources, host.ipConfig().pool().asSet().size() - occupiedIps)); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java index 890d190c24e..b3198a72d1b 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java @@ -193,8 +193,8 @@ public class LoadBalancerProvisioner { Optional<LoadBalancer> loadBalancer = db.readLoadBalancer(id); LoadBalancer newLoadBalancer; LoadBalancer.State fromState = loadBalancer.map(LoadBalancer::state).orElse(null); - boolean recreateLoadBalancer = loadBalancer.isPresent() && (!inAccount(cloudAccount, loadBalancer.get()) - || !hasCorrectVisibility(loadBalancer.get(), zoneEndpoint)); + boolean recreateLoadBalancer = loadBalancer.isPresent() && ( ! inAccount(cloudAccount, loadBalancer.get()) + || ! hasCorrectVisibility(loadBalancer.get(), zoneEndpoint)); if (recreateLoadBalancer) { // We have a load balancer, but with the wrong account or visibility. // Load balancer must be removed before we can provision a new one with the wanted visibility diff --git a/predicate-search/src/main/java/com/yahoo/search/predicate/benchmarks/ResultMetrics.java b/predicate-search/src/main/java/com/yahoo/search/predicate/benchmarks/ResultMetrics.java index 11103a2a66a..ef65d9e2efa 100644 --- a/predicate-search/src/main/java/com/yahoo/search/predicate/benchmarks/ResultMetrics.java +++ b/predicate-search/src/main/java/com/yahoo/search/predicate/benchmarks/ResultMetrics.java @@ -62,8 +62,8 @@ public class ResultMetrics { } private double percentile(double percentile) { - int targetCount = (int) Math.round(totalQueries * percentile); - int currentCount = 0; + long targetCount = Math.round(totalQueries * percentile); + long currentCount = 0; int index = 0; while (currentCount < targetCount && index < SLOTS) { currentCount += latencyHistogram[index]; diff --git a/predicate-search/src/main/java/com/yahoo/search/predicate/index/CachedPostingListCounter.java b/predicate-search/src/main/java/com/yahoo/search/predicate/index/CachedPostingListCounter.java index e6db1dec7c3..eb8b0b9927b 100644 --- a/predicate-search/src/main/java/com/yahoo/search/predicate/index/CachedPostingListCounter.java +++ b/predicate-search/src/main/java/com/yahoo/search/predicate/index/CachedPostingListCounter.java @@ -75,7 +75,7 @@ public class CachedPostingListCounter { private void countUsingBitVector(byte[] nPostingListsForDocument, int postingListBitmap) { for (int docId = 0; docId < nDocuments; docId++) { - nPostingListsForDocument[docId] += Integer.bitCount(bitVector[docId] & postingListBitmap); + nPostingListsForDocument[docId] += (byte)Integer.bitCount(bitVector[docId] & postingListBitmap); } } @@ -88,8 +88,7 @@ public class CachedPostingListCounter { } public CachedPostingListCounter rebuildCache() { - MinMaxPriorityQueue<Entry> mostExpensive = MinMaxPriorityQueue - .maximumSize(32).expectedSize(32).create(); + MinMaxPriorityQueue<Entry> mostExpensive = MinMaxPriorityQueue.maximumSize(32).expectedSize(32).create(); synchronized (this) { for (ObjectLongPair<int[]> p : frequency.keyValuesView()) { mostExpensive.add(new Entry(p.getOne(), p.getTwo())); diff --git a/searchlib/src/main/java/com/yahoo/searchlib/expression/Int16ResultNode.java b/searchlib/src/main/java/com/yahoo/searchlib/expression/Int16ResultNode.java index ae7d0a67b2f..b0f98685578 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/expression/Int16ResultNode.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/expression/Int16ResultNode.java @@ -80,7 +80,7 @@ public class Int16ResultNode extends NumericResultNode { @Override public void add(ResultNode rhs) { - value += rhs.getInteger(); + value += (short)rhs.getInteger(); } @Override @@ -90,7 +90,7 @@ public class Int16ResultNode extends NumericResultNode { @Override public void multiply(ResultNode rhs) { - value *= rhs.getInteger(); + value *= (short)rhs.getInteger(); } @Override @@ -101,7 +101,7 @@ public class Int16ResultNode extends NumericResultNode { @Override public void modulo(ResultNode rhs) { - value %= rhs.getInteger(); + value %= (short)rhs.getInteger(); } @Override diff --git a/searchlib/src/main/java/com/yahoo/searchlib/expression/Int32ResultNode.java b/searchlib/src/main/java/com/yahoo/searchlib/expression/Int32ResultNode.java index da31cbc236a..711b8f1bd3f 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/expression/Int32ResultNode.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/expression/Int32ResultNode.java @@ -80,7 +80,7 @@ public class Int32ResultNode extends NumericResultNode { @Override public void add(ResultNode rhs) { - value += rhs.getInteger(); + value += (int)rhs.getInteger(); } @Override @@ -90,7 +90,7 @@ public class Int32ResultNode extends NumericResultNode { @Override public void multiply(ResultNode rhs) { - value *= rhs.getInteger(); + value *= (int)rhs.getInteger(); } @Override @@ -101,7 +101,7 @@ public class Int32ResultNode extends NumericResultNode { @Override public void modulo(ResultNode rhs) { - value %= rhs.getInteger(); + value %= (int)rhs.getInteger(); } @Override @@ -122,7 +122,7 @@ public class Int32ResultNode extends NumericResultNode { @Override public Object getNumber() { - return Integer.valueOf(value); + return value; } @Override diff --git a/searchlib/src/main/java/com/yahoo/searchlib/expression/Int8ResultNode.java b/searchlib/src/main/java/com/yahoo/searchlib/expression/Int8ResultNode.java index ae53cf45a6f..d6706ce1dfe 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/expression/Int8ResultNode.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/expression/Int8ResultNode.java @@ -78,7 +78,7 @@ public class Int8ResultNode extends NumericResultNode { @Override public void add(ResultNode rhs) { - value += rhs.getInteger(); + value += (byte)rhs.getInteger(); } @Override @@ -88,7 +88,7 @@ public class Int8ResultNode extends NumericResultNode { @Override public void multiply(ResultNode rhs) { - value *= rhs.getInteger(); + value *= (byte)rhs.getInteger(); } @Override @@ -99,7 +99,7 @@ public class Int8ResultNode extends NumericResultNode { @Override public void modulo(ResultNode rhs) { - value %= rhs.getInteger(); + value %= (byte)rhs.getInteger(); } @Override diff --git a/searchlib/src/main/java/com/yahoo/searchlib/ranking/features/fieldmatch/FieldMatchMetrics.java b/searchlib/src/main/java/com/yahoo/searchlib/ranking/features/fieldmatch/FieldMatchMetrics.java index 2b5efdb1ffe..5b6a53a7019 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/ranking/features/fieldmatch/FieldMatchMetrics.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/ranking/features/fieldmatch/FieldMatchMetrics.java @@ -16,7 +16,7 @@ import static java.lang.Math.*; public final class FieldMatchMetrics implements Cloneable { /** The calculator creating this - given on initialization */ - private FieldMatchMetricsComputer source; + private final FieldMatchMetricsComputer source; /** The trace accumulated during execution - empty if no tracing */ private final Trace trace = new Trace(); @@ -75,7 +75,7 @@ public final class FieldMatchMetrics implements Cloneable { currentSequence=0; segmentStarts.clear(); - queryLength=source.getQuery().getTerms().length; + queryLength = source.getQuery().getTerms().length; } /** Are these metrics representing a complete match */ @@ -93,7 +93,7 @@ public final class FieldMatchMetrics implements Cloneable { */ public float get(String name) { try { - Method getter=getClass().getMethod("get" + name.substring(0,1).toUpperCase() + name.substring(1)); + Method getter = getClass().getMethod("get" + name.substring(0, 1).toUpperCase() + name.substring(1)); return ((Number)getter.invoke(this)).floatValue(); } catch (NoSuchMethodException e) { @@ -140,7 +140,7 @@ public final class FieldMatchMetrics implements Cloneable { * segment or out of order */ public float getAbsoluteProximity() { - if (pairs <1) return 0.1f; + if (pairs < 1) return 0.1f; return proximity/pairs; } @@ -151,7 +151,7 @@ public final class FieldMatchMetrics implements Cloneable { * following each other in sequence, and close to 0 if they are far from each other or out of order */ public float getUnweightedProximity() { - if (pairs <1) return 1f; + if (pairs < 1) return 1f; return unweightedProximity/pairs; } @@ -271,33 +271,33 @@ public final class FieldMatchMetrics implements Cloneable { * <code>queryCompleteness * ( 1 - fieldCompletenessImportance) + fieldCompletenessImportance * fieldCompleteness</code> */ public float getCompleteness() { - float fieldCompletenessImportance=source.getParameters().getFieldCompletenessImportance(); + float fieldCompletenessImportance = source.getParameters().getFieldCompletenessImportance(); return getQueryCompleteness() * ( 1 - fieldCompletenessImportance) + fieldCompletenessImportance*getFieldCompleteness(); } /** Returns how well the order of the terms agreed in segments: <code>1-outOfOrder/pairs</code> */ public float getOrderness() { - if (pairs ==0) return 1f; + if (pairs == 0) return 1f; return 1-(float)outOfOrder/pairs; } /** Returns the degree to which different terms are related (occurring in the same segment): <code>1-segments/(matches-1)</code> */ public float getRelatedness() { - if (matches==0) return 0; - if (matches==1) return 1; - return 1-(float)(segments-1)/(matches-1); + if (matches == 0) return 0; + if (matches == 1) return 1; + return 1 - (float)(segments - 1) / (matches - 1); } /** Returns <code>longestSequence/matches</code> */ public float getLongestSequenceRatio() { - if (matches==0) return 0; - return (float)longestSequence/matches; + if (matches == 0) return 0; + return (float)longestSequence / matches; } /** Returns the closeness of the segments in the field: <code>1-segmentDistance/fieldLength</code> */ public float getSegmentProximity() { - if (matches==0) return 0; - return 1-segmentDistance/source.getField().terms().size(); + if (matches == 0) return 0; + return 1 - segmentDistance / source.getField().terms().size(); } /** @@ -306,14 +306,14 @@ public final class FieldMatchMetrics implements Cloneable { * This is absoluteProximity/average connectedness. */ public float getProximity() { - float totalConnectedness=0; - for (int i=1; i<queryLength; i++) { - totalConnectedness+=Math.max(0.1,source.getQuery().getTerms()[i].getConnectedness()); + float totalConnectedness = 0; + for (int i = 1; i < queryLength; i++) { + totalConnectedness += (float)Math.max(0.1, source.getQuery().getTerms()[i].getConnectedness()); } - float averageConnectedness=0.1f; - if (queryLength>1) - averageConnectedness=totalConnectedness/(queryLength-1); - return getAbsoluteProximity()/averageConnectedness; + float averageConnectedness = 0.1f; + if (queryLength > 1) + averageConnectedness = totalConnectedness / (queryLength - 1); + return getAbsoluteProximity() / averageConnectedness; } /** @@ -378,7 +378,7 @@ public final class FieldMatchMetrics implements Cloneable { * not only when the metrics are complete, because this metric is used to choose segments during calculation.</p> */ float getSegmentationScore() { - if (segments==0) return 0; + if (segments == 0) return 0; return getAbsoluteProximity() * getExactness() / (segments * segments); } @@ -389,7 +389,7 @@ public final class FieldMatchMetrics implements Cloneable { /** Called once for every match */ void onMatch(int i, int j) { - if (matches>=source.getField().terms().size()) return; + if (matches >= source.getField().terms().size()) return; matches++; weight += (float)source.getQuery().getTerms()[i].getWeight() / source.getQuery().getTotalTermWeight(); significance += source.getQuery().getTerms()[i].getSignificance() / source.getQuery().getTotalSignificance(); @@ -418,42 +418,42 @@ public final class FieldMatchMetrics implements Cloneable { } /** Called once when this value is calculated, before onComplete */ - void setOccurrence(float occurrence) { this.occurrence=occurrence; } + void setOccurrence(float occurrence) { this.occurrence = occurrence; } /** Called once when this value is calculated, before onComplete */ - void setWeightedOccurrence(float weightedOccurrence) { this.weightedOccurrence=weightedOccurrence; } + void setWeightedOccurrence(float weightedOccurrence) { this.weightedOccurrence = weightedOccurrence; } /** Called once when this value is calculated, before onComplete */ - void setAbsoluteOccurrence(float absoluteOccurrence) { this.absoluteOccurrence=absoluteOccurrence; } + void setAbsoluteOccurrence(float absoluteOccurrence) { this.absoluteOccurrence = absoluteOccurrence; } /** Called once when this value is calculated, before onComplete */ - void setWeightedAbsoluteOccurrence(float weightedAbsoluteOccurrence) { this.weightedAbsoluteOccurrence=weightedAbsoluteOccurrence; } + void setWeightedAbsoluteOccurrence(float weightedAbsoluteOccurrence) { this.weightedAbsoluteOccurrence = weightedAbsoluteOccurrence; } /** Called once when this value is calculated, before onComplete */ - void setSignificantOccurrence(float significantOccurrence) { this.significantOccurrence =significantOccurrence; } + void setSignificantOccurrence(float significantOccurrence) { this.significantOccurrence = significantOccurrence; } /** Called once when matching is complete */ void onComplete() { // segment distance - calculated from sorted segment starts - if (segmentStarts.size()<=1) { - segmentDistance=0; + if (segmentStarts.size() <= 1) { + segmentDistance = 0; } else { Collections.sort(segmentStarts); - for (int i=1; i<segmentStarts.size(); i++) { - segmentDistance+=segmentStarts.get(i)-segmentStarts.get(i-1)+1; + for (int i = 1; i < segmentStarts.size(); i++) { + segmentDistance += segmentStarts.get(i) - segmentStarts.get(i - 1) + 1; } } - if (head==-1) head=0; - if (tail==-1) tail=0; + if (head == -1) head = 0; + if (tail == -1) tail = 0; } // Events on pairs ---------- /** Called when <i>any</i> pair is encountered */ void onPair(int i, int j, int previousJ) { - int distance = j-previousJ-1; + int distance = j - previousJ - 1; if (distance < 0) distance++; // Discontinuity where the two terms are in the same position if (abs(distance) > source.getParameters().getProximityLimit()) return; // Contribution=0 @@ -463,7 +463,7 @@ public final class FieldMatchMetrics implements Cloneable { unweightedProximity += pairProximity; float connectedness = source.getQuery().getTerms()[i].getConnectedness(); - proximity += pow(pairProximity, connectedness/0.1) * max(0.1, connectedness); + proximity += (float)pow(pairProximity, connectedness / 0.1) * (float)max(0.1, connectedness); pairs++; } @@ -498,8 +498,8 @@ public final class FieldMatchMetrics implements Cloneable { @Override public FieldMatchMetrics clone() { try { - FieldMatchMetrics clone=(FieldMatchMetrics)super.clone(); - clone.segmentStarts=new ArrayList<>(segmentStarts); + FieldMatchMetrics clone = (FieldMatchMetrics)super.clone(); + clone.segmentStarts = new ArrayList<>(segmentStarts); return clone; } catch (CloneNotSupportedException e) { @@ -514,19 +514,19 @@ public final class FieldMatchMetrics implements Cloneable { public String toStringDump() { try { - StringBuilder b=new StringBuilder(); + StringBuilder b = new StringBuilder(); for (Method m : this.getClass().getDeclaredMethods()) { if ( ! m.getName().startsWith("get")) continue; - if (m.getReturnType()!=Integer.TYPE && m.getReturnType()!=Float.TYPE) continue; - if ( m.getParameterTypes().length!=0 ) continue; + if (m.getReturnType() != Integer.TYPE && m.getReturnType() != Float.TYPE) continue; + if ( m.getParameterTypes().length != 0 ) continue; - Object value=m.invoke(this,new Object[0]); - b.append(m.getName().substring(3,4).toLowerCase() + m.getName().substring(4) + ": " + value + "\n"); + Object value = m.invoke(this, new Object[0]); + b.append(m.getName().substring(3, 4).toLowerCase() + m.getName().substring(4) + ": " + value + "\n"); } return b.toString(); } catch (Exception e) { - throw new RuntimeException("Programming error",e); + throw new RuntimeException("Programming error", e); } } diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/gbdtoptimization/GBDTNode.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/gbdtoptimization/GBDTNode.java index 949e1f026f7..df721a4309e 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/gbdtoptimization/GBDTNode.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/gbdtoptimization/GBDTNode.java @@ -70,13 +70,13 @@ public final class GBDTNode extends ExpressionNode { int offset = (int)nextValue - MAX_LEAF_VALUE; boolean comparisonIsTrue = false; if (offset < MAX_VARIABLES) { - comparisonIsTrue = context.getDouble(offset)<values[pc++]; + comparisonIsTrue = context.getDouble(offset) < values[pc++]; } - else if (offset < MAX_VARIABLES*2) { - comparisonIsTrue = context.getDouble(offset-MAX_VARIABLES)==values[pc++]; + else if (offset < MAX_VARIABLES * 2) { + comparisonIsTrue = context.getDouble(offset - MAX_VARIABLES) == values[pc++]; } - else if (offset<MAX_VARIABLES*3) { - double testValue = context.getDouble(offset-MAX_VARIABLES*2); + else if (offset < MAX_VARIABLES * 3) { + double testValue = context.getDouble(offset - MAX_VARIABLES * 2); int setValuesLeft = (int)values[pc++]; while (setValuesLeft > 0) { // test each value in the set setValuesLeft--; @@ -88,13 +88,13 @@ public final class GBDTNode extends ExpressionNode { pc += setValuesLeft; // jump to after the set } else { // offset<MAX_VARIABLES*4 - comparisonIsTrue = ! (context.getDouble(offset-MAX_VARIABLES*3)>=values[pc++]); + comparisonIsTrue = ! (context.getDouble(offset - MAX_VARIABLES * 3) >= values[pc++]); } if (comparisonIsTrue) pc++; // true branch - skip the jump value else - pc += values[pc]; // false branch - jump + pc += (int)values[pc]; // false branch - jump } else { // a leaf return nextValue; diff --git a/searchlib/src/test/java/com/yahoo/searchlib/gbdt/GbdtConverterTestCase.java b/searchlib/src/test/java/com/yahoo/searchlib/gbdt/GbdtConverterTestCase.java index d846d322720..8a33f320bb0 100644 --- a/searchlib/src/test/java/com/yahoo/searchlib/gbdt/GbdtConverterTestCase.java +++ b/searchlib/src/test/java/com/yahoo/searchlib/gbdt/GbdtConverterTestCase.java @@ -3,14 +3,12 @@ package com.yahoo.searchlib.gbdt; import com.yahoo.searchlib.rankingexpression.RankingExpression; import com.yahoo.searchlib.rankingexpression.parser.ParseException; -import org.junit.After; -import org.junit.Before; import org.junit.Test; import java.io.ByteArrayOutputStream; import java.io.PrintStream; import java.io.UnsupportedEncodingException; -import java.security.Permission; +import java.nio.charset.StandardCharsets; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertEquals; @@ -21,36 +19,6 @@ import static org.junit.Assert.fail; */ public class GbdtConverterTestCase { - @Before - @SuppressWarnings("removal") - public void enableSecurityManager() { - System.setSecurityManager(new NoExitSecurityManager()); - } - - @After - @SuppressWarnings("removal") - public void disableSecurityManager() { - System.setSecurityManager(null); - } - - @Test - public void testOnlyOneArgumentIsAccepted() throws UnsupportedEncodingException { - assertError("Usage: GbdtConverter <filename>\n", new String[0]); - assertError("Usage: GbdtConverter <filename>\n", new String[] { "foo", "bar" }); - } - - @Test - public void testFileIsFound() throws UnsupportedEncodingException { - assertError("Could not find file 'not.found'.\n", new String[] { "not.found" }); - } - - @Test - public void testFileParsingExceptionIsCaught() throws UnsupportedEncodingException { - assertError("An error occurred while parsing the content of file 'src/test/files/gbdt_err.xml': " + - "Node 'Unknown' has no 'DecisionTree' children.\n", - new String[] { "src/test/files/gbdt_err.xml" }); - } - @Test public void testEmptyTreesAreIgnored() throws Exception { assertConvert("src/test/files/gbdt_empty_tree.xml", @@ -125,7 +93,7 @@ public class GbdtConverterTestCase { ByteArrayOutputStream out = new ByteArrayOutputStream(); System.setOut(new PrintStream(out)); GbdtConverter.main(new String[] { gbdtModelFile }); - String actualExpression = out.toString("UTF-8"); + String actualExpression = out.toString(StandardCharsets.UTF_8); assertEquals(expectedExpression, actualExpression); assertNotNull(new RankingExpression(actualExpression)); } @@ -138,26 +106,7 @@ public class GbdtConverterTestCase { fail(); } catch (ExitException e) { assertEquals(1, e.status); - assertEquals(expected, err.toString("UTF-8")); - } - } - - @SuppressWarnings("removal") - private static class NoExitSecurityManager extends SecurityManager { - - @Override - public void checkPermission(Permission perm) { - // allow anything - } - - @Override - public void checkPermission(Permission perm, Object context) { - // allow anything - } - - @Override - public void checkExit(int status) { - throw new ExitException(status); + assertEquals(expected, err.toString(StandardCharsets.UTF_8)); } } @@ -169,4 +118,5 @@ public class GbdtConverterTestCase { this.status = status; } } + } diff --git a/searchlib/src/tests/attribute/enumstore/enumstore_test.cpp b/searchlib/src/tests/attribute/enumstore/enumstore_test.cpp index e9260a55f57..2b01c266e80 100644 --- a/searchlib/src/tests/attribute/enumstore/enumstore_test.cpp +++ b/searchlib/src/tests/attribute/enumstore/enumstore_test.cpp @@ -390,7 +390,7 @@ TEST(EnumStoreTest, address_space_usage_is_reported) TEST(EnumStoreTest, provided_memory_allocator_is_used) { AllocStats stats; - NumericEnumStore ses(false, DictionaryConfig::Type::BTREE, std::make_unique<MemoryAllocatorObserver>(stats)); + NumericEnumStore ses(false, DictionaryConfig::Type::BTREE, std::make_unique<MemoryAllocatorObserver>(stats), attribute::getUndefined<NumericEnumStore::EntryType>()); EXPECT_EQ(AllocStats(1, 0), stats); } diff --git a/searchlib/src/vespa/searchlib/attribute/enumattribute.hpp b/searchlib/src/vespa/searchlib/attribute/enumattribute.hpp index a28a100cde5..f0f518f64f7 100644 --- a/searchlib/src/vespa/searchlib/attribute/enumattribute.hpp +++ b/searchlib/src/vespa/searchlib/attribute/enumattribute.hpp @@ -15,7 +15,7 @@ EnumAttribute<B>:: EnumAttribute(const vespalib::string &baseFileName, const AttributeVector::Config &cfg) : B(baseFileName, cfg), - _enumStore(cfg.fastSearch(), cfg.get_dictionary_config(), this->get_memory_allocator()) + _enumStore(cfg.fastSearch(), cfg.get_dictionary_config(), this->get_memory_allocator(), this->_defaultValue._data.raw()) { this->setEnum(true); } @@ -96,5 +96,3 @@ EnumAttribute<B>::cache_change_data_entry_ref(const Change& c) const } } // namespace search - - diff --git a/searchlib/src/vespa/searchlib/attribute/enumstore.h b/searchlib/src/vespa/searchlib/attribute/enumstore.h index 4bc17a9092f..f6467194d74 100644 --- a/searchlib/src/vespa/searchlib/attribute/enumstore.h +++ b/searchlib/src/vespa/searchlib/attribute/enumstore.h @@ -80,7 +80,7 @@ private: std::unique_ptr<EntryComparator> allocate_optionally_folded_comparator(bool folded) const; ComparatorType make_optionally_folded_comparator(bool folded) const; public: - EnumStoreT(bool has_postings, const search::DictionaryConfig& dict_cfg, std::shared_ptr<vespalib::alloc::MemoryAllocator> memory_allocator); + EnumStoreT(bool has_postings, const search::DictionaryConfig& dict_cfg, std::shared_ptr<vespalib::alloc::MemoryAllocator> memory_allocator, EntryType default_value); EnumStoreT(bool has_postings, const search::DictionaryConfig & dict_cfg); ~EnumStoreT() override; diff --git a/searchlib/src/vespa/searchlib/attribute/enumstore.hpp b/searchlib/src/vespa/searchlib/attribute/enumstore.hpp index 373102076d5..c0eebee8e94 100644 --- a/searchlib/src/vespa/searchlib/attribute/enumstore.hpp +++ b/searchlib/src/vespa/searchlib/attribute/enumstore.hpp @@ -73,14 +73,14 @@ EnumStoreT<EntryT>::load_unique_value(const void* src, size_t available, Index& } template <typename EntryT> -EnumStoreT<EntryT>::EnumStoreT(bool has_postings, const DictionaryConfig& dict_cfg, std::shared_ptr<vespalib::alloc::MemoryAllocator> memory_allocator) +EnumStoreT<EntryT>::EnumStoreT(bool has_postings, const DictionaryConfig& dict_cfg, std::shared_ptr<vespalib::alloc::MemoryAllocator> memory_allocator, EntryType default_value) : _store(std::move(memory_allocator)), _dict(), _is_folded(dict_cfg.getMatch() == DictionaryConfig::Match::UNCASED), _comparator(_store.get_data_store()), _foldedComparator(make_optionally_folded_comparator(is_folded())), _compaction_spec(), - _default_value(attribute::getUndefined<EntryT>()), + _default_value(default_value), _default_value_ref() { _store.set_dictionary(make_enum_store_dictionary(*this, has_postings, dict_cfg, @@ -92,7 +92,7 @@ EnumStoreT<EntryT>::EnumStoreT(bool has_postings, const DictionaryConfig& dict_c template <typename EntryT> EnumStoreT<EntryT>::EnumStoreT(bool has_postings, const DictionaryConfig& dict_cfg) - : EnumStoreT<EntryT>(has_postings, dict_cfg, {}) + : EnumStoreT<EntryT>(has_postings, dict_cfg, {}, attribute::getUndefined<EntryType>()) { } diff --git a/security-utils/src/main/java/com/yahoo/security/SideChannelSafe.java b/security-utils/src/main/java/com/yahoo/security/SideChannelSafe.java index 1f160d94c6a..bd085f6f624 100644 --- a/security-utils/src/main/java/com/yahoo/security/SideChannelSafe.java +++ b/security-utils/src/main/java/com/yahoo/security/SideChannelSafe.java @@ -46,7 +46,7 @@ public class SideChannelSafe { // differed in any byte compared between the two arrays. byte accu = 0; for (int i = 0; i < lhs.length; ++i) { - accu |= (lhs[i] ^ rhs[i]); + accu |= (byte)(lhs[i] ^ rhs[i]); } return (accu == 0); } diff --git a/security-utils/src/test/java/com/yahoo/security/SharedKeyTest.java b/security-utils/src/test/java/com/yahoo/security/SharedKeyTest.java index 26627e9a5fa..90b8beb461f 100644 --- a/security-utils/src/test/java/com/yahoo/security/SharedKeyTest.java +++ b/security-utils/src/test/java/com/yahoo/security/SharedKeyTest.java @@ -285,12 +285,12 @@ public class SharedKeyTest { String plaintext = "...hello world?"; byte[] encrypted = streamEncryptString(plaintext, myShared); // Corrupt MAC tag in ciphertext - encrypted[encrypted.length - 1] ^= 0x80; + encrypted[encrypted.length - 1] ^= (byte)0x80; // We don't necessarily know _which_ exception is thrown, but one _should_ be thrown! assertThrows(Exception.class, () -> doOutputStreamCipherDecrypt(myShared, encrypted)); // Also try with corrupted ciphertext (pre MAC tag) - encrypted[encrypted.length - 1] ^= 0x80; // Flip MAC bit back to correct state - encrypted[encrypted.length - 17] ^= 0x80; // Pre 128-bit MAC tag + encrypted[encrypted.length - 1] ^= (byte)0x80; // Flip MAC bit back to correct state + encrypted[encrypted.length - 17] ^= (byte)0x80; // Pre 128-bit MAC tag assertThrows(Exception.class, () -> doOutputStreamCipherDecrypt(myShared, encrypted)); } diff --git a/vdslib/src/main/java/com/yahoo/vdslib/distribution/Distribution.java b/vdslib/src/main/java/com/yahoo/vdslib/distribution/Distribution.java index 9a451ac56ec..a83e2a4f89c 100644 --- a/vdslib/src/main/java/com/yahoo/vdslib/distribution/Distribution.java +++ b/vdslib/src/main/java/com/yahoo/vdslib/distribution/Distribution.java @@ -187,11 +187,11 @@ public class Distribution { } private int getStorageSeed(BucketId bucket, ClusterState state) { - int seed = (int) lastNBits(bucket.getRawId(), state.getDistributionBitCount()); + int seed = (int)lastNBits(bucket.getRawId(), state.getDistributionBitCount()); if (bucket.getUsedBits() > 33) { int usedBits = bucket.getUsedBits() - 1; - seed ^= lastNBits(bucket.getRawId() >> 32, usedBits - 32) << 6; + seed ^= (int)lastNBits(bucket.getRawId() >> 32, usedBits - 32) << 6; } return seed; } diff --git a/vespa-feed-client-api/pom.xml b/vespa-feed-client-api/pom.xml index 5509c339eee..0782bb6b28e 100644 --- a/vespa-feed-client-api/pom.xml +++ b/vespa-feed-client-api/pom.xml @@ -42,6 +42,12 @@ <configuration> <release>${vespaClients.jdk.releaseVersion}</release> <showDeprecation>true</showDeprecation> + <compilerArgs> <!-- Remove (to use default) when not compiling for 8 --> + <arg>-Xlint:all</arg> + <arg>-Xlint:-rawtypes</arg> + <arg>-Xlint:-unchecked</arg> + <arg>-Xlint:-serial</arg> + </compilerArgs> </configuration> </plugin> <plugin> diff --git a/vespa-feed-client-cli/pom.xml b/vespa-feed-client-cli/pom.xml index 46679906fc4..b917a39b675 100644 --- a/vespa-feed-client-cli/pom.xml +++ b/vespa-feed-client-cli/pom.xml @@ -53,6 +53,12 @@ <configuration> <release>${vespaClients.jdk.releaseVersion}</release> <showDeprecation>true</showDeprecation> + <compilerArgs> <!-- Remove (to use default) when not compiling for 8 --> + <arg>-Xlint:all</arg> + <arg>-Xlint:-rawtypes</arg> + <arg>-Xlint:-unchecked</arg> + <arg>-Xlint:-serial</arg> + </compilerArgs> </configuration> </plugin> <plugin> diff --git a/vespa-feed-client/pom.xml b/vespa-feed-client/pom.xml index 01b9b00b8a0..b6440653a78 100644 --- a/vespa-feed-client/pom.xml +++ b/vespa-feed-client/pom.xml @@ -65,6 +65,12 @@ <goal>compile</goal> </goals> <configuration> + <compilerArgs> <!-- Remove (to use default) when not compiling for 8 --> + <arg>-Xlint:all</arg> + <arg>-Xlint:-rawtypes</arg> + <arg>-Xlint:-unchecked</arg> + <arg>-Xlint:-serial</arg> + </compilerArgs> <release>${vespaClients.jdk.releaseVersion}</release> <showDeprecation>true</showDeprecation> </configuration> diff --git a/vespajlib/src/main/java/com/yahoo/slime/BinaryEncoder.java b/vespajlib/src/main/java/com/yahoo/slime/BinaryEncoder.java index f12496f7a76..e0b4fb2c672 100644 --- a/vespajlib/src/main/java/com/yahoo/slime/BinaryEncoder.java +++ b/vespajlib/src/main/java/com/yahoo/slime/BinaryEncoder.java @@ -28,7 +28,7 @@ final class BinaryEncoder implements ArrayTraverser, ObjectSymbolTraverser { byte next = (byte)(value & 0x7f); value >>>= 7; // unsigned shift while (value != 0) { - next |= 0x80; + next |= (byte)0x80; out.put(next); next = (byte)(value & 0x7f); value >>>= 7; diff --git a/vespajlib/src/main/java/com/yahoo/tensor/functions/ReduceJoin.java b/vespajlib/src/main/java/com/yahoo/tensor/functions/ReduceJoin.java index 11996b6a23d..de1c30e6414 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/functions/ReduceJoin.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/functions/ReduceJoin.java @@ -333,7 +333,7 @@ public class ReduceJoin<NAMETYPE extends Name> extends CompositeTensorFunction<N private final long[] bounds; private final long[] iterator; - private int remaining; + private long remaining; MultiDimensionIterator(TensorType type) { bounds = new long[type.dimensions().size()]; diff --git a/vespajlib/src/test/java/com/yahoo/io/FatalErrorHandlerTestCase.java b/vespajlib/src/test/java/com/yahoo/io/FatalErrorHandlerTestCase.java deleted file mode 100644 index dab91b6a995..00000000000 --- a/vespajlib/src/test/java/com/yahoo/io/FatalErrorHandlerTestCase.java +++ /dev/null @@ -1,58 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.io; - -import static org.junit.Assert.*; - -import java.security.Permission; - -import org.junit.After; -import org.junit.Before; -import org.junit.Test; - -/** - * Just to remove noise from the coverage report. - * - * @author Steinar Knutsen - */ -public class FatalErrorHandlerTestCase { - @SuppressWarnings("removal") - private static final class AvoidExiting extends SecurityManager { - - @Override - public void checkPermission(Permission perm) { - } - - @Override - public void checkExit(int status) { - throw new SecurityException(); - } - - } - - private FatalErrorHandler h; - - @Before - @SuppressWarnings("removal") - public void setUp() throws Exception { - h = new FatalErrorHandler(); - System.setSecurityManager(new AvoidExiting()); - } - - @After - @SuppressWarnings("removal") - public void tearDown() throws Exception { - System.setSecurityManager(null); - } - - @Test - public final void testHandle() { - boolean caught = false; - try { - h.handle(new Throwable(), "abc"); - } catch (SecurityException e) { - caught = true; - } - assertTrue(caught); - } - -} |