diff options
Diffstat (limited to 'controller-server')
43 files changed, 345 insertions, 253 deletions
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java index dd89dbc14ae..745436e8924 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java @@ -26,6 +26,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.github.GitHub; import com.yahoo.vespa.hosted.controller.api.integration.organization.Mailer; import com.yahoo.vespa.hosted.controller.api.integration.routing.RoutingGenerator; import com.yahoo.vespa.hosted.controller.api.integration.zone.CloudName; +import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry; import com.yahoo.vespa.hosted.controller.athenz.impl.ZmsClientFacade; import com.yahoo.vespa.hosted.controller.deployment.JobController; @@ -48,6 +49,7 @@ import java.util.TreeSet; import java.util.function.Predicate; import java.util.function.Supplier; import java.util.logging.Logger; +import java.util.stream.Collectors; /** * API to the controller. This contains the object model of everything the controller cares about, mainly tenants and @@ -231,7 +233,7 @@ public class Controller extends AbstractComponent { if (version.isEmpty()) { throw new IllegalArgumentException("Invalid version '" + version.toFullString() + "'"); } - if (zoneRegistry.zones().all().ids().stream().noneMatch(zone -> cloud.equals(zone.cloud()))) { + if (!clouds().contains(cloud)) { throw new IllegalArgumentException("Cloud '" + cloud.value() + "' does not exist in this system"); } try (Lock lock = curator.lockOsVersions()) { @@ -255,6 +257,14 @@ public class Controller extends AbstractComponent { /** Replace the current OS version status with a new one */ public void updateOsVersionStatus(OsVersionStatus newStatus) { try (Lock lock = curator.lockOsVersionStatus()) { + OsVersionStatus currentStatus = curator.readOsVersionStatus(); + for (CloudName cloud : clouds()) { + Set<Version> newVersions = newStatus.versionsIn(cloud); + if (currentStatus.versionsIn(cloud).size() > 1 && newVersions.size() == 1) { + log.info("All nodes in " + cloud + " cloud upgraded to OS version " + + newVersions.iterator().next()); + } + } curator.writeOsVersionStatus(newStatus); } } @@ -288,6 +298,12 @@ public class Controller extends AbstractComponent { return curator; } + private Set<CloudName> clouds() { + return zoneRegistry.zones().all().ids().stream() + .map(ZoneId::cloud) + .collect(Collectors.toUnmodifiableSet()); + } + private static String printableVersion(Optional<VespaVersion> vespaVersion) { return vespaVersion.map(v -> v.versionNumber().toFullString()).orElse("unknown"); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationList.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationList.java index 936b0f3fce2..dec1847e751 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationList.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationList.java @@ -10,6 +10,7 @@ import com.yahoo.vespa.hosted.controller.ApplicationController; import java.time.Instant; import java.util.Collection; +import java.util.Collections; import java.util.Comparator; import java.util.List; import java.util.Optional; @@ -64,13 +65,18 @@ public class ApplicationList { } /** Returns the subset of applications which are not pinned to a certain Vespa version. */ - public ApplicationList notPinned() { + public ApplicationList unpinned() { return listOf(list.stream().filter(application -> ! application.change().isPinned())); } /** Returns the subset of applications which are currently not upgrading to the given version */ public ApplicationList notUpgradingTo(Version version) { - return listOf(list.stream().filter(application -> ! isUpgradingTo(version, application))); + return notUpgradingTo(Collections.singletonList(version)); + } + + /** Returns the subset of applications which are currently not upgrading to any of the given versions */ + public ApplicationList notUpgradingTo(Collection<Version> versions) { + return listOf(list.stream().filter(application -> versions.stream().noneMatch(version -> isUpgradingTo(version, application)))); } /** @@ -78,7 +84,7 @@ public class ApplicationList { * or returns all if no version is specified */ public ApplicationList notUpgradingTo(Optional<Version> version) { - if ( ! version.isPresent()) return this; + if (version.isEmpty()) return this; return notUpgradingTo(version.get()); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/LoadBalancerAlias.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/LoadBalancerAlias.java index 4e1c79248a9..d8b90d5aa65 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/LoadBalancerAlias.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/LoadBalancerAlias.java @@ -77,9 +77,8 @@ public class LoadBalancerAlias { List<String> parts = Arrays.asList(ignorePartIfDefault(clusterId.value()), ignorePartIfDefault(applicationId.instance().value()), applicationId.application().value(), - applicationId.tenant().value(), - zoneId.value(), - "vespa.oath.cloud" + applicationId.tenant().value() + + "." + zoneId.value() + "." + "vespa.oath.cloud" ); return parts.stream() .filter(s -> !Strings.isNullOrEmpty((s))) diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentSteps.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentSteps.java index c400ad4c9cc..96b7f68e2e0 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentSteps.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentSteps.java @@ -66,7 +66,7 @@ public class DeploymentSteps { public List<JobType> toJobs(DeploymentSpec.Step step) { return step.zones().stream() .map(this::toJob) - .filter(Optional::isPresent).map(Optional::get) + .flatMap(Optional::stream) .collect(collectingAndThen(Collectors.toList(), Collections::unmodifiableList)); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java index 1956a56a6a5..51d4033044f 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java @@ -45,7 +45,6 @@ import java.net.URI; import java.nio.charset.StandardCharsets; import java.time.Duration; import java.util.ArrayList; -import java.util.Arrays; import java.util.List; import java.util.Map; import java.util.Optional; @@ -65,6 +64,7 @@ import static com.yahoo.vespa.hosted.controller.api.integration.configserver.Con import static com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException.ErrorCode.BAD_REQUEST; import static com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException.ErrorCode.INVALID_APPLICATION_PACKAGE; import static com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException.ErrorCode.OUT_OF_CAPACITY; +import static com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException.ErrorCode.PARENT_HOST_NOT_READY; import static com.yahoo.vespa.hosted.controller.api.integration.configserver.Node.State.active; import static com.yahoo.vespa.hosted.controller.api.integration.configserver.Node.State.reserved; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.aborted; @@ -221,7 +221,8 @@ public class InternalStepRunner implements StepRunner { catch (ConfigServerException e) { if ( e.getErrorCode() == OUT_OF_CAPACITY && type.isTest() || e.getErrorCode() == ACTIVATION_CONFLICT - || e.getErrorCode() == APPLICATION_LOCK_FAILURE) { + || e.getErrorCode() == APPLICATION_LOCK_FAILURE + || e.getErrorCode() == PARENT_HOST_NOT_READY) { logger.log("Will retry, because of '" + e.getErrorCode() + "' deploying:\n" + e.getMessage()); return Optional.empty(); } @@ -646,7 +647,7 @@ public class InternalStepRunner implements StepRunner { } private void log(String... messages) { - log(Arrays.asList(messages)); + log(List.of(messages)); } private void log(List<String> messages) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java index fcead86893a..063248071fb 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java @@ -189,7 +189,7 @@ public class JobController { return copyOf(applications().stream() .flatMap(id -> Stream.of(JobType.values()) .map(type -> last(id, type)) - .filter(Optional::isPresent).map(Optional::get) + .flatMap(Optional::stream) .filter(run -> ! run.hasEnded())) .iterator()); } @@ -340,7 +340,7 @@ public class JobController { return badges.overview(id, steps.jobs().stream() .map(type -> last(id, type)) - .filter(Optional::isPresent).map(Optional::get) + .flatMap(Optional::stream) .collect(toList())); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java index b38f55826d6..5045ab0877c 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java @@ -75,7 +75,7 @@ public class ControllerMaintenance extends AbstractComponent { clusterUtilizationMaintainer = new ClusterUtilizationMaintainer(controller, Duration.ofHours(2), jobControl); deploymentMetricsMaintainer = new DeploymentMetricsMaintainer(controller, Duration.ofMinutes(5), jobControl); applicationOwnershipConfirmer = new ApplicationOwnershipConfirmer(controller, Duration.ofHours(12), jobControl, ownershipIssues); - dnsMaintainer = new DnsMaintainer(controller, Duration.ofHours(12), jobControl, nameService); + dnsMaintainer = new DnsMaintainer(controller, Duration.ofMinutes(5), jobControl, nameService); systemUpgrader = new SystemUpgrader(controller, Duration.ofMinutes(1), jobControl); jobRunner = new JobRunner(controller, Duration.ofMinutes(2), jobControl); osUpgraders = osUpgraders(controller, jobControl); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DnsMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DnsMaintainer.java index df368c3c60c..36d83b8ee6a 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DnsMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DnsMaintainer.java @@ -12,7 +12,12 @@ import com.yahoo.vespa.hosted.controller.rotation.RotationLock; import com.yahoo.vespa.hosted.controller.rotation.RotationRepository; import java.time.Duration; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; import java.util.Map; +import java.util.Optional; +import java.util.concurrent.atomic.AtomicInteger; import java.util.logging.Logger; /** @@ -25,6 +30,7 @@ public class DnsMaintainer extends Maintainer { private static final Logger log = Logger.getLogger(DnsMaintainer.class.getName()); private final NameService nameService; + private final AtomicInteger rotationIndex = new AtomicInteger(0); public DnsMaintainer(Controller controller, Duration interval, JobControl jobControl, NameService nameService) { @@ -40,21 +46,37 @@ public class DnsMaintainer extends Maintainer { protected void maintain() { try (RotationLock lock = rotationRepository().lock()) { Map<RotationId, Rotation> unassignedRotations = rotationRepository().availableRotations(lock); - unassignedRotations.values().forEach(this::removeDnsAlias); + rotationToCheckOf(unassignedRotations.values()).ifPresent(this::removeDnsAlias); } } /** Remove DNS alias for unassigned rotation */ private void removeDnsAlias(Rotation rotation) { // When looking up CNAME by data, the data must be a FQDN - nameService.findRecord(Record.Type.CNAME, RecordData.fqdn(rotation.name())).stream() - .filter(DnsMaintainer::canUpdate) - .forEach(record -> { - log.info(String.format("Removing DNS record %s (%s) because it points to the unassigned " + - "rotation %s (%s)", record.id().asString(), - record.name().asString(), rotation.id().asString(), rotation.name())); - nameService.removeRecord(record.id()); - }); + nameService.findRecords(Record.Type.CNAME, RecordData.fqdn(rotation.name())).stream() + .filter(DnsMaintainer::canUpdate) + .forEach(record -> { + log.info(String.format("Removing DNS record %s (%s) because it points to the unassigned " + + "rotation %s (%s)", record.id().asString(), + record.name().asString(), rotation.id().asString(), rotation.name())); + nameService.removeRecord(record.id()); + }); + } + + /** + * Returns the rotation that should be checked in this run. We check only one rotation per run to avoid running into + * rate limits that may be imposed by the {@link NameService} implementation. + */ + private Optional<Rotation> rotationToCheckOf(Collection<Rotation> rotations) { + if (rotations.isEmpty()) return Optional.empty(); + List<Rotation> rotationList = new ArrayList<>(rotations); + int index = rotationIndex.getAndUpdate((i)-> { + if (i < rotationList.size() - 1) { + return ++i; + } + return 0; + }); + return Optional.of(rotationList.get(index)); } /** Returns whether we can update the given record */ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/LoadBalancerAliasMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/LoadBalancerAliasMaintainer.java index 6c5000e3e3d..407fdfbe466 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/LoadBalancerAliasMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/LoadBalancerAliasMaintainer.java @@ -95,11 +95,15 @@ public class LoadBalancerAliasMaintainer extends Maintainer { HostName alias = HostName.from(LoadBalancerAlias.createAlias(loadBalancer.cluster(), application, zone)); RecordName name = RecordName.from(alias.value()); RecordData data = RecordData.fqdn(loadBalancer.hostname().value()); - Optional<Record> existingRecord = nameService.findRecord(Record.Type.CNAME, name); + List<Record> existingRecords = nameService.findRecords(Record.Type.CNAME, name); + if (existingRecords.size() > 1) { + throw new IllegalStateException("Found more than 1 CNAME record for " + name.asString() + ": " + existingRecords); + } + Optional<Record> record = existingRecords.stream().findFirst(); RecordId id; - if(existingRecord.isPresent()) { - id = existingRecord.get().id(); - nameService.updateRecord(existingRecord.get().id(), data); + if (record.isPresent()) { + id = record.get().id(); + nameService.updateRecord(id, data); } else { id = nameService.createCname(name, data); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java index 2ae38fabb94..07680fc8b72 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java @@ -20,7 +20,6 @@ import com.yahoo.vespa.hosted.controller.rotation.RotationLock; import java.time.Clock; import java.time.Duration; import java.time.Instant; -import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -80,15 +79,15 @@ public class MetricsReporter extends Maintainer { query += " AND hosted_system:" + system; } PartialNodeResult nodeResult = chefClient.partialSearchNodes(query, - Arrays.asList( + List.of( AttributeMapping.simpleMapping("fqdn"), AttributeMapping.simpleMapping("ohai_time"), - AttributeMapping.deepMapping("tenant", Arrays.asList("hosted", "owner", "tenant")), - AttributeMapping.deepMapping("application", Arrays.asList("hosted", "owner", "application")), - AttributeMapping.deepMapping("instance", Arrays.asList("hosted", "owner", "instance")), - AttributeMapping.deepMapping("environment", Arrays.asList("hosted", "environment")), - AttributeMapping.deepMapping("region", Arrays.asList("hosted", "region")), - AttributeMapping.deepMapping("system", Arrays.asList("hosted", "system")) + AttributeMapping.deepMapping("tenant", List.of("hosted", "owner", "tenant")), + AttributeMapping.deepMapping("application", List.of("hosted", "owner", "application")), + AttributeMapping.deepMapping("instance", List.of("hosted", "owner", "instance")), + AttributeMapping.deepMapping("environment", List.of("hosted", "environment")), + AttributeMapping.deepMapping("region", List.of("hosted", "region")), + AttributeMapping.deepMapping("system", List.of("hosted", "system")) )); // The above search will return a correct list if the system is CD. However for main, it will diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgrader.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgrader.java index 7f3b2400736..dcdd28544c1 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgrader.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgrader.java @@ -61,7 +61,7 @@ public class OsUpgrader extends InfrastructureUpgrader { protected Optional<Version> targetVersion() { // Return target if we have nodes in this cloud on a lower version return controller().osVersion(cloud) - .filter(target -> controller().osVersionStatus().nodeVersionsIn(cloud).stream() + .filter(target -> controller().osVersionStatus().nodesIn(cloud).stream() .anyMatch(node -> node.version().isBefore(target.version()))) .map(OsVersion::version); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java index 4f1f7d4b277..3180cc33b8c 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java @@ -8,21 +8,21 @@ import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.application.ApplicationList; import com.yahoo.vespa.hosted.controller.application.Change; -import com.yahoo.vespa.hosted.controller.deployment.DeploymentTrigger.ChangesToCancel; import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; import com.yahoo.vespa.hosted.controller.versions.VespaVersion; import com.yahoo.vespa.hosted.controller.versions.VespaVersion.Confidence; -import com.yahoo.yolean.Exceptions; import java.time.Duration; -import java.util.ArrayList; +import java.util.Collection; +import java.util.Comparator; import java.util.LinkedHashMap; -import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Optional; -import java.util.logging.Level; +import java.util.function.BinaryOperator; +import java.util.function.Function; import java.util.logging.Logger; +import java.util.stream.Collectors; import static com.yahoo.vespa.hosted.controller.deployment.DeploymentTrigger.ChangesToCancel.PLATFORM; @@ -50,8 +50,8 @@ public class Upgrader extends Maintainer { public void maintain() { // Determine target versions for each upgrade policy Optional<Version> canaryTarget = controller().versionStatus().systemVersion().map(VespaVersion::versionNumber); - Optional<Version> defaultTarget = newestVersionWithConfidence(Confidence.normal); - Optional<Version> conservativeTarget = newestVersionWithConfidence(Confidence.high); + Collection<Version> defaultTargets = targetVersions(Confidence.normal); + Collection<Version> conservativeTargets = targetVersions(Confidence.high); // Cancel upgrades to broken targets (let other ongoing upgrades complete to avoid starvation) for (VespaVersion version : controller().versionStatus().versions()) { @@ -66,34 +66,31 @@ public class Upgrader extends Maintainer { // Cancel *failed* upgrades to earlier versions, as the new version may fix it String reason = "Failing on outdated version"; - cancelUpgradesOf(applications().with(UpgradePolicy.defaultPolicy).upgrading().failing().notUpgradingTo(defaultTarget), reason); - cancelUpgradesOf(applications().with(UpgradePolicy.conservative).upgrading().failing().notUpgradingTo(conservativeTarget), reason); + cancelUpgradesOf(applications().with(UpgradePolicy.defaultPolicy).upgrading().failing().notUpgradingTo(defaultTargets), reason); + cancelUpgradesOf(applications().with(UpgradePolicy.conservative).upgrading().failing().notUpgradingTo(conservativeTargets), reason); // Schedule the right upgrades canaryTarget.ifPresent(target -> upgrade(applications().with(UpgradePolicy.canary), target)); - defaultTarget.ifPresent(target -> upgrade(applications().with(UpgradePolicy.defaultPolicy), target)); - conservativeTarget.ifPresent(target -> upgrade(applications().with(UpgradePolicy.conservative), target)); + defaultTargets.forEach(target -> upgrade(applications().with(UpgradePolicy.defaultPolicy), target)); + conservativeTargets.forEach(target -> upgrade(applications().with(UpgradePolicy.conservative), target)); } - private Optional<Version> newestVersionWithConfidence(Confidence confidence) { - return reversed(controller().versionStatus().versions()).stream() - // Ensure we never pick a version newer than the system - .filter(v -> !v.versionNumber().isAfter(controller().systemVersion())) - .filter(v -> v.confidence().equalOrHigherThan(confidence)) - .findFirst() - .map(VespaVersion::versionNumber); - } - - private List<VespaVersion> reversed(List<VespaVersion> versions) { - List<VespaVersion> reversed = new ArrayList<>(versions.size()); - for (int i = 0; i < versions.size(); i++) - reversed.add(versions.get(versions.size() - 1 - i)); - return reversed; + /** Returns the target versions for given confidence, one per major version in the system */ + private Collection<Version> targetVersions(Confidence confidence) { + return controller().versionStatus().versions().stream() + // Ensure we never pick a version newer than the system + .filter(v -> !v.versionNumber().isAfter(controller().systemVersion())) + .filter(v -> v.confidence().equalOrHigherThan(confidence)) + .map(VespaVersion::versionNumber) + .collect(Collectors.toMap(Version::getMajor, // Key on major version + Function.identity(), // Use version as value + BinaryOperator.<Version>maxBy(Comparator.naturalOrder()))) // Pick highest version when merging versions within this major + .values(); } /** Returns a list of all applications, except those which are pinned — these should not be manipulated by the Upgrader */ private ApplicationList applications() { - return ApplicationList.from(controller().applications().asList()).notPinned(); + return ApplicationList.from(controller().applications().asList()).unpinned(); } private void upgrade(ApplicationList applications, Version version) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/BufferedLogStore.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/BufferedLogStore.java index 2833bc8614e..b761c194cae 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/BufferedLogStore.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/BufferedLogStore.java @@ -95,7 +95,7 @@ public class BufferedLogStore { while (firstChunk > 0 && chunkIds[--firstChunk] > after + 1); return logSerializer.fromJson(Arrays.stream(chunkIds, firstChunk, chunkIds.length) .mapToObj(chunkId -> buffer.readLog(id, type, chunkId)) - .filter(Optional::isPresent).map(Optional::get) + .flatMap(Optional::stream) .collect(Collectors.toList()), after); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java index d509b2b62d4..466e00cef2a 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java @@ -324,8 +324,7 @@ public class CuratorDb { public List<Tenant> readTenants() { return readTenantNames().stream() .map(this::readTenant) - .filter(Optional::isPresent) - .map(Optional::get) + .flatMap(Optional::stream) .collect(collectingAndThen(Collectors.toList(), Collections::unmodifiableList)); } @@ -362,8 +361,7 @@ public class CuratorDb { .map(ApplicationId::fromSerializedForm) .filter(applicationFilter) .map(this::readApplication) - .filter(Optional::isPresent) - .map(Optional::get) + .flatMap(Optional::stream) .collect(collectingAndThen(Collectors.toList(), Collections::unmodifiableList)); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutorImpl.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutorImpl.java index ae1102e2cef..2f10dce0e0a 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutorImpl.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutorImpl.java @@ -39,9 +39,7 @@ import java.net.URI; import java.security.cert.X509Certificate; import java.time.Duration; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collections; -import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Optional; @@ -60,7 +58,7 @@ public class ConfigServerRestExecutorImpl implements ConfigServerRestExecutor { private static final Logger log = Logger.getLogger(ConfigServerRestExecutorImpl.class.getName()); private static final Duration PROXY_REQUEST_TIMEOUT = Duration.ofSeconds(10); - private static final Set<String> HEADERS_TO_COPY = new HashSet<>(Arrays.asList("X-HTTP-Method-Override", "Content-Type")); + private static final Set<String> HEADERS_TO_COPY = Set.of("X-HTTP-Method-Override", "Content-Type"); private final ZoneRegistry zoneRegistry; private final ServiceIdentityProvider sslContextProvider; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java index 218fac75185..2fb493cad42 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java @@ -86,7 +86,7 @@ class JobControllerApiHandlerHelper { Map<JobType, Run> running = steps.jobs().stream() .map(type -> controller.jobController().last(id, type)) - .filter(Optional::isPresent).map(Optional::get) + .flatMap(Optional::stream) .filter(run -> ! run.hasEnded()) .collect(toMap(run -> run.id().type(), run -> run)); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/ControllerAuthorizationFilter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/ControllerAuthorizationFilter.java index b8e868ed8a4..77e626509b3 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/ControllerAuthorizationFilter.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/ControllerAuthorizationFilter.java @@ -17,8 +17,8 @@ import com.yahoo.vespa.athenz.api.AthenzUser; import com.yahoo.vespa.athenz.client.zms.ZmsClientException; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.TenantController; -import com.yahoo.vespa.hosted.controller.athenz.ApplicationAction; import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzClientFactory; +import com.yahoo.vespa.hosted.controller.athenz.ApplicationAction; import com.yahoo.vespa.hosted.controller.athenz.impl.ZmsClientFacade; import com.yahoo.vespa.hosted.controller.tenant.AthenzTenant; import com.yahoo.vespa.hosted.controller.tenant.Tenant; @@ -30,7 +30,6 @@ import javax.ws.rs.ForbiddenException; import javax.ws.rs.InternalServerErrorException; import javax.ws.rs.NotAuthorizedException; import javax.ws.rs.WebApplicationException; -import java.util.Arrays; import java.util.List; import java.util.Optional; import java.util.Set; @@ -52,7 +51,7 @@ import static com.yahoo.vespa.hosted.controller.athenz.HostedAthenzIdentities.SC @Provides("ControllerAuthorizationFilter") public class ControllerAuthorizationFilter extends CorsRequestFilterBase { - private static final List<Method> WHITELISTED_METHODS = Arrays.asList(GET, OPTIONS, HEAD); + private static final List<Method> WHITELISTED_METHODS = List.of(GET, OPTIONS, HEAD); private static final Logger log = Logger.getLogger(ControllerAuthorizationFilter.class.getName()); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/OsVersionStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/OsVersionStatus.java index 871a7872a9c..aa174b0df2b 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/OsVersionStatus.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/OsVersionStatus.java @@ -19,10 +19,9 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Set; import java.util.stream.Collectors; -import static java.util.stream.Collectors.collectingAndThen; - /** * Information about OS versions in this system. * @@ -44,12 +43,20 @@ public class OsVersionStatus { return versions; } - /** Returns node versions that exist in given cloud */ - public List<Node> nodeVersionsIn(CloudName cloud) { + /** Returns nodes eligible for OS upgrades that exist in given cloud */ + public List<Node> nodesIn(CloudName cloud) { return versions.entrySet().stream() .filter(entry -> entry.getKey().cloud().equals(cloud)) .flatMap(entry -> entry.getValue().stream()) - .collect(collectingAndThen(Collectors.toList(), Collections::unmodifiableList)); + .collect(Collectors.toUnmodifiableList()); + } + + /** Returns versions that exist in given cloud */ + public Set<Version> versionsIn(CloudName cloud) { + return versions.keySet().stream() + .filter(osVersion -> osVersion.cloud().equals(cloud)) + .map(OsVersion::version) + .collect(Collectors.toUnmodifiableSet()); } /** Compute the current OS versions in this system. This is expensive and should be called infrequently */ @@ -84,7 +91,7 @@ public class OsVersionStatus { return controller.zoneRegistry().osUpgradePolicies().stream() .flatMap(upgradePolicy -> upgradePolicy.asList().stream()) .flatMap(Collection::stream) - .collect(Collectors.toList()); + .collect(Collectors.toUnmodifiableList()); } /** A node in this system and its current OS version */ diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java index 343deaec752..06417da8157 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java @@ -33,7 +33,7 @@ import com.yahoo.vespa.hosted.controller.rotation.RotationLock; import org.junit.Test; import java.time.Duration; -import java.util.Arrays; +import java.util.List; import java.util.Map; import java.util.Optional; import java.util.function.Function; @@ -227,7 +227,7 @@ public class ControllerTest { ZoneId zone = ZoneId.from(Environment.defaultEnvironment(), RegionName.defaultName()); ApplicationId app = ApplicationId.from("tenant", "app1", "default"); DeploymentId deployment = new DeploymentId(app, zone); - tester.routingGenerator().putEndpoints(deployment, Arrays.asList( + tester.routingGenerator().putEndpoints(deployment, List.of( new RoutingEndpoint("http://old-endpoint.vespa.yahooapis.com:4080", "host1", false, "upstream2"), new RoutingEndpoint("http://qrs-endpoint.vespa.yahooapis.com:4080", "host1", false, "upstream1"), new RoutingEndpoint("http://feeding-endpoint.vespa.yahooapis.com:4080", "host2", false, "upstream3"), @@ -256,7 +256,7 @@ public class ControllerTest { assertEquals("unit-test", findStatusByUpstream.apply("upstream1").get().getReason()); // Deployment without a global endpoint - tester.routingGenerator().putEndpoints(deployment, Arrays.asList( + tester.routingGenerator().putEndpoints(deployment, List.of( new RoutingEndpoint("http://old-endpoint.vespa.yahooapis.com:4080", "host1", false, "upstream2"), new RoutingEndpoint("http://qrs-endpoint.vespa.yahooapis.com:4080", "host1", false, "upstream1"), new RoutingEndpoint("http://feeding-endpoint.vespa.yahooapis.com:4080", "host2", false, "upstream3") @@ -280,26 +280,26 @@ public class ControllerTest { .region("us-central-1") // Two deployments should result in each DNS alias being registered once .build(); + Function<String, Optional<Record>> findCname = (name) -> tester.controllerTester().nameService() + .findRecords(Record.Type.CNAME, + RecordName.from(name)) + .stream() + .findFirst(); + tester.deployCompletely(application, applicationPackage); assertEquals(3, tester.controllerTester().nameService().records().size()); - Optional<Record> record = tester.controllerTester().nameService().findRecord( - Record.Type.CNAME, RecordName.from("app1--tenant1.global.vespa.yahooapis.com") - ); + Optional<Record> record = findCname.apply("app1--tenant1.global.vespa.yahooapis.com"); assertTrue(record.isPresent()); assertEquals("app1--tenant1.global.vespa.yahooapis.com", record.get().name().asString()); assertEquals("rotation-fqdn-01.", record.get().data().asString()); - record = tester.controllerTester().nameService().findRecord( - Record.Type.CNAME, RecordName.from("app1--tenant1.global.vespa.oath.cloud") - ); + record = findCname.apply("app1--tenant1.global.vespa.oath.cloud"); assertTrue(record.isPresent()); assertEquals("app1--tenant1.global.vespa.oath.cloud", record.get().name().asString()); assertEquals("rotation-fqdn-01.", record.get().data().asString()); - record = tester.controllerTester().nameService().findRecord( - Record.Type.CNAME, RecordName.from("app1.tenant1.global.vespa.yahooapis.com") - ); + record = findCname.apply("app1.tenant1.global.vespa.yahooapis.com"); assertTrue(record.isPresent()); assertEquals("app1.tenant1.global.vespa.yahooapis.com", record.get().name().asString()); assertEquals("rotation-fqdn-01.", record.get().data().asString()); @@ -309,6 +309,12 @@ public class ControllerTest { public void testUpdatesExistingDnsAlias() { DeploymentTester tester = new DeploymentTester(); + Function<String, Optional<Record>> findCname = (name) -> tester.controllerTester().nameService() + .findRecords(Record.Type.CNAME, + RecordName.from(name)) + .stream() + .findFirst(); + // Application 1 is deployed and deleted { Application app1 = tester.createApplication("app1", "tenant1", 1, 1L); @@ -322,16 +328,12 @@ public class ControllerTest { tester.deployCompletely(app1, applicationPackage); assertEquals(3, tester.controllerTester().nameService().records().size()); - Optional<Record> record = tester.controllerTester().nameService().findRecord( - Record.Type.CNAME, RecordName.from("app1--tenant1.global.vespa.yahooapis.com") - ); + Optional<Record> record = findCname.apply("app1--tenant1.global.vespa.yahooapis.com"); assertTrue(record.isPresent()); assertEquals("app1--tenant1.global.vespa.yahooapis.com", record.get().name().asString()); assertEquals("rotation-fqdn-01.", record.get().data().asString()); - record = tester.controllerTester().nameService().findRecord( - Record.Type.CNAME, RecordName.from("app1.tenant1.global.vespa.yahooapis.com") - ); + record = findCname.apply("app1.tenant1.global.vespa.yahooapis.com"); assertTrue(record.isPresent()); assertEquals("app1.tenant1.global.vespa.yahooapis.com", record.get().name().asString()); assertEquals("rotation-fqdn-01.", record.get().data().asString()); @@ -353,19 +355,13 @@ public class ControllerTest { } // Records remain - record = tester.controllerTester().nameService().findRecord( - Record.Type.CNAME, RecordName.from("app1--tenant1.global.vespa.yahooapis.com") - ); + record = findCname.apply("app1--tenant1.global.vespa.yahooapis.com"); assertTrue(record.isPresent()); - record = tester.controllerTester().nameService().findRecord( - Record.Type.CNAME, RecordName.from("app1--tenant1.global.vespa.oath.cloud") - ); + record = findCname.apply("app1--tenant1.global.vespa.oath.cloud"); assertTrue(record.isPresent()); - record = tester.controllerTester().nameService().findRecord( - Record.Type.CNAME, RecordName.from("app1.tenant1.global.vespa.yahooapis.com") - ); + record = findCname.apply("app1.tenant1.global.vespa.yahooapis.com"); assertTrue(record.isPresent()); } @@ -381,27 +377,20 @@ public class ControllerTest { tester.deployCompletely(app2, applicationPackage); assertEquals(6, tester.controllerTester().nameService().records().size()); - Optional<Record> record = tester.controllerTester().nameService().findRecord( - Record.Type.CNAME, RecordName.from("app2--tenant2.global.vespa.yahooapis.com") - ); + Optional<Record> record = findCname.apply("app2--tenant2.global.vespa.yahooapis.com"); assertTrue(record.isPresent()); assertEquals("app2--tenant2.global.vespa.yahooapis.com", record.get().name().asString()); assertEquals("rotation-fqdn-01.", record.get().data().asString()); - record = tester.controllerTester().nameService().findRecord( - Record.Type.CNAME, RecordName.from("app2--tenant2.global.vespa.oath.cloud") - ); + record = findCname.apply("app2--tenant2.global.vespa.oath.cloud"); assertTrue(record.isPresent()); assertEquals("app2--tenant2.global.vespa.oath.cloud", record.get().name().asString()); assertEquals("rotation-fqdn-01.", record.get().data().asString()); - record = tester.controllerTester().nameService().findRecord( - Record.Type.CNAME, RecordName.from("app2.tenant2.global.vespa.yahooapis.com") - ); + record = findCname.apply("app2.tenant2.global.vespa.yahooapis.com"); assertTrue(record.isPresent()); assertEquals("app2.tenant2.global.vespa.yahooapis.com", record.get().name().asString()); assertEquals("rotation-fqdn-01.", record.get().data().asString()); - } // Application 1 is recreated, deployed and assigned a new rotation @@ -421,24 +410,17 @@ public class ControllerTest { // Existing DNS records are updated to point to the newly assigned rotation assertEquals(6, tester.controllerTester().nameService().records().size()); - Optional<Record> record = tester.controllerTester().nameService().findRecord( - Record.Type.CNAME, RecordName.from("app1--tenant1.global.vespa.yahooapis.com") - ); + Optional<Record> record = findCname.apply("app1--tenant1.global.vespa.yahooapis.com"); assertTrue(record.isPresent()); assertEquals("rotation-fqdn-02.", record.get().data().asString()); - record = tester.controllerTester().nameService().findRecord( - Record.Type.CNAME, RecordName.from("app1--tenant1.global.vespa.oath.cloud") - ); + record = findCname.apply("app1--tenant1.global.vespa.oath.cloud"); assertTrue(record.isPresent()); assertEquals("rotation-fqdn-02.", record.get().data().asString()); - record = tester.controllerTester().nameService().findRecord( - Record.Type.CNAME, RecordName.from("app1.tenant1.global.vespa.yahooapis.com") - ); + record = findCname.apply("app1.tenant1.global.vespa.yahooapis.com"); assertTrue(record.isPresent()); assertEquals("rotation-fqdn-02.", record.get().data().asString()); - } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java index 1e5e642cfd4..fe2394d872d 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java @@ -67,6 +67,8 @@ import static org.junit.Assert.assertNotNull; */ public final class ControllerTester { + public static final int availableRotations = 10; + private final AthenzDbMock athenzDb; private final ManualClock clock; private final ConfigServerMock configServer; @@ -332,7 +334,7 @@ public final class ControllerTester { private static RotationsConfig defaultRotationsConfig() { RotationsConfig.Builder builder = new RotationsConfig.Builder(); - for (int i = 1; i <= 10; i++) { + for (int i = 1; i <= availableRotations; i++) { String id = String.format("%02d", i); builder = builder.rotations("rotation-id-" + id, "rotation-fqdn-" + id); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/LoadBalancerAliasTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/LoadBalancerAliasTest.java index 965a639a68c..a331ee07239 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/LoadBalancerAliasTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/LoadBalancerAliasTest.java @@ -17,13 +17,13 @@ public class LoadBalancerAliasTest { @Test public void test_endpoint_names() { ZoneId zoneId = ZoneId.from("prod", "us-north-1"); - ApplicationId withInstanceName = ApplicationId.from("tenant", "application", "instance"); - testAlias("instance--application--tenant--prod.us-north-1--vespa.oath.cloud", "default", withInstanceName, zoneId); - testAlias("cluster--instance--application--tenant--prod.us-north-1--vespa.oath.cloud", "cluster", withInstanceName, zoneId); + ApplicationId withInstance = ApplicationId.from("tenant", "application", "instance"); + testAlias("instance--application--tenant.prod.us-north-1.vespa.oath.cloud", "default", withInstance, zoneId); + testAlias("cluster--instance--application--tenant.prod.us-north-1.vespa.oath.cloud", "cluster", withInstance, zoneId); ApplicationId withDefaultInstance = ApplicationId.from("tenant", "application", "default"); - testAlias("application--tenant--prod.us-north-1--vespa.oath.cloud", "default", withDefaultInstance, zoneId); - testAlias("cluster--application--tenant--prod.us-north-1--vespa.oath.cloud", "cluster", withDefaultInstance, zoneId); + testAlias("application--tenant.prod.us-north-1.vespa.oath.cloud", "default", withDefaultInstance, zoneId); + testAlias("cluster--application--tenant.prod.us-north-1.vespa.oath.cloud", "cluster", withDefaultInstance, zoneId); } private void testAlias(String expected, String clusterName, ApplicationId applicationId, ZoneId zoneId) { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/BadgesTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/BadgesTest.java index f219dd66dd6..e5fe658cf3b 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/BadgesTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/BadgesTest.java @@ -12,9 +12,9 @@ import java.net.URI; import java.net.URLEncoder; import java.nio.charset.StandardCharsets; import java.time.Instant; -import java.util.Arrays; import java.util.Collections; import java.util.EnumMap; +import java.util.List; import java.util.Optional; import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.stagingTest; @@ -55,12 +55,12 @@ public class BadgesTest { "/" + systemTest.jobName() + ";" + Badges.blue + "/%20;" + Badges.blue + ";s%7B" + Badges.white + "%7D" + "/%20;" + Badges.purple + ";s%7B" + Badges.white + "%7D"), - badges.historic(id, Optional.of(success), Arrays.asList(success, running))); + badges.historic(id, Optional.of(success), List.of(success, running))); assertEquals(URI.create("https://badges.tld/api/tenant.application;" + Badges.dark + "/" + systemTest.jobName() + ";" + Badges.purple + "/" + stagingTest.jobName() + ";" + Badges.red), - badges.overview(id, Arrays.asList(running, failure))); + badges.overview(id, List.of(running, failure))); } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java index c5484e1db07..d6365eff807 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java @@ -29,8 +29,8 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.time.Duration; -import java.util.Arrays; import java.util.Collections; +import java.util.List; import java.util.Optional; import static com.yahoo.vespa.hosted.controller.api.integration.LogEntry.Type.debug; @@ -283,7 +283,7 @@ public class InternalStepRunnerTest { } private void assertTestLogEntries(RunId id, Step step, LogEntry... entries) { - assertEquals(Arrays.asList(entries), tester.jobs().details(id).get().get(step)); + assertEquals(List.of(entries), tester.jobs().details(id).get().get(step)); } @Test diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java index 45e87859576..4c60e8be677 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java @@ -29,7 +29,6 @@ import com.yahoo.vespa.serviceview.bindings.ClusterView; import com.yahoo.vespa.serviceview.bindings.ServiceView; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; @@ -84,7 +83,7 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer } public void bootstrap(List<ZoneId> zones, SystemApplication... applications) { - bootstrap(zones, Arrays.asList(applications), Optional.empty()); + bootstrap(zones, List.of(applications), Optional.empty()); } public void bootstrap(List<ZoneId> zones, List<SystemApplication> applications, Optional<NodeType> type) { @@ -215,7 +214,7 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer info.time = 2; info.message = "The info"; - return Arrays.asList(warning, info); + return List.of(warning, info); } @Override diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/NodeRepositoryClientMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/NodeRepositoryClientMock.java index 14de73e3f75..daddc46589d 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/NodeRepositoryClientMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/NodeRepositoryClientMock.java @@ -10,8 +10,8 @@ import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeRepo import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeState; import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; -import java.util.Arrays; import java.util.Collection; +import java.util.List; /** * @author bjorncs @@ -37,14 +37,14 @@ public class NodeRepositoryClientMock implements NodeRepositoryClientInterface { public NodeList listNodes(ZoneId zone, boolean recursive) { NodeRepositoryNode nodeA = createNodeA(); NodeRepositoryNode nodeB = createNodeB(); - return new NodeList(Arrays.asList(nodeA, nodeB)); + return new NodeList(List.of(nodeA, nodeB)); } @Override public NodeList listNodes(ZoneId zone, String tenant, String applicationId, String instance) { NodeRepositoryNode nodeA = createNodeA(); NodeRepositoryNode nodeB = createNodeB(); - return new NodeList(Arrays.asList(nodeA, nodeB)); + return new NodeList(List.of(nodeA, nodeB)); } private static NodeRepositoryNode createNodeA() { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/RoutingGeneratorMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/RoutingGeneratorMock.java index 7c32e593eec..410d7950e97 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/RoutingGeneratorMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/RoutingGeneratorMock.java @@ -1,11 +1,10 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.integration; +import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.integration.routing.RoutingEndpoint; import com.yahoo.vespa.hosted.controller.api.integration.routing.RoutingGenerator; -import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; -import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Map; @@ -22,11 +21,11 @@ public class RoutingGeneratorMock implements RoutingGenerator { private final Map<DeploymentId, List<RoutingEndpoint>> routingTable = new ConcurrentHashMap<>(); private static final List<RoutingEndpoint> defaultEndpoints = - Arrays.asList(new RoutingEndpoint("http://old-endpoint.vespa.yahooapis.com:4080", "host1", false, "upstream3"), - new RoutingEndpoint("http://qrs-endpoint.vespa.yahooapis.com:4080", "host1", false, "upstream1"), - new RoutingEndpoint("http://feeding-endpoint.vespa.yahooapis.com:4080", "host2", false, "upstream2"), - new RoutingEndpoint("http://global-endpoint.vespa.yahooapis.com:4080", "host1", true, "upstream1"), - new RoutingEndpoint("http://alias-endpoint.vespa.yahooapis.com:4080", "host1", true, "upstream1")); + List.of(new RoutingEndpoint("http://old-endpoint.vespa.yahooapis.com:4080", "host1", false, "upstream3"), + new RoutingEndpoint("http://qrs-endpoint.vespa.yahooapis.com:4080", "host1", false, "upstream1"), + new RoutingEndpoint("http://feeding-endpoint.vespa.yahooapis.com:4080", "host2", false, "upstream2"), + new RoutingEndpoint("http://global-endpoint.vespa.yahooapis.com:4080", "host1", true, "upstream1"), + new RoutingEndpoint("http://alias-endpoint.vespa.yahooapis.com:4080", "host1", true, "upstream1")); @Override public List<RoutingEndpoint> endpoints(DeploymentId deployment) { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneRegistryMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneRegistryMock.java index 1e9a17ede12..9b13749a71a 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneRegistryMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneRegistryMock.java @@ -8,20 +8,19 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.SystemName; +import com.yahoo.vespa.athenz.api.AthenzService; +import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; import com.yahoo.vespa.hosted.controller.api.integration.zone.CloudName; import com.yahoo.vespa.hosted.controller.api.integration.zone.UpgradePolicy; -import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; -import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; -import com.yahoo.vespa.athenz.api.AthenzService; -import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry; import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneFilter; import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneFilterMock; +import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; +import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry; import java.net.URI; import java.time.Duration; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -64,7 +63,7 @@ public class ZoneRegistryMock extends AbstractComponent implements ZoneRegistry } public ZoneRegistryMock setZones(ZoneId... zone) { - return setZones(Arrays.asList(zone)); + return setZones(List.of(zone)); } public ZoneRegistryMock setSystemName(SystemName system) { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ContactInformationMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ContactInformationMaintainerTest.java index 6a7096dbfae..ae16362e2a7 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ContactInformationMaintainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ContactInformationMaintainerTest.java @@ -4,20 +4,17 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.config.provision.TenantName; import com.yahoo.vespa.hosted.controller.ControllerTester; import com.yahoo.vespa.hosted.controller.api.identifiers.PropertyId; -import com.yahoo.vespa.hosted.controller.api.integration.organization.User; -import com.yahoo.vespa.hosted.controller.tenant.AthenzTenant; import com.yahoo.vespa.hosted.controller.api.integration.organization.Contact; +import com.yahoo.vespa.hosted.controller.tenant.AthenzTenant; import org.junit.Before; import org.junit.Test; import java.net.URI; import java.time.Duration; -import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Optional; import java.util.function.Supplier; -import java.util.stream.Collectors; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -63,8 +60,8 @@ public class ContactInformationMaintainerTest { URI contactUrl = URI.create("http://contact1.test"); URI issueTrackerUrl = URI.create("http://issue-tracker1.test"); URI propertyUrl = URI.create("http://property1.test"); - List<List<String>> persons = Arrays.asList(Collections.singletonList("alice"), - Collections.singletonList("bob")); + List<List<String>> persons = List.of(Collections.singletonList("alice"), + Collections.singletonList("bob")); String queue = "queue"; Optional<String> component = Optional.empty(); return new Contact(contactUrl, propertyUrl, issueTrackerUrl, persons, queue, component); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DnsMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DnsMaintainerTest.java index 92f3e87f001..69b183d9711 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DnsMaintainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DnsMaintainerTest.java @@ -6,17 +6,26 @@ import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.RegionName; import com.yahoo.vespa.athenz.api.OktaAccessToken; import com.yahoo.vespa.hosted.controller.Application; +import com.yahoo.vespa.hosted.controller.ControllerTester; import com.yahoo.vespa.hosted.controller.api.integration.dns.Record; +import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordData; +import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordId; import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordName; import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; +import com.yahoo.vespa.hosted.controller.application.GlobalDnsName; import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; import com.yahoo.vespa.hosted.controller.persistence.MockCuratorDb; +import com.yahoo.vespa.hosted.controller.rotation.Rotation; +import com.yahoo.vespa.hosted.controller.rotation.RotationId; +import org.junit.Before; import org.junit.Test; import java.time.Duration; +import java.util.Map; import java.util.Optional; +import java.util.function.Function; import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.component; import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.systemTest; @@ -29,13 +38,20 @@ import static org.junit.Assert.assertTrue; */ public class DnsMaintainerTest { + private DeploymentTester tester; + private DnsMaintainer maintainer; + + @Before + public void before() { + tester = new DeploymentTester(); + maintainer = new DnsMaintainer(tester.controller(), Duration.ofHours(12), + new JobControl(new MockCuratorDb()), + tester.controllerTester().nameService()); + } + @Test public void removes_record_for_unassigned_rotation() { - DeploymentTester tester = new DeploymentTester(); Application application = tester.createApplication("app1", "tenant1", 1, 1L); - DnsMaintainer dnsMaintainer = new DnsMaintainer(tester.controller(), Duration.ofHours(12), - new JobControl(new MockCuratorDb()), - tester.controllerTester().nameService()); ApplicationPackage applicationPackage = new ApplicationPackageBuilder() .environment(Environment.prod) @@ -44,39 +60,36 @@ public class DnsMaintainerTest { .region("us-central-1") .build(); + Function<String, Optional<Record>> findCname = (name) -> tester.controllerTester().nameService() + .findRecords(Record.Type.CNAME, + RecordName.from(name)) + .stream() + .findFirst(); + // Deploy application tester.deployCompletely(application, applicationPackage); - assertEquals(3, tester.controllerTester().nameService().records().size()); + assertEquals(3, records().size()); - Optional<Record> record = tester.controllerTester().nameService().findRecord( - Record.Type.CNAME, RecordName.from("app1--tenant1.global.vespa.yahooapis.com") - ); + Optional<Record> record = findCname.apply("app1--tenant1.global.vespa.yahooapis.com"); assertTrue(record.isPresent()); assertEquals("app1--tenant1.global.vespa.yahooapis.com", record.get().name().asString()); assertEquals("rotation-fqdn-01.", record.get().data().asString()); - record = tester.controllerTester().nameService().findRecord( - Record.Type.CNAME, RecordName.from("app1--tenant1.global.vespa.oath.cloud") - ); + record = findCname.apply("app1--tenant1.global.vespa.oath.cloud"); assertTrue(record.isPresent()); assertEquals("app1--tenant1.global.vespa.oath.cloud", record.get().name().asString()); assertEquals("rotation-fqdn-01.", record.get().data().asString()); - record = tester.controllerTester().nameService().findRecord( - Record.Type.CNAME, RecordName.from("app1.tenant1.global.vespa.yahooapis.com") - ); + record = findCname.apply("app1.tenant1.global.vespa.yahooapis.com"); assertTrue(record.isPresent()); assertEquals("app1.tenant1.global.vespa.yahooapis.com", record.get().name().asString()); assertEquals("rotation-fqdn-01.", record.get().data().asString()); // DnsMaintainer does nothing - dnsMaintainer.maintain(); - assertTrue("DNS record is not removed", tester.controllerTester().nameService().findRecord( - Record.Type.CNAME, RecordName.from("app1--tenant1.global.vespa.yahooapis.com")).isPresent()); - assertTrue("DNS record is not removed", tester.controllerTester().nameService().findRecord( - Record.Type.CNAME, RecordName.from("app1--tenant1.global.vespa.oath.cloud")).isPresent()); - assertTrue("DNS record is not removed", tester.controllerTester().nameService().findRecord( - Record.Type.CNAME, RecordName.from("app1.tenant1.global.vespa.yahooapis.com")).isPresent()); + maintainer.maintain(); + assertTrue("DNS record is not removed", findCname.apply("app1--tenant1.global.vespa.yahooapis.com").isPresent()); + assertTrue("DNS record is not removed", findCname.apply("app1--tenant1.global.vespa.oath.cloud").isPresent()); + assertTrue("DNS record is not removed", findCname.apply("app1.tenant1.global.vespa.yahooapis.com").isPresent()); // Remove application applicationPackage = new ApplicationPackageBuilder() @@ -91,15 +104,39 @@ public class DnsMaintainerTest { tester.applications().deleteApplication(application.id(), Optional.of(new OktaAccessToken("okta-token"))); // DnsMaintainer removes records - dnsMaintainer.maintain(); - assertFalse("DNS record removed", tester.controllerTester().nameService().findRecord( - Record.Type.CNAME, RecordName.from("app1--tenant1.global.vespa.yahooapis.com")).isPresent()); - dnsMaintainer.maintain(); - assertFalse("DNS record removed", tester.controllerTester().nameService().findRecord( - Record.Type.CNAME, RecordName.from("app1--tenant1.global.vespa.oath.cloud")).isPresent()); - dnsMaintainer.maintain(); - assertFalse("DNS record removed", tester.controllerTester().nameService().findRecord( - Record.Type.CNAME, RecordName.from("app1.tenant1.global.vespa.yahooapis.com")).isPresent()); + for (int i = 0; i < ControllerTester.availableRotations; i++) { + maintainer.maintain(); + } + assertFalse("DNS record removed", findCname.apply("app1--tenant1.global.vespa.yahooapis.com").isPresent()); + assertFalse("DNS record removed", findCname.apply("app1--tenant1.global.vespa.oath.cloud").isPresent()); + assertFalse("DNS record removed", findCname.apply("app1.tenant1.global.vespa.yahooapis.com").isPresent()); + } + + @Test + public void rate_limit_record_removal() { + // Create stale records + int staleTotal = ControllerTester.availableRotations; + for (int i = 1; i <= staleTotal; i++) { + Rotation r = rotation(i); + tester.controllerTester().nameService().createCname(RecordName.from("stale-record-" + i + "." + + GlobalDnsName.OATH_DNS_SUFFIX), + RecordData.from(r.name() + ".")); + } + + // One record is removed per run + for (int i = 1; i <= staleTotal*2; i++) { + maintainer.run(); + assertEquals(Math.max(staleTotal - i, 0), records().size()); + } + } + + private Map<RecordId, Record> records() { + return tester.controllerTester().nameService().records(); + } + + private static Rotation rotation(int n) { + String id = String.format("%02d", n); + return new Rotation(new RotationId("rotation-id-" + id), "rotation-fqdn-" + id); } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java index 8ebb8c108c0..2539687ea4d 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java @@ -4,8 +4,8 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; import com.yahoo.vespa.athenz.api.OktaAccessToken; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.SourceRevision; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; @@ -19,7 +19,6 @@ import com.yahoo.vespa.hosted.controller.deployment.Versions; import org.junit.Test; import java.time.Duration; -import java.util.Arrays; import java.util.Collections; import java.util.EnumMap; import java.util.List; @@ -75,7 +74,7 @@ public class JobRunnerTest { Optional.empty()); @Test - public void multiThreadedExecutionFinishes() throws InterruptedException { + public void multiThreadedExecutionFinishes() { DeploymentTester tester = new DeploymentTester(); JobController jobs = tester.controller().jobController(); StepRunner stepRunner = (step, id) -> id.type() == stagingTest && step.get() == startTests? Optional.of(error) : Optional.of(running); @@ -124,38 +123,38 @@ public class JobRunnerTest { Map<Step, Status> steps = run.get().steps(); runner.maintain(); assertEquals(steps, run.get().steps()); - assertEquals(Arrays.asList(deployReal, deployTester), run.get().readySteps()); + assertEquals(List.of(deployReal, deployTester), run.get().readySteps()); outcomes.put(deployReal, running); runner.maintain(); - assertEquals(Arrays.asList(installReal, deployTester), run.get().readySteps()); + assertEquals(List.of(installReal, deployTester), run.get().readySteps()); outcomes.put(installReal, running); runner.maintain(); - assertEquals(Arrays.asList(deployTester), run.get().readySteps()); + assertEquals(List.of(deployTester), run.get().readySteps()); outcomes.put(deployTester, running); runner.maintain(); - assertEquals(Arrays.asList(installTester), run.get().readySteps()); + assertEquals(List.of(installTester), run.get().readySteps()); outcomes.put(installTester, running); runner.maintain(); - assertEquals(Arrays.asList(startTests), run.get().readySteps()); + assertEquals(List.of(startTests), run.get().readySteps()); outcomes.put(startTests, running); runner.maintain(); - assertEquals(Arrays.asList(endTests), run.get().readySteps()); + assertEquals(List.of(endTests), run.get().readySteps()); // Failure ending tests fails the run, but run-always steps continue. outcomes.put(endTests, testFailure); runner.maintain(); assertTrue(run.get().hasFailed()); - assertEquals(Arrays.asList(deactivateReal, deactivateTester), run.get().readySteps()); + assertEquals(List.of(deactivateReal, deactivateTester), run.get().readySteps()); // Abortion does nothing, as the run has already failed. jobs.abort(run.get().id()); runner.maintain(); - assertEquals(Arrays.asList(deactivateReal, deactivateTester), run.get().readySteps()); + assertEquals(List.of(deactivateReal, deactivateTester), run.get().readySteps()); outcomes.put(deactivateReal, running); outcomes.put(deactivateTester, running); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MaintainerTest.java index d6fd9d88fd9..3aa1f2b5af2 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MaintainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MaintainerTest.java @@ -9,7 +9,6 @@ import org.junit.Test; import java.time.Duration; import java.time.Instant; -import java.util.Arrays; import java.util.EnumSet; import java.util.List; import java.util.concurrent.atomic.AtomicInteger; @@ -38,7 +37,7 @@ public class MaintainerTest { @Test public void staggering() { - List<HostName> cluster = Arrays.asList(HostName.from("cfg1"), HostName.from("cfg2"), HostName.from("cfg3")); + List<HostName> cluster = List.of(HostName.from("cfg1"), HostName.from("cfg2"), HostName.from("cfg3")); Instant now = Instant.ofEpochMilli(1001); Duration interval = Duration.ofMillis(300); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgraderTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgraderTest.java index a21c3bb272d..c406cd346be 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgraderTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgraderTest.java @@ -16,7 +16,6 @@ import org.junit.Before; import org.junit.Test; import java.time.Duration; -import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Optional; @@ -60,12 +59,12 @@ public class OsUpgraderTest { ); // Bootstrap system - tester.configServer().bootstrap(Arrays.asList(zone1, zone2, zone3, zone4, zone5), + tester.configServer().bootstrap(List.of(zone1, zone2, zone3, zone4, zone5), singletonList(SystemApplication.zone), Optional.of(NodeType.host)); // Add system applications that exist in a real system, but are currently not upgraded - tester.configServer().addNodes(Arrays.asList(zone1, zone2, zone3, zone4, zone5), + tester.configServer().addNodes(List.of(zone1, zone2, zone3, zone4, zone5), Collections.singletonList(SystemApplication.configServer), Optional.empty()); @@ -115,7 +114,7 @@ public class OsUpgraderTest { osUpgrader.maintain(); assertWanted(version1, SystemApplication.zone, zone1, zone2, zone3, zone4); statusUpdater.maintain(); - assertTrue("All nodes on target version", tester.controller().osVersionStatus().nodeVersionsIn(cloud).stream() + assertTrue("All nodes on target version", tester.controller().osVersionStatus().nodesIn(cloud).stream() .allMatch(node -> node.version().equals(version1))); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgraderTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgraderTest.java index 8f7b73bc0d1..d832a75eaaa 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgraderTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgraderTest.java @@ -12,7 +12,6 @@ import org.junit.Before; import org.junit.Test; import java.time.Duration; -import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Optional; @@ -50,7 +49,7 @@ public class SystemUpgraderTest { Version version1 = Version.fromString("6.5"); // Bootstrap a system without host applications - tester.configServer().bootstrap(Arrays.asList(zone1, zone2, zone3, zone4), SystemApplication.configServer, + tester.configServer().bootstrap(List.of(zone1, zone2, zone3, zone4), SystemApplication.configServer, SystemApplication.zone); // Fail a few nodes. Failed nodes should not affect versions failNodeIn(zone1, SystemApplication.configServer); @@ -184,7 +183,7 @@ public class SystemUpgraderTest { ); Version version1 = Version.fromString("6.5"); - tester.configServer().bootstrap(Arrays.asList(zone1, zone2, zone3, zone4), SystemApplication.all(), Optional.empty()); + tester.configServer().bootstrap(List.of(zone1, zone2, zone3, zone4), SystemApplication.all(), Optional.empty()); tester.upgradeSystem(version1); systemUpgrader.maintain(); assertCurrentVersion(SystemApplication.all(), version1, zone1, zone2, zone3, zone4); @@ -196,7 +195,7 @@ public class SystemUpgraderTest { // System upgrades in zone 1: systemUpgrader.maintain(); - List<SystemApplication> allExceptZone = Arrays.asList(SystemApplication.configServerHost, + List<SystemApplication> allExceptZone = List.of(SystemApplication.configServerHost, SystemApplication.proxyHost, SystemApplication.configServer); completeUpgrade(allExceptZone, version2, zone1); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java index 25631a48dc4..7953b462ab2 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java @@ -1233,4 +1233,44 @@ public class UpgraderTest { assertFalse(tester.application(application.id()).change().hasTargets()); } + @Test + public void upgradesToLatestAllowedMajor() { + DeploymentTester tester = new DeploymentTester(); + Version version0 = Version.fromString("6.1"); + tester.upgradeSystem(version0); + + // Apps target 6 by default + tester.upgrader().setTargetMajorVersion(Optional.of(6)); + + // All applications deploy on current version + Application app1 = tester.createAndDeploy("app1", 1, "default"); + Application app2 = tester.createAndDeploy("app2", 1, "default"); + + // Keep app 1 on current version + tester.controller().applications().lockIfPresent(app1.id(), app -> tester.controller().applications().store(app.withChange(app.get().change().withPin()))); + + // New version is released + Version version1 = Version.fromString("6.2"); + tester.upgradeSystem(version1); + tester.upgrader().maintain(); + + // App 2 upgrades + tester.completeUpgrade(app2, version1, "default"); + + // New major version is released + Version version2 = Version.fromString("7.1"); + tester.upgradeSystem(version2); + + // App 2 is allowed on new major and upgrades + tester.controller().applications().lockIfPresent(app2.id(), app -> tester.applications().store(app.withMajorVersion(7))); + tester.upgrader().maintain(); + assertEquals(version2, tester.controller().applications().require(app2.id()).change().platform().get()); + + // App 1 is unpinned and upgrades to latest 6 + tester.controller().applications().lockIfPresent(app1.id(), app -> tester.controller().applications().store(app.withChange(app.get().change().withoutPin()))); + tester.upgrader().maintain(); + assertEquals("Application upgrades to latest allowed major", version1, + tester.controller().applications().require(app1.id()).change().platform().get()); + } + } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/BufferedLogStoreTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/BufferedLogStoreTest.java index cf8962cc631..a6022d259be 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/BufferedLogStoreTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/BufferedLogStoreTest.java @@ -2,17 +2,18 @@ package com.yahoo.vespa.hosted.controller.persistence; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.vespa.hosted.controller.api.integration.LogEntry; import com.yahoo.vespa.hosted.controller.api.integration.RunDataStore; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; import com.yahoo.vespa.hosted.controller.api.integration.stubs.MockRunDataStore; -import com.yahoo.vespa.hosted.controller.api.integration.LogEntry; import com.yahoo.vespa.hosted.controller.deployment.RunLog; import com.yahoo.vespa.hosted.controller.deployment.Step; import org.junit.Test; import java.util.Arrays; import java.util.Collections; +import java.util.List; import java.util.Optional; import static org.junit.Assert.assertArrayEquals; @@ -42,23 +43,23 @@ public class BufferedLogStoreTest { assertEquals(RunLog.empty(), logs.readActive(id.application(), id.type(), -1)); logs.append(id.application(), id.type(), Step.deployReal, Collections.singletonList(entry)); - assertEquals(Arrays.asList(entry0), + assertEquals(List.of(entry0), logs.readActive(id.application(), id.type(), -1).get(Step.deployReal)); assertEquals(RunLog.empty(), logs.readActive(id.application(), id.type(), 0)); logs.append(id.application(), id.type(), Step.deployReal, Collections.singletonList(entry)); - assertEquals(Arrays.asList(entry0, entry1), + assertEquals(List.of(entry0, entry1), logs.readActive(id.application(), id.type(), -1).get(Step.deployReal)); - assertEquals(Arrays.asList(entry1), + assertEquals(List.of(entry1), logs.readActive(id.application(), id.type(), 0).get(Step.deployReal)); assertEquals(RunLog.empty(), logs.readActive(id.application(), id.type(), 1)); logs.append(id.application(), id.type(), Step.deployReal, Collections.singletonList(entry)); - assertEquals(Arrays.asList(entry0, entry1, entry2), + assertEquals(List.of(entry0, entry1, entry2), logs.readActive(id.application(), id.type(), -1).get(Step.deployReal)); - assertEquals(Arrays.asList(entry1, entry2), + assertEquals(List.of(entry1, entry2), logs.readActive(id.application(), id.type(), 0).get(Step.deployReal)); - assertEquals(Arrays.asList(entry2), + assertEquals(List.of(entry2), logs.readActive(id.application(), id.type(), 1).get(Step.deployReal)); assertEquals(RunLog.empty(), logs.readActive(id.application(), id.type(), 2)); @@ -72,11 +73,11 @@ public class BufferedLogStoreTest { assertArrayEquals(new long[]{}, buffer.getLogChunkIds(id.application(), id.type()).toArray()); assertEquals(RunLog.empty(), logs.readActive(id.application(), id.type(), -1)); - assertEquals(Arrays.asList(entry0, entry1, entry2), + assertEquals(List.of(entry0, entry1, entry2), logs.readFinished(id, -1).get().get(Step.deployReal)); - assertEquals(Arrays.asList(entry1, entry2), + assertEquals(List.of(entry1, entry2), logs.readFinished(id, 0).get().get(Step.deployReal)); - assertEquals(Arrays.asList(entry2), + assertEquals(List.of(entry2), logs.readFinished(id, 1).get().get(Step.deployReal)); assertEquals(Collections.emptyList(), logs.readFinished(id, 2).get().get(Step.deployReal)); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/LogSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/LogSerializerTest.java index c3e215cb792..a53a026eff6 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/LogSerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/LogSerializerTest.java @@ -10,7 +10,6 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.util.ArrayList; -import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -58,7 +57,7 @@ public class LogSerializerTest { expected.get(deployTester).add(second); expected.get(deployTester).add(fourth); - assertEquals(expected, serializer.fromJson(Arrays.asList(logJson, logJson), -1)); + assertEquals(expected, serializer.fromJson(List.of(logJson, logJson), -1)); } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/OsVersionStatusSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/OsVersionStatusSerializerTest.java index 9840bec0f9a..6e2298fd64e 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/OsVersionStatusSerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/OsVersionStatusSerializerTest.java @@ -10,7 +10,6 @@ import com.yahoo.vespa.hosted.controller.versions.OsVersion; import com.yahoo.vespa.hosted.controller.versions.OsVersionStatus; import org.junit.Test; -import java.util.Arrays; import java.util.List; import java.util.Map; import java.util.TreeMap; @@ -28,11 +27,11 @@ public class OsVersionStatusSerializerTest { Version version2 = Version.fromString("7.2"); Map<OsVersion, List<OsVersionStatus.Node>> versions = new TreeMap<>(); - versions.put(new OsVersion(version1, CloudName.defaultName()), Arrays.asList( + versions.put(new OsVersion(version1, CloudName.defaultName()), List.of( new OsVersionStatus.Node(HostName.from("node1"), version1, Environment.prod, RegionName.from("us-west")), new OsVersionStatus.Node(HostName.from("node2"), version1, Environment.prod, RegionName.from("us-east")) )); - versions.put(new OsVersion(version2, CloudName.defaultName()), Arrays.asList( + versions.put(new OsVersion(version2, CloudName.defaultName()), List.of( new OsVersionStatus.Node(HostName.from("node3"), version2, Environment.prod, RegionName.from("us-west")), new OsVersionStatus.Node(HostName.from("node4"), version2, Environment.prod, RegionName.from("us-east")) diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializerTest.java index f3790e6d291..2e0d7715d7d 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializerTest.java @@ -5,14 +5,14 @@ import com.yahoo.config.provision.TenantName; import com.yahoo.vespa.athenz.api.AthenzDomain; import com.yahoo.vespa.hosted.controller.api.identifiers.Property; import com.yahoo.vespa.hosted.controller.api.identifiers.PropertyId; -import com.yahoo.vespa.hosted.controller.tenant.AthenzTenant; import com.yahoo.vespa.hosted.controller.api.integration.organization.Contact; +import com.yahoo.vespa.hosted.controller.tenant.AthenzTenant; import com.yahoo.vespa.hosted.controller.tenant.UserTenant; import org.junit.Test; import java.net.URI; -import java.util.Arrays; import java.util.Collections; +import java.util.List; import java.util.Optional; import static org.junit.Assert.assertEquals; @@ -75,7 +75,7 @@ public class TenantSerializerTest { URI.create("http://contact1.test"), URI.create("http://property1.test"), URI.create("http://issue-tracker-1.test"), - Arrays.asList( + List.of( Collections.singletonList("person1"), Collections.singletonList("person2") ), diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializerTest.java index 87e145f9f93..5e6f9811376 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializerTest.java @@ -30,10 +30,10 @@ public class VersionStatusSerializerTest { DeploymentStatistics statistics = new DeploymentStatistics( Version.fromString("5.0"), Collections.singletonList(ApplicationId.from("tenant1", "failing1", "default")), - Arrays.asList(ApplicationId.from("tenant2", "success1", "default"), - ApplicationId.from("tenant2", "success2", "default")), - Arrays.asList(ApplicationId.from("tenant1", "failing1", "default"), - ApplicationId.from("tenant2", "success2", "default")) + List.of(ApplicationId.from("tenant2", "success1", "default"), + ApplicationId.from("tenant2", "success2", "default")), + List.of(ApplicationId.from("tenant1", "failing1", "default"), + ApplicationId.from("tenant2", "success2", "default")) ); vespaVersions.add(new VespaVersion(statistics, "dead", Instant.now(), false, false, asHostnames("cfg1", "cfg2", "cfg3"), VespaVersion.Confidence.normal)); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java index 38dd6877703..cd804e2ebfb 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java @@ -71,7 +71,6 @@ import java.net.URI; import java.nio.charset.StandardCharsets; import java.time.Instant; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -1485,13 +1484,13 @@ public class ApplicationApiTest extends ControllerContainerTest { } private void updateContactInformation() { - Contact contact = new Contact(URI.create("www.contacts.tld/1234"), URI.create("www.properties.tld/1234"), URI.create("www.issues.tld/1234"), Arrays.asList(Arrays.asList("alice"), Arrays.asList("bob")), "queue", Optional.empty()); + Contact contact = new Contact(URI.create("www.contacts.tld/1234"), URI.create("www.properties.tld/1234"), URI.create("www.issues.tld/1234"), List.of(List.of("alice"), List.of("bob")), "queue", Optional.empty()); tester.controller().tenants().lockIfPresent(TenantName.from("tenant2"), lockedTenant -> tester.controller().tenants().store(lockedTenant.with(contact))); } private void registerContact(long propertyId) { PropertyId p = new PropertyId(String.valueOf(propertyId)); - contactRetriever().addContact(p, new Contact(URI.create("www.issues.tld/" + p.id()), URI.create("www.contacts.tld/" + p.id()), URI.create("www.properties.tld/" + p.id()), Arrays.asList(Collections.singletonList("alice"), + contactRetriever().addContact(p, new Contact(URI.create("www.issues.tld/" + p.id()), URI.create("www.contacts.tld/" + p.id()), URI.create("www.properties.tld/" + p.id()), List.of(Collections.singletonList("alice"), Collections.singletonList("bob")), "queue", Optional.empty())); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/ControllerAuthorizationFilterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/ControllerAuthorizationFilterTest.java index 19aa247edb4..9b7c36c49e1 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/ControllerAuthorizationFilterTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/ControllerAuthorizationFilterTest.java @@ -31,7 +31,6 @@ import static com.yahoo.jdisc.http.HttpRequest.Method.DELETE; import static com.yahoo.jdisc.http.HttpRequest.Method.POST; import static com.yahoo.jdisc.http.HttpRequest.Method.PUT; import static com.yahoo.jdisc.http.HttpResponse.Status.FORBIDDEN; -import static java.util.Arrays.asList; import static java.util.Collections.singleton; import static java.util.Collections.singletonList; import static org.junit.Assert.assertEquals; @@ -87,7 +86,7 @@ public class ControllerAuthorizationFilterTest { ControllerAuthorizationFilter filter = createFilter(controllerTester); - List<AthenzIdentity> allowed = asList(HOSTED_OPERATOR, TENANT_ADMIN); + List<AthenzIdentity> allowed = List.of(HOSTED_OPERATOR, TENANT_ADMIN); List<AthenzIdentity> forbidden = singletonList(USER); testApiAccess(DELETE, "/application/v4/tenant/mytenant", @@ -110,8 +109,8 @@ public class ControllerAuthorizationFilterTest { ControllerAuthorizationFilter filter = createFilter(controllerTester); - List<AthenzIdentity> allowed = asList(HOSTED_OPERATOR, TENANT_PIPELINE); - List<AthenzIdentity> forbidden = asList(TENANT_ADMIN, USER); + List<AthenzIdentity> allowed = List.of(HOSTED_OPERATOR, TENANT_PIPELINE); + List<AthenzIdentity> forbidden = List.of(TENANT_ADMIN, USER); testApiAccess(POST, "/application/v4/tenant/mytenant/application/myapp/environment/prod/region/myregion/instance/default/deploy", allowed, forbidden, filter); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/zone/v1/ZoneApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/zone/v1/ZoneApiTest.java index 4fbdbdcc813..a7832ccbc1f 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/zone/v1/ZoneApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/zone/v1/ZoneApiTest.java @@ -3,15 +3,14 @@ package com.yahoo.vespa.hosted.controller.restapi.zone.v1; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.RegionName; -import com.yahoo.vespa.hosted.controller.integration.ZoneRegistryMock; import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; +import com.yahoo.vespa.hosted.controller.integration.ZoneRegistryMock; import com.yahoo.vespa.hosted.controller.restapi.ContainerControllerTester; import com.yahoo.vespa.hosted.controller.restapi.ControllerContainerTest; import org.junit.Before; import org.junit.Test; import java.io.File; -import java.util.Arrays; import java.util.List; /** @@ -20,7 +19,7 @@ import java.util.List; public class ZoneApiTest extends ControllerContainerTest { private static final String responseFiles = "src/test/java/com/yahoo/vespa/hosted/controller/restapi/zone/v1/responses/"; - private static final List<ZoneId> zones = Arrays.asList( + private static final List<ZoneId> zones = List.of( ZoneId.from(Environment.prod, RegionName.from("us-north-1")), ZoneId.from(Environment.dev, RegionName.from("us-north-2")), ZoneId.from(Environment.test, RegionName.from("us-north-3")), @@ -39,7 +38,7 @@ public class ZoneApiTest extends ControllerContainerTest { } @Test - public void test_requests() throws Exception { + public void test_requests() { // GET /zone/v1 tester.containerTester().assertResponse(authenticatedRequest("http://localhost:8080/zone/v1"), new File("root.json")); @@ -54,7 +53,7 @@ public class ZoneApiTest extends ControllerContainerTest { } @Test - public void test_invalid_requests() throws Exception { + public void test_invalid_requests() { // GET /zone/v1/environment/prod/default: No default region tester.containerTester().assertResponse(authenticatedRequest("http://localhost:8080/zone/v1/environment/prod/default"), new File("no-default-region.json"), diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/ZoneApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/ZoneApiTest.java index 2123d32953d..f62f6e21910 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/ZoneApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/ZoneApiTest.java @@ -8,16 +8,15 @@ import com.yahoo.config.provision.RegionName; import com.yahoo.text.Utf8; import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.athenz.api.AthenzUser; +import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; import com.yahoo.vespa.hosted.controller.integration.ConfigServerProxyMock; import com.yahoo.vespa.hosted.controller.integration.ZoneRegistryMock; -import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; import com.yahoo.vespa.hosted.controller.restapi.ContainerControllerTester; import com.yahoo.vespa.hosted.controller.restapi.ControllerContainerTest; import org.junit.Before; import org.junit.Test; import java.io.File; -import java.util.Arrays; import java.util.List; import static org.junit.Assert.assertEquals; @@ -30,7 +29,7 @@ public class ZoneApiTest extends ControllerContainerTest { private static final AthenzIdentity HOSTED_VESPA_OPERATOR = AthenzUser.fromUserId("johnoperator"); private static final String responseFiles = "src/test/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/responses/"; - private static final List<ZoneId> zones = Arrays.asList( + private static final List<ZoneId> zones = List.of( ZoneId.from(Environment.prod, RegionName.from("us-north-1")), ZoneId.from(Environment.dev, RegionName.from("us-north-2")), ZoneId.from(Environment.test, RegionName.from("us-north-3")), @@ -52,7 +51,7 @@ public class ZoneApiTest extends ControllerContainerTest { } @Test - public void test_requests() throws Exception { + public void test_requests() { // GET /zone/v2 tester.containerTester().assertResponse(authenticatedRequest("http://localhost:8080/zone/v2"), new File("root.json")); @@ -111,7 +110,7 @@ public class ZoneApiTest extends ControllerContainerTest { } @Test - public void test_invalid_requests() throws Exception { + public void test_invalid_requests() { // POST /zone/v2/prod/us-north-34/nodes/v2 tester.containerTester().assertResponse(hostedOperatorRequest("http://localhost:8080/zone/v2/prod/us-north-42/nodes/v2", new byte[0], Method.POST), |