diff options
Diffstat (limited to 'node-repository/src')
10 files changed, 131 insertions, 30 deletions
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java index 52b68708eee..539f3128091 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java @@ -137,7 +137,7 @@ public class NodeRepository extends AbstractComponent implements HealthCheckerPr this.resourcesCalculator = provisionServiceProvider.getHostResourcesCalculator(); this.nodeResourceLimits = new NodeResourceLimits(this); this.nameResolver = nameResolver; - this.osVersions = new OsVersions(this); + this.osVersions = new OsVersions(this, provisionServiceProvider.getHostProvisioner()); this.infrastructureVersions = new InfrastructureVersions(db); this.firmwareChecks = new FirmwareChecks(db, clock); this.containerImages = new ContainerImages(containerImage, tenantContainerImage, tenantGpuContainerImage); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/DelegatingOsUpgrader.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/DelegatingOsUpgrader.java index c2d3f511711..5d8296d6f9d 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/DelegatingOsUpgrader.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/DelegatingOsUpgrader.java @@ -35,7 +35,7 @@ public class DelegatingOsUpgrader extends OsUpgrader { // This upgrader cannot downgrade nodes. We therefore consider only nodes // on a lower version than the target .osVersionIsBefore(target.version()) - .matching(node -> canUpgradeAt(now, node)) + .matching(node -> canUpgradeTo(target.version(), now, node)) .byIncreasingOsVersion() .first(upgradeSlots(target, activeNodes)); if (nodesToUpgrade.size() == 0) return; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/OsUpgrader.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/OsUpgrader.java index 436181f99ba..f56e75518a3 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/OsUpgrader.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/OsUpgrader.java @@ -1,6 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.os; +import com.yahoo.component.Version; import com.yahoo.config.provision.NodeType; import com.yahoo.vespa.flags.IntFlag; import com.yahoo.vespa.flags.PermanentFlags; @@ -10,20 +11,27 @@ import com.yahoo.vespa.hosted.provision.NodeRepository; import java.time.Duration; import java.time.Instant; +import java.util.Objects; +import java.util.Set; +import java.util.logging.Level; +import java.util.logging.Logger; /** - * Interface for an OS upgrader. + * Interface for an OS upgrader. Instances of this are created on-demand because multiple implementations may be used + * within a single zone. This and subclasses should not have any state. * * @author mpolden */ public abstract class OsUpgrader { + private final Logger LOG = Logger.getLogger(OsUpgrader.class.getName()); + private final IntFlag maxActiveUpgrades; final NodeRepository nodeRepository; public OsUpgrader(NodeRepository nodeRepository) { - this.nodeRepository = nodeRepository; + this.nodeRepository = Objects.requireNonNull(nodeRepository); this.maxActiveUpgrades = PermanentFlags.MAX_OS_UPGRADES.bindTo(nodeRepository.flagSource()); } @@ -43,10 +51,23 @@ public abstract class OsUpgrader { return Math.max(0, max - upgrading); } - /** Returns whether node can change version at given instant */ - final boolean canUpgradeAt(Instant instant, Node node) { - return node.status().osVersion().downgrading() || // Fast-track downgrades - node.history().age(instant).compareTo(gracePeriod()) > 0; + /** Returns whether node can upgrade to version at given instant */ + final boolean canUpgradeTo(Version version, Instant instant, Node node) { + if (deferringUpgrade(node, instant)) return false; + Set<Version> versions = nodeRepository.osVersions().availableTo(node, version); + boolean versionAvailable = versions.contains(version); + if (!versionAvailable) { + LOG.log(Level.WARNING, "Want to upgrade host " + node.hostname() + " to OS version " + + version.toFullString() + ", but this version does not exist in " + + node.cloudAccount() + ". Found " + versions.stream().sorted().toList()); + } + return versionAvailable; + } + + /** Returns whether node is deferring upgrade at given instant */ + final boolean deferringUpgrade(Node node, Instant instant) { + return !node.status().osVersion().downgrading() && // Never defer downgrades + node.history().age(instant).compareTo(gracePeriod()) <= 0; } /** The duration this leaves new nodes alone before scheduling any upgrade */ diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/OsVersions.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/OsVersions.java index daed86dc2ab..f5706d3b8c9 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/OsVersions.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/OsVersions.java @@ -1,8 +1,11 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.os; +import com.google.common.cache.Cache; +import com.google.common.cache.CacheBuilder; import com.yahoo.component.Version; import com.yahoo.config.provision.Cloud; +import com.yahoo.config.provision.CloudAccount; import com.yahoo.config.provision.CloudName; import com.yahoo.config.provision.NodeType; import com.yahoo.vespa.curator.Lock; @@ -10,11 +13,17 @@ import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.node.Status; import com.yahoo.vespa.hosted.provision.persistence.CuratorDb; +import com.yahoo.vespa.hosted.provision.provisioning.HostProvisioner; +import com.yahoo.yolean.Exceptions; import java.util.List; import java.util.Objects; import java.util.Optional; +import java.util.Set; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; import java.util.function.UnaryOperator; +import java.util.logging.Level; import java.util.logging.Logger; /** @@ -30,20 +39,27 @@ import java.util.logging.Logger; */ public class OsVersions { - private static final Logger log = Logger.getLogger(OsVersions.class.getName()); + private static final Logger LOG = Logger.getLogger(OsVersions.class.getName()); private final NodeRepository nodeRepository; private final CuratorDb db; private final Cloud cloud; - - public OsVersions(NodeRepository nodeRepository) { - this(nodeRepository, nodeRepository.zone().cloud()); + private final Optional<HostProvisioner> hostProvisioner; + // Version is queried for each host to upgrade, so we cache the results for a while to avoid excessive + // API calls to the host provisioner + private final Cache<CloudAccount, Set<Version>> availableVersions = CacheBuilder.newBuilder() + .expireAfterWrite(10, TimeUnit.MINUTES) + .build(); + + public OsVersions(NodeRepository nodeRepository, Optional<HostProvisioner> hostProvisioner) { + this(nodeRepository, nodeRepository.zone().cloud(), hostProvisioner); } - OsVersions(NodeRepository nodeRepository, Cloud cloud) { + OsVersions(NodeRepository nodeRepository, Cloud cloud, Optional<HostProvisioner> hostProvisioner) { this.nodeRepository = Objects.requireNonNull(nodeRepository); this.db = nodeRepository.database(); this.cloud = Objects.requireNonNull(cloud); + this.hostProvisioner = Objects.requireNonNull(hostProvisioner); // Read and write all versions to make sure they are stored in the latest version of the serialized format try (var lock = db.lockOsVersionChange()) { @@ -104,11 +120,30 @@ public class OsVersions { + currentTarget.get().version().toFullString()); } - log.info("Set OS target version for " + nodeType + " nodes to " + version.toFullString()); + LOG.info("Set OS target version for " + nodeType + " nodes to " + version.toFullString()); return change.withTarget(version, nodeType); }); } + /** Returns the versions available to given host */ + public Set<Version> availableTo(Node host, Version requestedVersion) { + if (hostProvisioner.isEmpty()) { + return Set.of(requestedVersion); + } + try { + return availableVersions.get(host.cloudAccount(), + () -> hostProvisioner.get().osVersions(host, requestedVersion.getMajor())); + } catch (ExecutionException e) { + LOG.log(Level.WARNING, "Failed to list supported OS versions in " + host.cloudAccount() + ": " + Exceptions.toMessageString(e)); + return Set.of(); + } + } + + /** Invalidate cached versions. For testing purposes */ + void invalidate() { + availableVersions.invalidateAll(); + } + /** Resume or halt upgrade of given node type */ public void resumeUpgradeOf(NodeType nodeType, boolean resume) { require(nodeType); @@ -124,9 +159,9 @@ public class OsVersions { } } - /** Returns whether node can be upgraded now */ - public boolean canUpgrade(Node node) { - return chooseUpgrader(node.type(), Optional.empty()).canUpgradeAt(nodeRepository.clock().instant(), node); + /** Returns whether node is currently deferring its upgrade */ + public boolean deferringUpgrade(Node node) { + return chooseUpgrader(node.type(), Optional.empty()).deferringUpgrade(node, nodeRepository.clock().instant()); } /** Returns the upgrader to use when upgrading given node type to target */ diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/RebuildingOsUpgrader.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/RebuildingOsUpgrader.java index 108093d8379..a5565a6accb 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/RebuildingOsUpgrader.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/RebuildingOsUpgrader.java @@ -71,7 +71,7 @@ public class RebuildingOsUpgrader extends OsUpgrader { List<Node> hostsToRebuild = new ArrayList<>(rebuildLimit); NodeList candidates = hosts.not().rebuilding(softRebuild) .not().onOsVersion(target.version()) - .matching(node -> canUpgradeAt(now, node)) + .matching(node -> canUpgradeTo(target.version(), now, node)) .byIncreasingOsVersion(); for (Node host : candidates) { if (hostsToRebuild.size() == rebuildLimit) break; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/RetiringOsUpgrader.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/RetiringOsUpgrader.java index c1e8f2b6fa4..cb6c7683f23 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/RetiringOsUpgrader.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/RetiringOsUpgrader.java @@ -54,7 +54,7 @@ public class RetiringOsUpgrader extends OsUpgrader { } return nodes.not().deprovisioning() .not().onOsVersion(target.version()) - .matching(node -> canUpgradeAt(instant, node)) + .matching(node -> canUpgradeTo(target.version(), instant, node)) .byIncreasingOsVersion() .first(upgradeSlots(target, nodes.deprovisioning())); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostProvisioner.java index 8ef4b6c8bd1..4214f543c60 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostProvisioner.java @@ -1,6 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.provisioning; +import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.CloudAccount; import com.yahoo.config.provision.HostEvent; @@ -11,6 +12,7 @@ import com.yahoo.vespa.hosted.provision.Node; import java.util.Collection; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.function.Consumer; import java.util.function.Predicate; @@ -94,6 +96,9 @@ public interface HostProvisioner { /** Returns whether flavor for given host can be upgraded to a newer generation */ boolean canUpgradeFlavor(Node host, Node child, Predicate<NodeResources> realHostResourcesWithinLimits); + /** Returns all OS versions available to host for the given major version */ + Set<Version> osVersions(Node host, int majorVersion); + /** Updates the given hosts to indicate that they are allocated to the given application. */ default void updateAllocation(Collection<Node> hosts, ApplicationId owner) { } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesResponse.java index 5c379fb1608..b8c841771f5 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesResponse.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesResponse.java @@ -167,7 +167,7 @@ class NodesResponse extends SlimeJsonResponse { node.status().osVersion().current().ifPresent(version -> object.setString("currentOsVersion", version.toFullString())); node.status().osVersion().wanted().ifPresent(version -> object.setString("wantedOsVersion", version.toFullString())); if (node.type().isHost()) { - object.setBool("deferOsUpgrade", !nodeRepository.osVersions().canUpgrade(node)); + object.setBool("deferOsUpgrade", nodeRepository.osVersions().deferringUpgrade(node)); } node.status().firmwareVerifiedAt().ifPresent(instant -> object.setLong("currentFirmwareCheck", instant.toEpochMilli())); if (node.type().isHost()) diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockHostProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockHostProvisioner.java index b5bb91af71a..73985075319 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockHostProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockHostProvisioner.java @@ -1,6 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.testutils; +import com.yahoo.component.Version; import com.yahoo.config.provision.CloudAccount; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Flavor; @@ -32,6 +33,7 @@ import java.util.Optional; import java.util.Set; import java.util.function.Consumer; import java.util.function.Predicate; +import java.util.stream.Collectors; import java.util.stream.IntStream; import static com.yahoo.config.provision.NodeType.host; @@ -50,6 +52,7 @@ public class MockHostProvisioner implements HostProvisioner { private final Map<ClusterSpec.Type, Flavor> hostFlavors = new HashMap<>(); private final Set<String> upgradableFlavors = new HashSet<>(); private final Map<Behaviour, Integer> behaviours = new HashMap<>(); + private final Set<Version> osVersions = new HashSet<>(); private int deprovisionedHosts = 0; @@ -146,6 +149,11 @@ public class MockHostProvisioner implements HostProvisioner { return upgradableFlavors.contains(host.flavor().name()); } + @Override + public Set<Version> osVersions(Node host, int majorVersion) { + return osVersions.stream().filter(v -> v.getMajor() == majorVersion).collect(Collectors.toUnmodifiableSet()); + } + /** Returns the hosts that have been provisioned by this */ public List<ProvisionedHost> provisionedHosts() { return Collections.unmodifiableList(provisionedHosts); @@ -214,6 +222,11 @@ public class MockHostProvisioner implements HostProvisioner { return this; } + public MockHostProvisioner addOsVersion(Version version) { + osVersions.add(version); + return this; + } + public boolean compatible(Flavor flavor, NodeResources resources) { NodeResources resourcesToVerify = resources.withMemoryGb(resources.memoryGb() - memoryTaxGb); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/os/OsVersionsTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/os/OsVersionsTest.java index 0be90dcb888..dcbac44a37f 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/os/OsVersionsTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/os/OsVersionsTest.java @@ -18,6 +18,7 @@ import com.yahoo.vespa.hosted.provision.node.Allocation; import com.yahoo.vespa.hosted.provision.node.OsVersion; import com.yahoo.vespa.hosted.provision.node.Status; import com.yahoo.vespa.hosted.provision.provisioning.ProvisioningTester; +import com.yahoo.vespa.hosted.provision.testutils.MockHostProvisioner; import org.junit.Test; import java.time.Duration; @@ -41,7 +42,7 @@ public class OsVersionsTest { @Test public void upgrade() { - var versions = new OsVersions(tester.nodeRepository()); + var versions = new OsVersions(tester.nodeRepository(), Optional.ofNullable(tester.hostProvisioner())); provisionInfraApplication(10); Supplier<NodeList> hostNodes = () -> tester.nodeRepository().nodes().list().nodeType(NodeType.host); @@ -94,7 +95,7 @@ public class OsVersionsTest { public void max_active_upgrades() { int totalNodes = 20; int maxActiveUpgrades = 5; - var versions = new OsVersions(tester.nodeRepository(), Cloud.defaultCloud()); + var versions = new OsVersions(tester.nodeRepository(), Cloud.defaultCloud(), Optional.ofNullable(tester.hostProvisioner())); setMaxActiveUpgrades(maxActiveUpgrades); provisionInfraApplication(totalNodes); Supplier<NodeList> hostNodes = () -> tester.nodeRepository().nodes().list().state(Node.State.active).hosts(); @@ -140,7 +141,7 @@ public class OsVersionsTest { @Test public void newer_upgrade_aborts_upgrade_to_stale_version() { - var versions = new OsVersions(tester.nodeRepository()); + var versions = new OsVersions(tester.nodeRepository(), Optional.ofNullable(tester.hostProvisioner())); provisionInfraApplication(10); Supplier<NodeList> hostNodes = () -> tester.nodeRepository().nodes().list().hosts(); @@ -160,7 +161,7 @@ public class OsVersionsTest { @Test public void upgrade_and_downgrade_by_retiring() { int maxActiveUpgrades = 2; - var versions = new OsVersions(tester.nodeRepository(), Cloud.builder().dynamicProvisioning(true).build()); + var versions = new OsVersions(tester.nodeRepository(), Cloud.builder().dynamicProvisioning(true).build(), Optional.ofNullable(tester.hostProvisioner())); setMaxActiveUpgrades(maxActiveUpgrades); int hostCount = 10; // Provision hosts and children @@ -229,7 +230,7 @@ public class OsVersionsTest { @Test public void upgrade_by_retiring_everything_at_once() { - var versions = new OsVersions(tester.nodeRepository(), Cloud.builder().dynamicProvisioning(true).build()); + var versions = new OsVersions(tester.nodeRepository(), Cloud.builder().dynamicProvisioning(true).build(), Optional.ofNullable(tester.hostProvisioner())); setMaxActiveUpgrades(Integer.MAX_VALUE); int hostCount = 3; provisionInfraApplication(hostCount, NodeType.host); @@ -253,7 +254,7 @@ public class OsVersionsTest { @Test public void upgrade_by_rebuilding() { - var versions = new OsVersions(tester.nodeRepository(), Cloud.defaultCloud()); + var versions = new OsVersions(tester.nodeRepository(), Cloud.defaultCloud(), Optional.ofNullable(tester.hostProvisioner())); setMaxActiveUpgrades(1); int hostCount = 10; provisionInfraApplication(hostCount + 1); @@ -337,7 +338,8 @@ public class OsVersionsTest { .dynamicProvisioning(true) .name(CloudName.AWS) .account(CloudAccount.from("000000000000")) - .build()); + .build(), + Optional.ofNullable(tester.hostProvisioner())); provisionInfraApplication(hostCount, NodeType.host, NodeResources.StorageType.remote, NodeResources.Architecture.x86_64); Supplier<NodeList> hostNodes = () -> tester.nodeRepository().nodes().list().nodeType(NodeType.host); @@ -382,7 +384,7 @@ public class OsVersionsTest { @Test public void upgrade_by_rebuilding_multiple_host_types() { setMaxActiveUpgrades(1); - var versions = new OsVersions(tester.nodeRepository(), Cloud.defaultCloud()); + var versions = new OsVersions(tester.nodeRepository(), Cloud.defaultCloud(), Optional.ofNullable(tester.hostProvisioner())); int hostCount = 3; provisionInfraApplication(hostCount, NodeType.host); provisionInfraApplication(hostCount, NodeType.confighost); @@ -415,7 +417,7 @@ public class OsVersionsTest { @Test public void upgrade_by_rebuilding_is_limited_by_stateful_clusters() { setMaxActiveUpgrades(3); - var versions = new OsVersions(tester.nodeRepository(), Cloud.defaultCloud()); + var versions = new OsVersions(tester.nodeRepository(), Cloud.defaultCloud(), Optional.ofNullable(tester.hostProvisioner())); int hostCount = 5; ApplicationId app1 = ApplicationId.from("t1", "a1", "i1"); ApplicationId app2 = ApplicationId.from("t2", "a2", "i2"); @@ -493,7 +495,7 @@ public class OsVersionsTest { public void upgrade_by_rebuilding_limits_infrastructure_host() { int hostCount = 3; setMaxActiveUpgrades(hostCount); - var versions = new OsVersions(tester.nodeRepository(), Cloud.defaultCloud()); + var versions = new OsVersions(tester.nodeRepository(), Cloud.defaultCloud(), Optional.ofNullable(tester.hostProvisioner())); provisionInfraApplication(hostCount, NodeType.proxyhost); Supplier<NodeList> hosts = () -> tester.nodeRepository().nodes().list().nodeType(NodeType.proxyhost); @@ -515,6 +517,31 @@ public class OsVersionsTest { } } + @Test + public void skips_unavailable_version() { + MockHostProvisioner hostProvisioner = new MockHostProvisioner(List.of()); + ProvisioningTester tester = new ProvisioningTester.Builder().dynamicProvisioning(true, false).hostProvisioner(hostProvisioner).build(); + OsVersions versions = tester.nodeRepository().osVersions(); + tester.makeReadyHosts(1, new NodeResources(2,4,8,100)); + tester.activateTenantHosts(); + Supplier<Node> host = () -> tester.nodeRepository().nodes().list().nodeType(NodeType.host).first().get(); + tester.clock().advance(Duration.ofDays(1)); + + hostProvisioner.addOsVersion(Version.fromString("7.0")); + Version version0 = Version.fromString("8.0"); + versions.setTarget(NodeType.host, version0, false); + versions.resumeUpgradeOf(NodeType.host, true); + assertTrue("Upgrade is not triggered to unavailable version", host.get().status().osVersion().wanted().isEmpty()); + + // Version becomes available, but is not used until cache expires + hostProvisioner.addOsVersion(version0); + versions.resumeUpgradeOf(NodeType.host, true); + assertTrue(host.get().status().osVersion().wanted().isEmpty()); + versions.invalidate(); + versions.resumeUpgradeOf(NodeType.host, true); + assertEquals("Host upgrade is triggered", version0, host.get().status().osVersion().wanted().get()); + } + private void setMaxActiveUpgrades(int max) { tester.flagSource().withIntFlag(PermanentFlags.MAX_OS_UPGRADES.id(), max); } |