diff options
author | Martin Polden <mpolden@mpolden.no> | 2019-01-10 11:18:18 +0100 |
---|---|---|
committer | Martin Polden <mpolden@mpolden.no> | 2019-01-10 11:18:18 +0100 |
commit | c4dbbd028d77f6425acb1f0ab47916c0ee5f5b39 (patch) | |
tree | b33a50cefe610fc96233f35ef20ec8f70f200b18 /node-repository/src | |
parent | a9c77576da201d3da009e8f86800d23070284a41 (diff) |
Expire inactive load balancers
Diffstat (limited to 'node-repository/src')
8 files changed, 220 insertions, 29 deletions
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancer.java index effc5b1a41d..4ac3a839ae1 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancer.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.hosted.provision.lb; import com.google.common.collect.ImmutableList; import com.google.common.collect.Ordering; import com.yahoo.config.provision.HostName; +import com.yahoo.vespa.hosted.provision.maintenance.LoadBalancerExpirer; import java.util.List; import java.util.Objects; @@ -50,8 +51,8 @@ public class LoadBalancer { } /** - * Returns whether this load balancer is inactive. Inactive load balancers cannot be reactivated, and are - * eventually deleted + * Returns whether this load balancer is inactive. Inactive load balancers cannot be re-activated, and are + * eventually removed by {@link LoadBalancerExpirer}. */ public boolean inactive() { return inactive; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerServiceMock.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerServiceMock.java index b589e5aed2f..010b983ca12 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerServiceMock.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerServiceMock.java @@ -17,6 +17,10 @@ public class LoadBalancerServiceMock implements LoadBalancerService { private final Map<LoadBalancerId, LoadBalancer> loadBalancers = new HashMap<>(); + public Map<LoadBalancerId, LoadBalancer> loadBalancers() { + return loadBalancers; + } + @Override public Protocol protocol() { return Protocol.ipv4; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirer.java new file mode 100644 index 00000000000..4b66dff3032 --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirer.java @@ -0,0 +1,83 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.maintenance; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.log.LogLevel; +import com.yahoo.vespa.curator.Lock; +import com.yahoo.vespa.hosted.provision.NodeRepository; +import com.yahoo.vespa.hosted.provision.lb.LoadBalancerId; +import com.yahoo.vespa.hosted.provision.lb.LoadBalancerService; +import com.yahoo.vespa.hosted.provision.persistence.CuratorDatabaseClient; + +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.stream.Collectors; + +/** + * Periodically remove inactive load balancers permanently. + * + * When an application is removed, any associated load balancers are only deactivated. This maintainer ensures that + * such resources are eventually freed. + * + * @author mpolden + */ +public class LoadBalancerExpirer extends Maintainer { + + private final LoadBalancerService service; + private final CuratorDatabaseClient db; + + public LoadBalancerExpirer(NodeRepository nodeRepository, Duration interval, JobControl jobControl, + LoadBalancerService service) { + super(nodeRepository, interval, jobControl); + this.service = Objects.requireNonNull(service, "service must be non-null"); + this.db = nodeRepository.database(); + } + + @Override + protected void maintain() { + removeInactive(); + } + + private void removeInactive() { + List<LoadBalancerId> failed = new ArrayList<>(); + Exception lastException = null; + try (Lock lock = db.lockLoadBalancers()) { + for (LoadBalancerId loadBalancer : inactiveLoadBlancers()) { + if (hasNodes(loadBalancer.application())) { // Defer removal if there are still nodes allocated to application + continue; + } + try { + service.remove(loadBalancer); + db.removeLoadBalancer(loadBalancer); + } catch (Exception e) { + failed.add(loadBalancer); + lastException = e; + } + } + } + if (!failed.isEmpty()) { + log.log(LogLevel.WARNING, String.format("Failed to remove %d load balancers: %s, retrying in %s", + failed.size(), + failed.stream() + .map(LoadBalancerId::serializedForm) + .collect(Collectors.joining(", ")), + interval()), + lastException); + } + } + + private boolean hasNodes(ApplicationId application) { + return !nodeRepository().getNodes(application).isEmpty(); + } + + private List<LoadBalancerId> inactiveLoadBlancers() { + return db.readLoadBalancers().entrySet().stream() + .filter(entry -> entry.getValue().inactive()) + .map(Map.Entry::getKey) + .collect(Collectors.toList()); + } + +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Maintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Maintainer.java index f5576ae00fc..49ede9962eb 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Maintainer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Maintainer.java @@ -56,8 +56,7 @@ public abstract class Maintainer extends AbstractComponent implements Runnable { try { if (jobControl.isActive(name())) maintain(); - } - catch (RuntimeException e) { + } catch (Throwable e) { log.log(Level.WARNING, this + " failed. Will retry in " + interval.toMinutes() + " minutes", e); } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java index 946c43ca8fc..2bc60de3c8d 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java @@ -12,6 +12,7 @@ import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.Zone; import com.yahoo.jdisc.Metric; import com.yahoo.vespa.hosted.provision.NodeRepository; +import com.yahoo.vespa.hosted.provision.lb.LoadBalancerService; import com.yahoo.vespa.hosted.provision.maintenance.retire.RetireIPv4OnlyNodes; import com.yahoo.vespa.hosted.provision.maintenance.retire.RetirementPolicy; import com.yahoo.vespa.hosted.provision.maintenance.retire.RetirementPolicyList; @@ -50,6 +51,7 @@ public class NodeRepositoryMaintenance extends AbstractComponent { private final NodeRetirer nodeRetirer; private final MetricsReporter metricsReporter; private final InfrastructureProvisioner infrastructureProvisioner; + private final LoadBalancerExpirer loadBalancerExpirer; private final JobControl jobControl; private final InfrastructureVersions infrastructureVersions; @@ -59,15 +61,17 @@ public class NodeRepositoryMaintenance extends AbstractComponent { HostLivenessTracker hostLivenessTracker, ServiceMonitor serviceMonitor, Zone zone, Orchestrator orchestrator, Metric metric, ConfigserverConfig configserverConfig, - DuperModelInfraApi duperModelInfraApi) { + DuperModelInfraApi duperModelInfraApi, + LoadBalancerService loadBalancerService) { this(nodeRepository, deployer, provisioner, hostLivenessTracker, serviceMonitor, zone, Clock.systemUTC(), - orchestrator, metric, configserverConfig, duperModelInfraApi); + orchestrator, metric, configserverConfig, duperModelInfraApi, loadBalancerService); } public NodeRepositoryMaintenance(NodeRepository nodeRepository, Deployer deployer, Provisioner provisioner, HostLivenessTracker hostLivenessTracker, ServiceMonitor serviceMonitor, Zone zone, Clock clock, Orchestrator orchestrator, Metric metric, - ConfigserverConfig configserverConfig, DuperModelInfraApi duperModelInfraApi) { + ConfigserverConfig configserverConfig, DuperModelInfraApi duperModelInfraApi, + LoadBalancerService loadBalancerService) { DefaultTimes defaults = new DefaultTimes(zone); jobControl = new JobControl(nodeRepository.database()); infrastructureVersions = new InfrastructureVersions(nodeRepository.database()); @@ -84,6 +88,7 @@ public class NodeRepositoryMaintenance extends AbstractComponent { nodeRebooter = new NodeRebooter(nodeRepository, clock, durationFromEnv("reboot_interval").orElse(defaults.rebootInterval), jobControl); metricsReporter = new MetricsReporter(nodeRepository, metric, orchestrator, serviceMonitor, periodicApplicationMaintainer::pendingDeployments, durationFromEnv("metrics_interval").orElse(defaults.metricsInterval), jobControl); infrastructureProvisioner = new InfrastructureProvisioner(provisioner, nodeRepository, infrastructureVersions, durationFromEnv("infrastructure_provision_interval").orElse(defaults.infrastructureProvisionInterval), jobControl, duperModelInfraApi); + loadBalancerExpirer = new LoadBalancerExpirer(nodeRepository, durationFromEnv("load_balancer_expiry").orElse(defaults.loadBalancerExpiry), jobControl, loadBalancerService); // The DuperModel is filled with infrastructure applications by the infrastructure provisioner, so explicitly run that now infrastructureProvisioner.maintain(); @@ -109,6 +114,7 @@ public class NodeRepositoryMaintenance extends AbstractComponent { provisionedExpirer.deconstruct(); metricsReporter.deconstruct(); infrastructureProvisioner.deconstruct(); + loadBalancerExpirer.deconstruct(); } public JobControl jobControl() { return jobControl; } @@ -156,6 +162,7 @@ public class NodeRepositoryMaintenance extends AbstractComponent { private final Duration metricsInterval; private final Duration retiredInterval; private final Duration infrastructureProvisionInterval; + private final Duration loadBalancerExpiry; private final NodeFailer.ThrottlePolicy throttlePolicy; @@ -171,6 +178,7 @@ public class NodeRepositoryMaintenance extends AbstractComponent { metricsInterval = Duration.ofMinutes(1); infrastructureProvisionInterval = Duration.ofMinutes(3); throttlePolicy = NodeFailer.ThrottlePolicy.hosted; + loadBalancerExpiry = Duration.ofHours(1); if (zone.environment().equals(Environment.prod) && zone.system() != SystemName.cd) { inactiveExpiry = Duration.ofHours(4); // enough time for the application owner to discover and redeploy diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java index c99cccd2ab9..ac42bb1a6a5 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java @@ -456,10 +456,10 @@ public class CuratorDatabaseClient { }); } - public void removeLoadBalancer(LoadBalancer loadBalancer) { + public void removeLoadBalancer(LoadBalancerId loadBalancer) { NestedTransaction transaction = new NestedTransaction(); CuratorTransaction curatorTransaction = newCuratorTransactionIn(transaction); - curatorTransaction.add(CuratorOperations.delete(loadBalancerPath(loadBalancer.id()).getAbsolute())); + curatorTransaction.add(CuratorOperations.delete(loadBalancerPath(loadBalancer).getAbsolute())); transaction.commit(); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirerTest.java new file mode 100644 index 00000000000..59323bfdeb5 --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirerTest.java @@ -0,0 +1,96 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.maintenance; + +import com.yahoo.component.Vtag; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.HostSpec; +import com.yahoo.config.provision.Zone; +import com.yahoo.transaction.NestedTransaction; +import com.yahoo.vespa.hosted.provision.flag.FlagId; +import com.yahoo.vespa.hosted.provision.lb.LoadBalancer; +import com.yahoo.vespa.hosted.provision.lb.LoadBalancerId; +import com.yahoo.vespa.hosted.provision.node.Agent; +import com.yahoo.vespa.hosted.provision.provisioning.ProvisioningTester; +import org.junit.Before; +import org.junit.Test; + +import java.time.Duration; +import java.util.List; +import java.util.Map; +import java.util.function.Supplier; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +/** + * @author mpolden + */ +public class LoadBalancerExpirerTest { + + private ProvisioningTester tester; + + @Before + public void before() { + tester = new ProvisioningTester(Zone.defaultZone()); + } + + @Test + public void test_maintain() { + LoadBalancerExpirer expirer = new LoadBalancerExpirer(tester.nodeRepository(), + Duration.ofDays(1), + new JobControl(tester.nodeRepository().database()), + tester.loadBalancerService()); + tester.nodeRepository().flags().setEnabled(FlagId.exclusiveLoadBalancer, true); + Supplier<Map<LoadBalancerId, LoadBalancer>> loadBalancers = () -> tester.nodeRepository().database().readLoadBalancers(); + + // Deploy two applications with load balancers + ClusterSpec.Id cluster = ClusterSpec.Id.from("qrs"); + ApplicationId app1 = tester.makeApplicationId(); + ApplicationId app2 = tester.makeApplicationId(); + LoadBalancerId lb1 = new LoadBalancerId(app1, cluster); + LoadBalancerId lb2 = new LoadBalancerId(app2, cluster); + deployApplication(app1, cluster); + deployApplication(app2, cluster); + assertEquals(2, loadBalancers.get().size()); + + // Remove one application deactivates load balancers for that application + removeApplication(app1); + assertTrue(loadBalancers.get().get(lb1).inactive()); + assertFalse(loadBalancers.get().get(lb2).inactive()); + + // Expirer defers removal while nodes are still allocated to application + expirer.maintain(); + assertEquals(2, tester.loadBalancerService().loadBalancers().size()); + + // Expirer removes load balancers once nodes are deallocated + dirtyNodesOf(app1); + expirer.maintain(); + assertFalse("Inactive load balancer removed", tester.loadBalancerService().loadBalancers().containsKey(lb1)); + + // Active load balancer is left alone + assertFalse(loadBalancers.get().get(lb2).inactive()); + assertTrue("Active load balancer is not removed", tester.loadBalancerService().loadBalancers().containsKey(lb2)); + } + + private void dirtyNodesOf(ApplicationId application) { + tester.nodeRepository().setDirty(tester.nodeRepository().getNodes(application), Agent.system, "unit-test"); + } + + private void removeApplication(ApplicationId application) { + NestedTransaction transaction = new NestedTransaction(); + tester.provisioner().remove(transaction, application); + transaction.commit(); + } + + private void deployApplication(ApplicationId application, ClusterSpec.Id cluster) { + tester.makeReadyNodes(10, "default"); + List<HostSpec> hosts = tester.prepare(application, ClusterSpec.request(ClusterSpec.Type.container, cluster, + Vtag.currentVersion, false), + 2, 1, + "default"); + tester.activate(application, hosts); + } + +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java index 5612c8dc665..2c63b9fd62c 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java @@ -101,7 +101,7 @@ public class ProvisioningTester { } } - static FlavorsConfig createConfig() { + public static FlavorsConfig createConfig() { FlavorConfigBuilder b = new FlavorConfigBuilder(); b.addFlavor("default", 2., 4., 100, Flavor.Type.BARE_METAL).cost(3); b.addFlavor("small", 1., 2., 50, Flavor.Type.BARE_METAL).cost(2); @@ -170,11 +170,11 @@ public class ProvisioningTester { deactivateTransaction.commit(); } - Collection<String> toHostNames(Collection<HostSpec> hosts) { + public Collection<String> toHostNames(Collection<HostSpec> hosts) { return hosts.stream().map(HostSpec::hostname).collect(Collectors.toSet()); } - Set<String> toHostNames(List<Node> nodes) { + public Set<String> toHostNames(List<Node> nodes) { return nodes.stream().map(Node::hostname).collect(Collectors.toSet()); } @@ -182,7 +182,7 @@ public class ProvisioningTester { * Asserts that each active node in this application has a restart count equaling the * number of matches to the given filters */ - void assertRestartCount(ApplicationId application, HostFilter... filters) { + public void assertRestartCount(ApplicationId application, HostFilter... filters) { for (Node node : nodeRepository.getNodes(application, Node.State.active)) { int expectedRestarts = 0; for (HostFilter filter : filters) @@ -199,7 +199,7 @@ public class ProvisioningTester { assertEquals(beforeFailCount + 1, failedNode.status().failCount()); } - void assertMembersOf(ClusterSpec requestedCluster, Collection<HostSpec> hosts) { + public void assertMembersOf(ClusterSpec requestedCluster, Collection<HostSpec> hosts) { Set<Integer> indices = new HashSet<>(); for (HostSpec host : hosts) { ClusterSpec nodeCluster = host.membership().get().cluster(); @@ -214,14 +214,14 @@ public class ProvisioningTester { assertEquals("Indexes in " + requestedCluster + " are disjunct", hosts.size(), indices.size()); } - HostSpec removeOne(Set<HostSpec> hosts) { + public HostSpec removeOne(Set<HostSpec> hosts) { Iterator<HostSpec> i = hosts.iterator(); HostSpec removed = i.next(); i.remove(); return removed; } - ApplicationId makeApplicationId() { + public ApplicationId makeApplicationId() { return ApplicationId.from( TenantName.from(UUID.randomUUID().toString()), ApplicationName.from(UUID.randomUUID().toString()), @@ -232,15 +232,15 @@ public class ProvisioningTester { return makeReadyNodes(n, flavor, NodeType.tenant); } - List<Node> makeReadyNodes(int n, String flavor, NodeType type) { + public List<Node> makeReadyNodes(int n, String flavor, NodeType type) { return makeReadyNodes(n, flavor, type, 0); } - List<Node> makeProvisionedNodes(int count, String flavor, NodeType type, int ipAddressPoolSize) { + public List<Node> makeProvisionedNodes(int count, String flavor, NodeType type, int ipAddressPoolSize) { return makeProvisionedNodes(count, flavor, type, ipAddressPoolSize, false); } - List<Node> makeProvisionedNodes(int n, String flavor, NodeType type, int ipAddressPoolSize, boolean dualStack) { + public List<Node> makeProvisionedNodes(int n, String flavor, NodeType type, int ipAddressPoolSize, boolean dualStack) { List<Node> nodes = new ArrayList<>(n); for (int i = 0; i < n; i++) { @@ -290,7 +290,7 @@ public class ProvisioningTester { return nodes; } - List<Node> makeConfigServers(int n, String flavor, Version configServersVersion) { + public List<Node> makeConfigServers(int n, String flavor, Version configServersVersion) { List<Node> nodes = new ArrayList<>(n); MockNameResolver nameResolver = (MockNameResolver)nodeRepository().nameResolver(); @@ -322,33 +322,33 @@ public class ProvisioningTester { return nodeRepository.getNodes(application.getApplicationId(), Node.State.active); } - List<Node> makeReadyNodes(int n, String flavor, NodeType type, int ipAddressPoolSize) { + public List<Node> makeReadyNodes(int n, String flavor, NodeType type, int ipAddressPoolSize) { return makeReadyNodes(n, flavor, type, ipAddressPoolSize, false); } - List<Node> makeReadyNodes(int n, String flavor, NodeType type, int ipAddressPoolSize, boolean dualStack) { + public List<Node> makeReadyNodes(int n, String flavor, NodeType type, int ipAddressPoolSize, boolean dualStack) { List<Node> nodes = makeProvisionedNodes(n, flavor, type, ipAddressPoolSize, dualStack); nodes = nodeRepository.setDirty(nodes, Agent.system, getClass().getSimpleName()); return nodeRepository.setReady(nodes, Agent.system, getClass().getSimpleName()); } /** Creates a set of virtual docker nodes on a single docker host */ - List<Node> makeReadyDockerNodes(int n, String flavor, String dockerHostId) { + public List<Node> makeReadyDockerNodes(int n, String flavor, String dockerHostId) { return makeReadyVirtualNodes(n, flavor, Optional.of(dockerHostId)); } /** Creates a set of virtual nodes on a single parent host */ - List<Node> makeReadyVirtualNodes(int n, String flavor, Optional<String> parentHostId) { + public List<Node> makeReadyVirtualNodes(int n, String flavor, Optional<String> parentHostId) { return makeReadyVirtualNodes(n, 0, flavor, parentHostId, index -> UUID.randomUUID().toString()); } /** Creates a set of virtual nodes on a single parent host */ - List<Node> makeReadyVirtualNode(int index, String flavor, String parentHostId) { + public List<Node> makeReadyVirtualNode(int index, String flavor, String parentHostId) { return makeReadyVirtualNodes(1, index, flavor, Optional.of(parentHostId), i -> String.format("node%03d", i)); } /** Creates a set of virtual nodes on a single parent host */ - List<Node> makeReadyVirtualNodes(int count, int startIndex, String flavor, Optional<String> parentHostId, + public List<Node> makeReadyVirtualNodes(int count, int startIndex, String flavor, Optional<String> parentHostId, Function<Integer, String> nodeNamer) { List<Node> nodes = new ArrayList<>(count); for (int i = startIndex; i < count + startIndex; i++) { @@ -362,16 +362,16 @@ public class ProvisioningTester { return nodes; } - List<Node> makeReadyVirtualNodes(int n, String flavor, String parentHostId) { + public List<Node> makeReadyVirtualNodes(int n, String flavor, String parentHostId) { return makeReadyVirtualNodes(n, flavor, Optional.of(parentHostId)); } /** Returns the hosts from the input list which are not retired */ - List<HostSpec> nonRetired(Collection<HostSpec> hosts) { + public List<HostSpec> nonRetired(Collection<HostSpec> hosts) { return hosts.stream().filter(host -> ! host.membership().get().retired()).collect(Collectors.toList()); } - void assertNumberOfNodesWithFlavor(List<HostSpec> hostSpecs, String flavor, int expectedCount) { + public void assertNumberOfNodesWithFlavor(List<HostSpec> hostSpecs, String flavor, int expectedCount) { long actualNodesWithFlavor = hostSpecs.stream() .map(HostSpec::hostname) .map(this::getNodeFlavor) |