diff options
24 files changed, 332 insertions, 413 deletions
diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/HostEvent.java b/config-provisioning/src/main/java/com/yahoo/config/provision/HostEvent.java new file mode 100644 index 00000000000..e7108bd6182 --- /dev/null +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/HostEvent.java @@ -0,0 +1,43 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.provision; + +import java.util.Objects; + +/** + * A maintenance event for a host. + * + * @author mpolden + */ +public class HostEvent { + + private final String id; + private final String hostId; + private final String description; + + public HostEvent(String id, String hostId, String description) { + this.id = Objects.requireNonNull(id); + this.hostId = Objects.requireNonNull(hostId); + this.description = Objects.requireNonNull(description); + } + + /** ID of the event */ + public String id() { + return id; + } + + /** ID of the host affected by this event, i.e. instance ID */ + public String hostId() { + return hostId; + } + + /** Human-readable description of the event */ + public String description() { + return description; + } + + @Override + public String toString() { + return "event " + id + " affecting host " + hostId + ": '" + description + "'"; + } + +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/ServiceRegistry.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/ServiceRegistry.java index 52f687e5708..c0adb7389c8 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/ServiceRegistry.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/ServiceRegistry.java @@ -7,7 +7,6 @@ import com.yahoo.vespa.hosted.controller.api.identifiers.ControllerVersion; import com.yahoo.vespa.hosted.controller.api.integration.archive.ArchiveService; import com.yahoo.vespa.hosted.controller.api.integration.artifact.ArtifactRegistry; import com.yahoo.vespa.hosted.controller.api.integration.athenz.AccessControlService; -import com.yahoo.vespa.hosted.controller.api.integration.aws.CloudEventFetcher; import com.yahoo.vespa.hosted.controller.api.integration.aws.ResourceTagger; import com.yahoo.vespa.hosted.controller.api.integration.aws.RoleService; import com.yahoo.vespa.hosted.controller.api.integration.billing.BillingController; @@ -77,8 +76,6 @@ public interface ServiceRegistry { CostReportConsumer costReportConsumer(); - CloudEventFetcher eventFetcherService(); - ArtifactRepository artifactRepository(); TesterCloud testerCloud(); diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/aws/CloudEvent.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/aws/CloudEvent.java deleted file mode 100644 index b5445791bf0..00000000000 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/aws/CloudEvent.java +++ /dev/null @@ -1,36 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.api.integration.aws; - -import java.util.Date; -import java.util.Optional; -import java.util.Set; - -/** - * A maintenance event in a cloud service. - * - * @author freva - */ -public class CloudEvent { - - public final String instanceEventId; - public final String code; - public final String description; - public final Optional<Date> notBefore; - public final Optional<Date> notBeforeDeadline; - public final Optional<Date> notAfter; - public final String awsRegionName; - public final Set<String> affectedInstances; - - public CloudEvent(String instanceEventId, String code, String description, Date notAfter, Date notBefore, - Date notBeforeDeadline, String awsRegionName, Set<String> affectedInstances) { - this.instanceEventId = instanceEventId; - this.code = code; - this.description = description; - this.notBefore = Optional.ofNullable(notBefore); - this.notBeforeDeadline = Optional.ofNullable(notBeforeDeadline); - this.notAfter = Optional.ofNullable(notAfter); - this.awsRegionName = awsRegionName; - this.affectedInstances = Set.copyOf(affectedInstances); - } - -} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/aws/CloudEventFetcher.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/aws/CloudEventFetcher.java deleted file mode 100644 index 0d08a5a8cb6..00000000000 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/aws/CloudEventFetcher.java +++ /dev/null @@ -1,13 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.api.integration.aws; - -import java.util.List; - -/** - * @author freva - */ -public interface CloudEventFetcher { - - List<CloudEvent> getEvents(String regionName); - -} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/aws/MockCloudEventFetcher.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/aws/MockCloudEventFetcher.java deleted file mode 100644 index 3300d8879ce..00000000000 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/aws/MockCloudEventFetcher.java +++ /dev/null @@ -1,25 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.api.integration.aws; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -/** - * @author freva - */ -public class MockCloudEventFetcher implements CloudEventFetcher { - - private final Map<String, List<CloudEvent>> mockedEvents = new HashMap<>(); - - @Override - public List<CloudEvent> getEvents(String regionName) { - return mockedEvents.getOrDefault(regionName, new ArrayList<>()); - } - - public void addEvent(String regionName, CloudEvent cloudEvent) { - mockedEvents.computeIfAbsent(regionName, i -> new ArrayList<>()).add(cloudEvent); - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java index 1c4f0994f8c..af56666c6eb 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java @@ -251,6 +251,19 @@ public class Controller extends AbstractComponent { } } + /** Clear the target OS version for given cloud in this system */ + public void cancelOsUpgradeIn(CloudName cloudName) { + try (Mutex lock = curator.lockOsVersions()) { + Map<CloudName, OsVersionTarget> targets = curator.readOsVersionTargets().stream() + .collect(Collectors.toMap(t -> t.osVersion().cloud(), + Function.identity())); + if (targets.remove(cloudName) == null) { + throw new IllegalArgumentException("Cloud '" + cloudName.value() + " has no OS upgrade target"); + } + curator.writeOsVersionTargets(new TreeSet<>(targets.values())); + } + } + /** Returns the current OS version status */ public OsVersionStatus osVersionStatus() { return curator.readOsVersionStatus(); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CloudEventTracker.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CloudEventTracker.java deleted file mode 100644 index 021c02fb6a0..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CloudEventTracker.java +++ /dev/null @@ -1,77 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.maintenance; - -import com.yahoo.config.provision.CloudName; -import com.yahoo.config.provision.zone.ZoneApi; -import com.yahoo.vespa.hosted.controller.Controller; -import com.yahoo.vespa.hosted.controller.api.integration.aws.CloudEvent; -import com.yahoo.vespa.hosted.controller.api.integration.aws.CloudEventFetcher; -import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; -import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeFilter; -import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeRepository; - -import java.time.Duration; -import java.util.List; -import java.util.Map; -import java.util.logging.Logger; -import java.util.stream.Collectors; - -/** - * This tracks maintenance events from cloud providers and deprovisions any affected hosts. - * - * @author mgimle - */ -public class CloudEventTracker extends ControllerMaintainer { - - private static final Logger log = Logger.getLogger(CloudEventTracker.class.getName()); - - private final CloudEventFetcher eventFetcher; - private final Map<String, List<ZoneApi>> zonesByCloudNativeRegion; - private final NodeRepository nodeRepository; - - CloudEventTracker(Controller controller, Duration interval) { - super(controller, interval); - this.eventFetcher = controller.serviceRegistry().eventFetcherService(); - this.nodeRepository = controller.serviceRegistry().configServer().nodeRepository(); - this.zonesByCloudNativeRegion = supportedZonesByRegion(); - } - - @Override - protected double maintain() { - for (var region : zonesByCloudNativeRegion.keySet()) { - for (var event : eventFetcher.getEvents(region)) { - deprovisionAffectedHosts(region, event); - } - } - return 1.0; - } - - /** Deprovision any host affected by given event */ - private void deprovisionAffectedHosts(String region, CloudEvent event) { - for (var zone : zonesByCloudNativeRegion.get(region)) { - for (var node : nodeRepository.list(zone.getId(), NodeFilter.all())) { - if (!deprovision(node, event)) continue; - log.info("Retiring and deprovisioning " + node.hostname().value() + " in " + zone.getId() + - ": Affected by maintenance event " + event.instanceEventId); - nodeRepository.retire(zone.getId(), node.hostname().value(), true, true); - } - } - } - - private static boolean deprovision(Node node, CloudEvent event) { - if (!node.type().isHost()) return false; // Non-hosts are never affected - if (node.wantToRetire() && node.wantToDeprovision()) return false; // Already deprovisioning - return event.affectedInstances.stream() - .anyMatch(instance -> node.hostname().value().contains(instance)); - } - - /** Returns zones supported by this, grouped by their native region name */ - private Map<String, List<ZoneApi>> supportedZonesByRegion() { - return controller().zoneRegistry().zones() - .ofCloud(CloudName.from("aws")) - .reachable() - .zones().stream() - .collect(Collectors.groupingBy(ZoneApi::getCloudNativeRegionName)); - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java index 193d171e334..795f14bbde1 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java @@ -59,7 +59,6 @@ public class ControllerMaintenance extends AbstractComponent { maintainers.add(new NameServiceDispatcher(controller, intervals.nameServiceDispatcher)); maintainers.add(new CostReportMaintainer(controller, intervals.costReportMaintainer, controller.serviceRegistry().costReportConsumer())); maintainers.add(new ResourceMeterMaintainer(controller, intervals.resourceMeterMaintainer, metric, controller.serviceRegistry().meteringService())); - maintainers.add(new CloudEventTracker(controller, intervals.cloudEventReporter)); maintainers.add(new ResourceTagMaintainer(controller, intervals.resourceTagMaintainer, controller.serviceRegistry().resourceTagger())); maintainers.add(new SystemRoutingPolicyMaintainer(controller, intervals.systemRoutingPolicyMaintainer)); maintainers.add(new ApplicationMetaDataGarbageCollector(controller, intervals.applicationMetaDataGarbageCollector)); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/os/OsApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/os/OsApiHandler.java index 88fd3a58d23..853739ee9c3 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/os/OsApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/os/OsApiHandler.java @@ -19,6 +19,7 @@ import com.yahoo.slime.Cursor; import com.yahoo.slime.Inspector; import com.yahoo.slime.Slime; import com.yahoo.slime.SlimeUtils; +import com.yahoo.slime.Type; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.auditlog.AuditLoggingRequestHandler; import com.yahoo.vespa.hosted.controller.versions.OsVersionTarget; @@ -32,6 +33,7 @@ import java.util.List; import java.util.Optional; import java.util.Set; import java.util.StringJoiner; +import java.util.function.Function; import java.util.logging.Level; import java.util.stream.Collectors; @@ -71,7 +73,7 @@ public class OsApiHandler extends AuditLoggingRequestHandler { private HttpResponse patch(HttpRequest request) { Path path = new Path(request.getUri()); - if (path.matches("/os/v1/")) return new SlimeJsonResponse(setOsVersion(request)); + if (path.matches("/os/v1/")) return setOsVersion(request); return ErrorResponse.notFoundError("Nothing at " + path); } @@ -130,36 +132,20 @@ public class OsApiHandler extends AuditLoggingRequestHandler { return zones.zones().stream().map(ZoneApi::getId).collect(Collectors.toList()); } - private Slime setOsVersion(HttpRequest request) { + private HttpResponse setOsVersion(HttpRequest request) { Slime requestData = toSlime(request.getData()); Inspector root = requestData.get(); - Inspector versionField = root.field("version"); - Inspector cloudField = root.field("cloud"); - Inspector upgradeBudgetField = root.field("upgradeBudget"); - boolean force = root.field("force").asBool(); - if (!versionField.valid() || !cloudField.valid() || !upgradeBudgetField.valid()) { - throw new IllegalArgumentException("Fields 'version', 'cloud' and 'upgradeBudget' are required"); - } - - CloudName cloud = CloudName.from(cloudField.asString()); - Version target; - try { - target = Version.fromString(versionField.asString()); - } catch (IllegalArgumentException e) { - throw new IllegalArgumentException("Invalid version '" + versionField.asString() + "'", e); - } - Duration upgradeBudget; - try { - upgradeBudget = Duration.parse(upgradeBudgetField.asString()); - } catch (Exception e) { - throw new IllegalArgumentException("Invalid duration '" + upgradeBudgetField.asString() + "'", e); + CloudName cloud = parseStringField("cloud", root, CloudName::from); + if (requireField("version", root).type() == Type.NIX) { + controller.cancelOsUpgradeIn(cloud); + return new MessageResponse("Cleared target OS version for cloud '" + cloud.value() + "'"); } + Version target = parseStringField("version", root, Version::fromString); + Duration upgradeBudget = parseStringField("upgradeBudget", root, Duration::parse); + boolean force = root.field("force").asBool(); controller.upgradeOsIn(cloud, target, upgradeBudget, force); - Slime response = new Slime(); - Cursor cursor = response.setObject(); - cursor.setString("message", "Set target OS version for cloud '" + cloud.value() + "' to " + - target.toFullString() + " with upgrade budget " + upgradeBudget); - return response; + return new MessageResponse("Set target OS version for cloud '" + cloud.value() + "' to " + + target.toFullString() + " with upgrade budget " + upgradeBudget); } private Slime osVersions() { @@ -196,4 +182,19 @@ public class OsApiHandler extends AuditLoggingRequestHandler { } } + private static <T> T parseStringField(String name, Inspector root, Function<String, T> parser) { + String fieldValue = requireField(name, root).asString(); + try { + return parser.apply(fieldValue); + } catch (Exception e) { + throw new IllegalArgumentException("Invalid " + name + " '" + fieldValue + "'", e); + } + } + + private static Inspector requireField(String name, Inspector root) { + Inspector field = root.field(name); + if (!field.valid()) throw new IllegalArgumentException("Field '" + name + "' is required"); + return field; + } + } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java index a91d81bb0c5..596335baeb5 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java @@ -1,10 +1,10 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.integration; -import com.yahoo.component.annotation.Inject; import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.component.AbstractComponent; import com.yahoo.component.Version; +import com.yahoo.component.annotation.Inject; import com.yahoo.config.provision.CloudName; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.SystemName; @@ -15,7 +15,6 @@ import com.yahoo.vespa.hosted.controller.api.integration.archive.ArchiveService; import com.yahoo.vespa.hosted.controller.api.integration.archive.MockArchiveService; import com.yahoo.vespa.hosted.controller.api.integration.athenz.AccessControlService; import com.yahoo.vespa.hosted.controller.api.integration.athenz.MockAccessControlService; -import com.yahoo.vespa.hosted.controller.api.integration.aws.MockCloudEventFetcher; import com.yahoo.vespa.hosted.controller.api.integration.aws.MockResourceTagger; import com.yahoo.vespa.hosted.controller.api.integration.aws.MockRoleService; import com.yahoo.vespa.hosted.controller.api.integration.aws.ResourceTagger; @@ -76,7 +75,6 @@ public class ServiceRegistryMock extends AbstractComponent implements ServiceReg private final MemoryEntityService memoryEntityService = new MemoryEntityService(); private final DummySystemMonitor systemMonitor = new DummySystemMonitor(); private final CostReportConsumerMock costReportConsumerMock = new CostReportConsumerMock(); - private final MockCloudEventFetcher mockAwsEventFetcher = new MockCloudEventFetcher(); private final ArtifactRepositoryMock artifactRepositoryMock = new ArtifactRepositoryMock(); private final MockTesterCloud mockTesterCloud; private final ApplicationStoreMock applicationStoreMock = new ApplicationStoreMock(); @@ -183,11 +181,6 @@ public class ServiceRegistryMock extends AbstractComponent implements ServiceReg } @Override - public MockCloudEventFetcher eventFetcherService() { - return mockAwsEventFetcher; - } - - @Override public ArtifactRepositoryMock artifactRepository() { return artifactRepositoryMock; } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CloudEventTrackerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CloudEventTrackerTest.java deleted file mode 100644 index 6ccd307f0d9..00000000000 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CloudEventTrackerTest.java +++ /dev/null @@ -1,145 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.maintenance; - -import com.yahoo.config.provision.HostName; -import com.yahoo.config.provision.NodeType; -import com.yahoo.config.provision.zone.ZoneId; -import com.yahoo.vespa.hosted.controller.ControllerTester; -import com.yahoo.vespa.hosted.controller.api.integration.aws.CloudEvent; -import com.yahoo.vespa.hosted.controller.api.integration.aws.MockCloudEventFetcher; -import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; -import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeFilter; -import com.yahoo.vespa.hosted.controller.integration.ZoneApiMock; -import org.junit.Test; - -import java.time.Duration; -import java.util.Arrays; -import java.util.Date; -import java.util.List; -import java.util.Set; -import java.util.stream.Collectors; - -import static org.junit.Assert.assertEquals; - -/** - * @author olaa - */ -public class CloudEventTrackerTest { - - private final ControllerTester tester = new ControllerTester(); - private final ZoneApiMock unsupportedZone = createZone("prod.zone3", "region-1", "other"); - private final ZoneApiMock zone1 = createZone("prod.zone1", "region-1", "aws"); - private final ZoneApiMock zone2 = createZone("prod.zone2", "region-2", "aws"); - - /** - * Test scenario: Consider three zones, two of which are supported - * - * We want to test the following: - * 1. Unsupported zone is completely ignored - * 2. Hosts affected by cloud event are deprovisioned - */ - @Test - public void maintain() { - setUpZones(); - CloudEventTracker cloudEventTracker = new CloudEventTracker(tester.controller(), Duration.ofMinutes(15)); - assertEquals(Set.of("host1.com", "host2.com", "host3.com"), hostsNotDeprovisioning(unsupportedZone.getId())); - assertEquals(Set.of("host1.com", "host2.com", "host3.com"), hostsNotDeprovisioning(zone1.getId())); - assertEquals(Set.of("host4.com", "host5.com", "confighost.com"), hostsNotDeprovisioning(zone2.getId())); - - mockEvents(); - cloudEventTracker.maintain(); - assertEquals(Set.of("host1.com", "host2.com", "host3.com"), hostsNotDeprovisioning(unsupportedZone.getId())); - assertEquals(Set.of("host3.com"), hostsNotDeprovisioning(zone1.getId())); - assertEquals(Set.of("host4.com"), hostsNotDeprovisioning(zone2.getId())); - } - - private void mockEvents() { - MockCloudEventFetcher eventFetcher = (MockCloudEventFetcher) tester.controller().serviceRegistry().eventFetcherService(); - - Date date = new Date(); - CloudEvent event1 = new CloudEvent("event 1", - "instance code", - "description", - date, - date, - date, - "region-1", - Set.of("host1", "host2")); - - CloudEvent event2 = new CloudEvent("event 2", - "instance code", - "description", - date, - date, - date, - "region-2", - Set.of("host5", "confighost")); - - eventFetcher.addEvent("region-1", event1); - eventFetcher.addEvent("region-2", event2); - } - - private void setUpZones() { - tester.zoneRegistry().setZones( - unsupportedZone, - zone1, - zone2); - - tester.configServer().nodeRepository().putNodes( - unsupportedZone.getId(), - createNodesWithHostnames( - "host1.com", - "host2.com", - "host3.com" - ) - ); - tester.configServer().nodeRepository().putNodes( - zone1.getId(), - createNodesWithHostnames( - "host1.com", - "host2.com", - "host3.com" - ) - ); - tester.configServer().nodeRepository().putNodes( - zone2.getId(), - createNodesWithHostnames( - "host4.com", - "host5.com" - ) - ); - tester.configServer().nodeRepository().putNodes( - zone2.getId(), - List.of(createNode("confighost.com", NodeType.confighost)) - ); - } - - private List<Node> createNodesWithHostnames(String... hostnames) { - return Arrays.stream(hostnames) - .map(hostname -> createNode(hostname, NodeType.host)) - .collect(Collectors.toUnmodifiableList()); - } - - private Node createNode(String hostname, NodeType nodeType) { - return Node.builder() - .hostname(HostName.of(hostname)) - .type(nodeType) - .build(); - } - - private Set<String> hostsNotDeprovisioning(ZoneId zoneId) { - return tester.configServer().nodeRepository().list(zoneId, NodeFilter.all()) - .stream() - .filter(node -> !node.wantToDeprovision()) - .map(node -> node.hostname().value()) - .collect(Collectors.toSet()); - } - - private ZoneApiMock createZone(String zoneId, String cloudNativeRegionName, String cloud) { - return ZoneApiMock.newBuilder().withId(zoneId) - .withCloudNativeRegionName(cloudNativeRegionName) - .withCloud(cloud) - .build(); - } - -} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json index 8b2e5578ae0..7110cbbd738 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json @@ -22,9 +22,6 @@ "name": "ChangeRequestMaintainer" }, { - "name": "CloudEventTracker" - }, - { "name": "CloudTrialExpirer" }, { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/OsApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/OsApiTest.java index 5c210616cb1..15e7a804143 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/OsApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/OsApiTest.java @@ -95,14 +95,18 @@ public class OsApiTest extends ControllerContainerTest { assertResponse(new Request("http://localhost:8080/os/v1/", "{\"version\": \"7.5.1\", \"cloud\": \"cloud1\", \"force\": true, \"upgradeBudget\": \"PT0S\"}", Request.Method.PATCH), "{\"message\":\"Set target OS version for cloud 'cloud1' to 7.5.1 with upgrade budget PT0S\"}", 200); + // Clear target for a given cloud + assertResponse(new Request("http://localhost:8080/os/v1/", "{\"version\": null, \"cloud\": \"cloud2\"}", Request.Method.PATCH), + "{\"message\":\"Cleared target OS version for cloud 'cloud2'\"}", 200); + // Error: Missing fields assertResponse(new Request("http://localhost:8080/os/v1/", "{\"version\": \"7.6\"}", Request.Method.PATCH), - "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Fields 'version', 'cloud' and 'upgradeBudget' are required\"}", 400); + "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Field 'cloud' is required\"}", 400); assertResponse(new Request("http://localhost:8080/os/v1/", "{\"cloud\": \"cloud1\"}", Request.Method.PATCH), - "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Fields 'version', 'cloud' and 'upgradeBudget' are required\"}", 400); + "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Field 'version' is required\"}", 400); // Error: Invalid versions - assertResponse(new Request("http://localhost:8080/os/v1/", "{\"version\": null, \"cloud\": \"cloud1\", \"upgradeBudget\": \"PT0S\"}", Request.Method.PATCH), + assertResponse(new Request("http://localhost:8080/os/v1/", "{\"version\": \"0.0.0\", \"cloud\": \"cloud1\", \"upgradeBudget\": \"PT0S\"}", Request.Method.PATCH), "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Invalid version '0.0.0'\"}", 400); assertResponse(new Request("http://localhost:8080/os/v1/", "{\"version\": \"foo\", \"cloud\": \"cloud1\", \"upgradeBudget\": \"PT0S\"}", Request.Method.PATCH), "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Invalid version 'foo': For input string: \\\"foo\\\"\"}", 400); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostRetirer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostRetirer.java new file mode 100644 index 00000000000..aa3a82d52c9 --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostRetirer.java @@ -0,0 +1,61 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.maintenance; + +import com.yahoo.config.provision.CloudAccount; +import com.yahoo.config.provision.HostEvent; +import com.yahoo.jdisc.Metric; +import com.yahoo.vespa.hosted.provision.NodeList; +import com.yahoo.vespa.hosted.provision.NodeRepository; +import com.yahoo.vespa.hosted.provision.node.Agent; +import com.yahoo.vespa.hosted.provision.provisioning.HostProvisioner; + +import java.time.Duration; +import java.time.Instant; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.logging.Logger; +import java.util.stream.Collectors; + +/** + * Retire and deprovision hosts that are scheduled for maintenance by the cloud provider. + * + * Only applies to dynamically provisioned zones, where a replacement host will be provisioned. + * + * @author mpolden + */ +public class HostRetirer extends NodeRepositoryMaintainer { + + private static final Logger LOG = Logger.getLogger(HostRetirer.class.getName()); + + private final HostProvisioner hostProvisioner; + + public HostRetirer(NodeRepository nodeRepository, Duration interval, Metric metric, HostProvisioner hostProvisioner) { + super(nodeRepository, interval, metric); + this.hostProvisioner = Objects.requireNonNull(hostProvisioner); + } + + @Override + protected double maintain() { + if (!nodeRepository().zone().getCloud().dynamicProvisioning()) return 1.0; + + NodeList candidates = nodeRepository().nodes().list() + .parents() + .not().deprovisioning(); + List<CloudAccount> cloudAccounts = candidates.stream().flatMap(c -> c.cloudAccount().stream()) + .distinct() + .collect(Collectors.toList()); + Map<String, List<HostEvent>> eventsByHostId = hostProvisioner.hostEventsIn(cloudAccounts).stream() + .collect(Collectors.groupingBy(HostEvent::hostId)); + Instant now = nodeRepository().clock().instant(); + for (var host : candidates) { + List<HostEvent> events = eventsByHostId.get(host.id()); + if (events == null || events.isEmpty()) continue; + + LOG.info("Deprovisioning " + host + " affected by maintenance event" + (events.size() > 1 ? "s" : "") + ": " + events); + nodeRepository().nodes().deprovision(host.hostname(), Agent.system, now); + } + return 1.0; + } + +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java index 29642bc25dd..5e703139f41 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java @@ -72,6 +72,9 @@ public class NodeRepositoryMaintenance extends AbstractComponent { provisionServiceProvider.getHostProvisioner() .map(hostProvisioner -> new DynamicProvisioningMaintainer(nodeRepository, defaults.dynamicProvisionerInterval, hostProvisioner, flagSource, metric)) .ifPresent(maintainers::add); + provisionServiceProvider.getHostProvisioner() + .map(hostProvisioner -> new HostRetirer(nodeRepository, defaults.hostRetirerInterval, metric, hostProvisioner)) + .ifPresent(maintainers::add); // The DuperModel is filled with infrastructure applications by the infrastructure provisioner, so explicitly run that now infrastructureProvisioner.maintainButThrowOnException(); } @@ -116,6 +119,7 @@ public class NodeRepositoryMaintenance extends AbstractComponent { private final Duration autoscalingInterval; private final Duration scalingSuggestionsInterval; private final Duration switchRebalancerInterval; + private final Duration hostRetirerInterval; private final NodeFailer.ThrottlePolicy throttlePolicy; @@ -145,6 +149,7 @@ public class NodeRepositoryMaintenance extends AbstractComponent { throttlePolicy = NodeFailer.ThrottlePolicy.hosted; inactiveConfigServerExpiry = Duration.ofMinutes(5); inactiveControllerExpiry = Duration.ofMinutes(5); + hostRetirerInterval = Duration.ofMinutes(30); if (zone.environment().isProduction() && ! zone.system().isCd()) { inactiveExpiry = Duration.ofHours(4); // enough time for the application owner to discover and redeploy diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostProvisioner.java index b849fccfaa5..567fa9098c9 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostProvisioner.java @@ -5,6 +5,7 @@ import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.CloudAccount; import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.HostEvent; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; import com.yahoo.vespa.hosted.provision.Node; @@ -14,7 +15,7 @@ import java.util.Optional; import java.util.Set; /** - * Service for provisioning physical docker tenant hosts inside the zone. + * A service which supports provisioning container hosts dynamically. * * @author freva */ @@ -78,4 +79,10 @@ public interface HostProvisioner { */ void deprovision(Node host); + /** + * Returns the maintenance events scheduled for hosts in this zone, in given cloud accounts. Host events in the + * zone's default cloud account are always included. + */ + List<HostEvent> hostEventsIn(List<CloudAccount> cloudAccounts); + } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockHostProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockHostProvisioner.java index c09376ff103..8d60dd30dd1 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockHostProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockHostProvisioner.java @@ -6,6 +6,7 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.CloudAccount; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Flavor; +import com.yahoo.config.provision.HostEvent; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.NodeAllocationException; @@ -32,6 +33,7 @@ import java.util.stream.IntStream; public class MockHostProvisioner implements HostProvisioner { private final List<ProvisionedHost> provisionedHosts = new ArrayList<>(); + private final List<HostEvent> hostEvents = new ArrayList<>(); private final List<Flavor> flavors; private final MockNameResolver nameResolver; private final int memoryTaxGb; @@ -100,6 +102,11 @@ public class MockHostProvisioner implements HostProvisioner { deprovisionedHosts++; } + @Override + public List<HostEvent> hostEventsIn(List<CloudAccount> cloudAccounts) { + return Collections.unmodifiableList(hostEvents); + } + /** Returns the hosts that have been provisioned by this */ public List<ProvisionedHost> provisionedHosts() { return Collections.unmodifiableList(provisionedHosts); @@ -130,6 +137,11 @@ public class MockHostProvisioner implements HostProvisioner { return this; } + public MockHostProvisioner addEvent(HostEvent event) { + hostEvents.add(event); + return this; + } + public boolean compatible(Flavor flavor, NodeResources resources) { NodeResources resourcesToVerify = resources.withMemoryGb(resources.memoryGb() - memoryTaxGb); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/HostRetirerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/HostRetirerTest.java new file mode 100644 index 00000000000..a46946b7cee --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/HostRetirerTest.java @@ -0,0 +1,62 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.maintenance; + +import com.yahoo.config.provision.Cloud; +import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.HostEvent; +import com.yahoo.config.provision.NodeFlavors; +import com.yahoo.config.provision.NodeResources; +import com.yahoo.config.provision.RegionName; +import com.yahoo.config.provision.SystemName; +import com.yahoo.config.provision.Zone; +import com.yahoo.jdisc.test.MockMetric; +import com.yahoo.vespa.hosted.provision.Node; +import com.yahoo.vespa.hosted.provision.NodeList; +import com.yahoo.vespa.hosted.provision.provisioning.FlavorConfigBuilder; +import com.yahoo.vespa.hosted.provision.provisioning.ProvisioningTester; +import com.yahoo.vespa.hosted.provision.testutils.MockHostProvisioner; +import org.junit.Test; + +import java.time.Duration; +import java.util.List; + +import static org.junit.Assert.assertEquals; + +/** + * @author mpolden + */ +public class HostRetirerTest { + + @Test + public void retire_hosts() { + NodeFlavors flavors = FlavorConfigBuilder.createDummies("default"); + MockHostProvisioner hostProvisioner = new MockHostProvisioner(flavors.getFlavors()); + ProvisioningTester tester = new ProvisioningTester.Builder().hostProvisioner(hostProvisioner) + .flavors(flavors.getFlavors()) + .zone(new Zone(Cloud.builder() + .dynamicProvisioning(true) + .build(), SystemName.defaultSystem(), + Environment.defaultEnvironment(), + RegionName.defaultName())) + .build(); + HostRetirer retirer = new HostRetirer(tester.nodeRepository(), Duration.ofDays(1), new MockMetric(), hostProvisioner); + tester.makeReadyHosts(3, new NodeResources(24, 48, 1000, 10)) + .activateTenantHosts(); + List<String> hostIds = tester.nodeRepository().nodes().list(Node.State.active).mapToList(Node::id); + + // No events scheduled + retirer.maintain(); + NodeList hosts = tester.nodeRepository().nodes().list(); + assertEquals(0, hosts.deprovisioning().size()); + + // Event is scheduled for one known host + hostProvisioner.addEvent(new HostEvent("event0", hostIds.get(1), getClass().getSimpleName())) + .addEvent(new HostEvent("event1", "unknown-host-id", getClass().getSimpleName())); + + // Next run retires host + retirer.maintain(); + hosts = tester.nodeRepository().nodes().list(); + assertEquals(1, hosts.deprovisioning().size()); + } + +} diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/ApacheGatewayConnection.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/ApacheGatewayConnection.java index 37e9961417d..6b1078fa393 100644 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/ApacheGatewayConnection.java +++ b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/ApacheGatewayConnection.java @@ -15,14 +15,26 @@ import com.yahoo.vespa.http.client.core.ServerResponseException; import com.yahoo.vespa.http.client.core.Vtag; import org.apache.http.Header; import org.apache.http.HttpHost; +import org.apache.http.HttpRequest; import org.apache.http.HttpResponse; import org.apache.http.StatusLine; +import org.apache.http.auth.AUTH; +import org.apache.http.auth.AuthScope; +import org.apache.http.auth.ChallengeState; +import org.apache.http.auth.Credentials; +import org.apache.http.auth.UsernamePasswordCredentials; import org.apache.http.client.config.RequestConfig; import org.apache.http.client.methods.HttpPost; +import org.apache.http.client.protocol.HttpClientContext; import org.apache.http.entity.InputStreamEntity; +import org.apache.http.impl.auth.BasicScheme; +import org.apache.http.impl.client.BasicAuthCache; +import org.apache.http.impl.client.BasicCredentialsProvider; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClientBuilder; import org.apache.http.message.BasicHeader; +import org.apache.http.protocol.BasicHttpContext; +import org.apache.http.protocol.HttpContext; import org.apache.http.util.EntityUtils; import javax.net.ssl.SSLContext; @@ -249,7 +261,14 @@ class ApacheGatewayConnection implements GatewayConnection { private InputStream executePost(HttpPost httpPost) throws ServerResponseException, IOException { if (httpClient == null) throw new IOException("Trying to executePost while not having a connection/http client"); - HttpResponse response = httpClient.execute(httpPost); + String proxyAuthzHeader = getCustomProxyAuthorizationHeader(connectionParams).orElse(null); + HttpResponse response; + if (connectionParams.getProxyHost() != null && proxyAuthzHeader != null) { + HttpContext context = createContextForcingPreemptiveProxyAuth(proxyAuthzHeader); + response = httpClient.execute(httpPost, context); + } else { + response = httpClient.execute(httpPost); + } try { verifyServerResponseCode(response); verifyServerVersion(response.getFirstHeader(Headers.VERSION)); @@ -264,6 +283,38 @@ class ApacheGatewayConnection implements GatewayConnection { return responseData == null ? null : new ByteArrayInputStream(responseData); } + private static Optional<String> getCustomProxyAuthorizationHeader(ConnectionParams params) { + return params.getHeaders().stream() + .filter(h -> h.getKey().equals(AUTH.PROXY_AUTH_RESP)) + .findAny() + .map(Map.Entry::getValue); + } + + private HttpContext createContextForcingPreemptiveProxyAuth(String proxyAuthzHeader) { + BasicAuthCache authCache = new BasicAuthCache(); + HttpHost proxy = new HttpHost(connectionParams.getProxyHost(), connectionParams.getProxyPort()); + authCache.put(proxy, new CustomAuthScheme(proxyAuthzHeader)); + HttpContext context = new BasicHttpContext(); + context.setAttribute(HttpClientContext.AUTH_CACHE, authCache); + BasicCredentialsProvider prov = new BasicCredentialsProvider(); + prov.setCredentials(new AuthScope(proxy), new UsernamePasswordCredentials("", "")); + context.setAttribute(HttpClientContext.CREDS_PROVIDER, prov); + return context; + } + private static class CustomAuthScheme extends BasicScheme { + final String proxyAuthzHeader; + @SuppressWarnings("deprecation") + CustomAuthScheme(String proxyAuthzHeader) { + super(ChallengeState.PROXY); + this.proxyAuthzHeader = proxyAuthzHeader; + } + @Override + public Header authenticate(Credentials credentials, HttpRequest request, HttpContext context) { + return new BasicHeader(AUTH.PROXY_AUTH_RESP, proxyAuthzHeader); + } + } + + private void verifyServerResponseCode(HttpResponse response) throws ServerResponseException { StatusLine statusLine = response.getStatusLine(); int statusCode = statusLine.getStatusCode(); diff --git a/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/AggregateTestRunner.java b/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/AggregateTestRunner.java index 15aeef18013..d00184f8b04 100644 --- a/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/AggregateTestRunner.java +++ b/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/AggregateTestRunner.java @@ -5,15 +5,20 @@ import java.util.ArrayList; import java.util.Collection; import java.util.List; import java.util.concurrent.CompletableFuture; +import java.util.logging.Level; import java.util.logging.LogRecord; +import java.util.logging.Logger; /** * @author jonmv */ public class AggregateTestRunner implements TestRunner { + private static final Logger log = Logger.getLogger(AggregateTestRunner.class.getName()); + private final List<TestRunner> wrapped; private int current = -1; + private boolean error = false; private final Object monitor = new Object(); private AggregateTestRunner(List<TestRunner> testRunners) { @@ -36,6 +41,7 @@ public class AggregateTestRunner implements TestRunner { @Override public Status getStatus() { + if (error) return Status.ERROR; synchronized (monitor) { if (current == -1) return Status.NOT_STARTED; @@ -70,7 +76,13 @@ public class AggregateTestRunner implements TestRunner { vessel.whenComplete((__, ___) -> { synchronized (monitor) { if (++current < wrapped.size()) - runNext(suite, config, wrapped.get(current).test(suite, config), aggregate); + try { + runNext(suite, config, wrapped.get(current).test(suite, config), aggregate); + } + catch (Throwable t) { + log.log(Level.SEVERE, "Failed running next suite (" + wrapped.get(current) + ")", t); + error = true; + } else aggregate.complete(null); } diff --git a/vespalib/src/tests/io/fileutil/fileutiltest.cpp b/vespalib/src/tests/io/fileutil/fileutiltest.cpp index 189bfef349b..51cf2055d33 100644 --- a/vespalib/src/tests/io/fileutil/fileutiltest.cpp +++ b/vespalib/src/tests/io/fileutil/fileutiltest.cpp @@ -1,9 +1,11 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/vespalib/io/fileutil.h> #include <vespa/vespalib/testkit/test_kit.h> +#include <filesystem> #include <iostream> #include <vector> #include <regex> +#include <system_error> #include <vespa/vespalib/util/exceptions.h> #include <vespa/vespalib/util/size_literals.h> @@ -237,9 +239,9 @@ TEST("require that vespalib::mkdir and vespalib::rmdir works") ASSERT_TRUE(mkdir("mydir/otherdir/evenmorestuff")); rmdir("mydir"); TEST_FATAL("Should not work without recursive option set"); - } catch (IoException& e) { + } catch (std::filesystem::filesystem_error& e) { //std::cerr << e.what() << "\n"; - EXPECT_EQUAL(IoException::DIRECTORY_HAVE_CONTENT, e.getType()); + EXPECT_EQUAL(make_error_code(std::errc::directory_not_empty), e.code()); } // Works with recursive option { @@ -247,19 +249,6 @@ TEST("require that vespalib::mkdir and vespalib::rmdir works") ASSERT_TRUE(!fileExists("mydir")); ASSERT_TRUE(!rmdir("mydir", true)); } - // Doesn't work on file - try{ - { - File f("myfile"); - f.open(File::CREATE); - f.write("foo", 3, 0); - } - rmdir("myfile"); - TEST_FATAL("Should have failed to run rmdir on file."); - } catch (IoException& e) { - //std::cerr << e.what() << "\n"; - EXPECT_EQUAL(IoException::ILLEGAL_PATH, e.getType()); - } // mkdir works when a path component is a symlink which points to // another directory and the final path component does not exist. diff --git a/vespalib/src/vespa/vespalib/io/fileutil.cpp b/vespalib/src/vespa/vespalib/io/fileutil.cpp index 9994e0d0d6f..2d28f07e600 100644 --- a/vespalib/src/vespa/vespalib/io/fileutil.cpp +++ b/vespalib/src/vespa/vespalib/io/fileutil.cpp @@ -5,9 +5,10 @@ #include <vespa/vespalib/stllike/asciistream.h> #include <vespa/vespalib/util/size_literals.h> #include <vespa/vespalib/util/stringfmt.h> -#include <vespa/fastos/file.h> #include <ostream> #include <cassert> +#include <filesystem> +#include <dirent.h> #include <unistd.h> #include <fcntl.h> #include <sys/stat.h> @@ -534,50 +535,12 @@ chdir(const string & directory) bool rmdir(const string & directory, bool recursive) { - string dirname(directory); - if (!dirname.empty() && *dirname.rbegin() == '/') { - dirname.resize(dirname.size() - 1); - } - if (dirname.empty()) { - LOG(debug, "rmdir(%s): Not allowing deletion of '/'.", directory.c_str()); - return false; - } + std::filesystem::path path(directory); if (recursive) { - FastOS_DirectoryScan dir(dirname.c_str()); - while (dir.ReadNext()) { - if (strcmp(dir.GetName(), "..") != 0 && - strcmp(dir.GetName(), ".") != 0) - { - string fullpath(dirname + "/" + dir.GetName()); - if (dir.IsDirectory()) { - rmdir(fullpath, true); - } else { - if (::unlink(fullpath.c_str()) != 0) { - asciistream ost; - ost << "rmdir(" << fullpath - << (recursive ? ", recursive" : "") - << "): Failed, errno(" << errno << "): " - << safeStrerror(errno); - throw IoException(ost.str(), - IoException::getErrorType(errno), - VESPA_STRLOC); - } - } - } - } - } - if (::rmdir(dirname.c_str()) == 0) { - LOG(debug, "rmdir(%s): Directory deleted.", directory.c_str()); - return true; - } - if (errno == ENOENT) { - LOG(debug, "rmdir(%s): No directory to delete.", directory.c_str()); - return false; + return std::filesystem::remove_all(path) > 0; + } else { + return std::filesystem::remove(path); } - asciistream ost; - ost << "rmdir(" << dirname << (recursive ? ", recursive" : "") - << "): Failed, errno(" << errno << "): " << safeStrerror(errno); - throw IoException(ost.str(), IoException::getErrorType(errno), VESPA_STRLOC); } FileInfo::UP diff --git a/vespalib/src/vespa/vespalib/io/fileutil.h b/vespalib/src/vespa/vespalib/io/fileutil.h index d9e1a00345f..4d2ce45358a 100644 --- a/vespalib/src/vespa/vespalib/io/fileutil.h +++ b/vespalib/src/vespa/vespalib/io/fileutil.h @@ -240,9 +240,12 @@ extern void chdir(const vespalib::string & directory); * @param directory The directory name. * @param recursive If set, remove all content of the directory to. If not * set, fail if the directory is not empty. - * @throw IoException If we failed to remove the directory. + * @throw std::filesystem::filesystem_error If we failed to remove the directory. * * @return True if directory existed, false if not. + * + * This function is deprecated. Use std::filesystem::remove or std::filesystem::remove_all + * instead. */ extern bool rmdir(const vespalib::string & directory, bool recursive = false); diff --git a/vespalib/src/vespa/vespalib/util/alloc.cpp b/vespalib/src/vespa/vespalib/util/alloc.cpp index 0a83d39ca09..8240fa2d8e6 100644 --- a/vespalib/src/vespa/vespalib/util/alloc.cpp +++ b/vespalib/src/vespa/vespalib/util/alloc.cpp @@ -228,6 +228,9 @@ getDefaultAutoAllocator(AutoAllocatorsMap & map) { } AutoAllocatorsMapWithDefault +createAutoAllocatorsWithDefault() __attribute__((noinline)); + +AutoAllocatorsMapWithDefault createAutoAllocatorsWithDefault() { AutoAllocatorsMapWithDefault tmp(createAutoAllocators(), nullptr); tmp.second = &getDefaultAutoAllocator(tmp.first); @@ -454,33 +457,33 @@ AutoAllocator::resize_inplace(PtrAndSize current, size_t newSize) const { MMapAllocator::PtrAndSize AutoAllocator::alloc(size_t sz) const { - if (useMMap(sz)) { - sz = roundUpToHugePages(sz); - return MMapAllocator::salloc(sz, nullptr); - } else { + if ( ! useMMap(sz)) { if (_alignment == 0) { return HeapAllocator::salloc(sz); } else { return AlignedHeapAllocator(_alignment).alloc(sz); } + } else { + sz = roundUpToHugePages(sz); + return MMapAllocator::salloc(sz, nullptr); } } void AutoAllocator::free(PtrAndSize alloc) const { - if (isMMapped(alloc.second)) { - return MMapAllocator::sfree(alloc); - } else { + if ( ! isMMapped(alloc.second)) { return HeapAllocator::sfree(alloc); + } else { + return MMapAllocator::sfree(alloc); } } void AutoAllocator::free(void * ptr, size_t sz) const { - if (useMMap(sz)) { - return MMapAllocator::sfree(PtrAndSize(ptr, roundUpToHugePages(sz))); - } else { + if ( ! useMMap(sz)) { return HeapAllocator::sfree(PtrAndSize(ptr, sz)); + } else { + return MMapAllocator::sfree(PtrAndSize(ptr, roundUpToHugePages(sz))); } } |