diff options
7 files changed, 38 insertions, 15 deletions
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/aws/MockResourceTagger.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/aws/MockResourceTagger.java index a74a362330b..0f05fa60db7 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/aws/MockResourceTagger.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/aws/MockResourceTagger.java @@ -15,15 +15,15 @@ import java.util.Optional; */ public class MockResourceTagger implements ResourceTagger { - Map<ZoneId, Map<HostName, Optional<ApplicationId>>> values = new HashMap<>(); + Map<ZoneId, Map<HostName, ApplicationId>> values = new HashMap<>(); @Override - public int tagResources(ZoneApi zone, Map<HostName, Optional<ApplicationId>> ownerOfHosts) { + public int tagResources(ZoneApi zone, Map<HostName, ApplicationId> ownerOfHosts) { values.put(zone.getId(), ownerOfHosts); return 0; } - public Map<ZoneId, Map<HostName, Optional<ApplicationId>>> getValues() { + public Map<ZoneId, Map<HostName, ApplicationId>> getValues() { return values; } } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/aws/ResourceTagger.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/aws/ResourceTagger.java index 61f8a57ac8b..988949456bb 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/aws/ResourceTagger.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/aws/ResourceTagger.java @@ -16,7 +16,7 @@ public interface ResourceTagger { /** * Returns number of tagged resources */ - int tagResources(ZoneApi zone, Map<HostName, Optional<ApplicationId>> ownerOfHosts); + int tagResources(ZoneApi zone, Map<HostName, ApplicationId> ownerOfHosts); static ResourceTagger empty() { return (zone, tenantOfHosts) -> 0; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java index 3b6a03a76c9..c49e3c88df3 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java @@ -198,7 +198,7 @@ public class InternalStepRunner implements StepRunner { logger) .filter(result -> { // If no tester cert, or deployment failed, propagate original result. - if (testerCertificate.isEmpty() || result != running) + if ( ! useTesterCertificate(id) || result != running) return true; // If tester cert, ensure real is deployed with the tester cert whose key was successfully deployed. return controller.jobController().run(id).get().stepStatus(deployTester).get() == succeeded @@ -828,16 +828,20 @@ public class InternalStepRunner implements StepRunner { return deployment.at().isBefore(controller.clock().instant().minus(timeout.minus(Duration.ofMinutes(1)))); } + private boolean useTesterCertificate(RunId id) { + return controller.system().isPublic() && id.type().environment().isTest(); + } + /** Returns the application package for the tester application, assembled from a generated config, fat-jar and services.xml. */ private ApplicationPackage testerPackage(RunId id) { ApplicationVersion version = controller.jobController().run(id).get().versions().targetApplication(); DeploymentSpec spec = controller.applications().requireApplication(TenantAndApplicationId.from(id.application())).deploymentSpec(); ZoneId zone = id.type().zone(controller.system()); - boolean useTesterCertificate = controller.system().isPublic() && id.type().environment().isTest(); + boolean useTesterCertificate = useTesterCertificate(id); boolean useOsgiBasedTestRuntime = testerPlatformVersion(id).isAfter(new Version(7, 247, 11)); - byte[] servicesXml = servicesXml(! controller.system().isPublic(), + byte[] servicesXml = servicesXml( ! controller.system().isPublic(), useTesterCertificate, useOsgiBasedTestRuntime, testerResourcesFor(zone, spec.requireInstance(id.application().instance())), diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java index d9d82b41e71..b622fc0bd75 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java @@ -76,9 +76,9 @@ import static java.util.stream.Collectors.toUnmodifiableList; */ public class JobController { - public static final int historyLength = 64; public static final Duration maxHistoryAge = Duration.ofDays(60); + private final int historyLength; private final Controller controller; private final CuratorDb curator; private final BufferedLogStore logs; @@ -88,6 +88,7 @@ public class JobController { private final AtomicReference<Consumer<Run>> runner = new AtomicReference<>(__ -> { }); public JobController(Controller controller) { + this.historyLength = controller.system().isCd() ? 256 : 64; this.controller = controller; this.curator = controller.curator(); this.logs = new BufferedLogStore(curator, controller.serviceRegistry().runDataStore()); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceTagMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceTagMaintainer.java index ab988bcf0ac..e0441df025b 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceTagMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceTagMaintainer.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.CloudName; import com.yahoo.config.provision.HostName; +import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.api.integration.aws.ResourceTagger; @@ -20,6 +21,9 @@ import java.util.stream.Collectors; */ public class ResourceTagMaintainer extends ControllerMaintainer { + static final ApplicationId SHARED_HOST_APPLICATION = ApplicationId.from("hosted-vespa", "shared-host", "default"); + static final ApplicationId INFRASTRUCTURE_APPLICATION = ApplicationId.from("hosted-vespa", "infrastructure", "default"); + private final ResourceTagger resourceTagger; public ResourceTagMaintainer(Controller controller, Duration interval, ResourceTagger resourceTagger) { @@ -33,7 +37,7 @@ public class ResourceTagMaintainer extends ControllerMaintainer { .ofCloud(CloudName.from("aws")) .reachable() .zones().forEach(zone -> { - Map<HostName, Optional<ApplicationId>> applicationOfHosts = getTenantOfParentHosts(zone.getId()); + Map<HostName, ApplicationId> applicationOfHosts = getTenantOfParentHosts(zone.getId()); int taggedResources = resourceTagger.tagResources(zone, applicationOfHosts); if (taggedResources > 0) log.log(Level.INFO, "Tagged " + taggedResources + " resources in " + zone.getId()); @@ -41,15 +45,21 @@ public class ResourceTagMaintainer extends ControllerMaintainer { return 1.0; } - private Map<HostName, Optional<ApplicationId>> getTenantOfParentHosts(ZoneId zoneId) { + private Map<HostName, ApplicationId> getTenantOfParentHosts(ZoneId zoneId) { return controller().serviceRegistry().configServer().nodeRepository() .list(zoneId, false) .stream() .filter(node -> node.type().isHost()) .collect(Collectors.toMap( Node::hostname, - Node::exclusiveTo, + this::getApplicationId, (node1, node2) -> node1 )); } + + private ApplicationId getApplicationId(Node node) { + if (node.type() == NodeType.host) + return node.exclusiveTo().orElse(SHARED_HOST_APPLICATION); + return INFRASTRUCTURE_APPLICATION; + } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceTagMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceTagMaintainerTest.java index 814dc2a3f50..516c28ab5cd 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceTagMaintainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceTagMaintainerTest.java @@ -17,6 +17,7 @@ import java.util.List; import java.util.Map; import java.util.Optional; +import static com.yahoo.vespa.hosted.controller.maintenance.ResourceTagMaintainer.SHARED_HOST_APPLICATION; import static org.junit.Assert.assertEquals; /** @@ -35,9 +36,9 @@ public class ResourceTagMaintainerTest { mockResourceTagger); resourceTagMaintainer.maintain(); assertEquals(2, mockResourceTagger.getValues().size()); - Map<HostName, Optional<ApplicationId>> applicationForHost = mockResourceTagger.getValues().get(ZoneId.from("prod.region-2")); - assertEquals(ApplicationId.from("t1", "a1", "i1"), applicationForHost.get(HostName.from("parentHostA.prod.region-2")).get()); - assertEquals(Optional.empty(), applicationForHost.get(HostName.from("parentHostB.prod.region-2"))); + Map<HostName, ApplicationId> applicationForHost = mockResourceTagger.getValues().get(ZoneId.from("prod.region-2")); + assertEquals(ApplicationId.from("t1", "a1", "i1"), applicationForHost.get(HostName.from("parentHostA.prod.region-2"))); + assertEquals(SHARED_HOST_APPLICATION, applicationForHost.get(HostName.from("parentHostB.prod.region-2"))); } private void setUpZones() { diff --git a/searchcore/src/tests/proton/flushengine/flushengine_test.cpp b/searchcore/src/tests/proton/flushengine/flushengine_test.cpp index d5823a8e055..13dcfb45dc9 100644 --- a/searchcore/src/tests/proton/flushengine/flushengine_test.cpp +++ b/searchcore/src/tests/proton/flushengine/flushengine_test.cpp @@ -117,6 +117,7 @@ public: search::SerialNum _oldestSerial; search::SerialNum _currentSerial; uint32_t _pendingDone; + uint32_t _taskDone; std::mutex _lock; vespalib::CountDownLatch _done; FlushDoneHistory _flushDoneHistory; @@ -131,6 +132,7 @@ public: _oldestSerial(0), _currentSerial(currentSerial), _pendingDone(0u), + _taskDone(0u), _lock(), _done(targets.size()), _flushDoneHistory() @@ -143,6 +145,11 @@ public: std::vector<IFlushTarget::SP> getFlushTargets() override { + { + std::lock_guard<std::mutex> guard(_lock); + _pendingDone += _taskDone; + _taskDone = 0; + } LOG(info, "SimpleHandler(%s)::getFlushTargets()", getName().c_str()); std::vector<IFlushTarget::SP> wrappedTargets; for (const auto &target : _targets) { @@ -154,7 +161,7 @@ public: // Called once by flush engine thread for each task done void taskDone() { std::lock_guard<std::mutex> guard(_lock); - ++_pendingDone; + ++_taskDone; } // Called by flush engine master thread after flush handler is |