diff options
Diffstat (limited to 'controller-server/src')
8 files changed, 217 insertions, 58 deletions
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/archive/CuratorArchiveBucketDb.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/archive/CuratorArchiveBucketDb.java index 2820fce38ed..3a232216d46 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/archive/CuratorArchiveBucketDb.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/archive/CuratorArchiveBucketDb.java @@ -5,9 +5,6 @@ import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.text.Text; -import com.yahoo.vespa.flags.BooleanFlag; -import com.yahoo.vespa.flags.FetchVector; -import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.api.integration.archive.ArchiveBucket; import com.yahoo.vespa.hosted.controller.api.integration.archive.ArchiveService; @@ -37,32 +34,21 @@ public class CuratorArchiveBucketDb { private final ArchiveService archiveService; private final CuratorDb curatorDb; - private final BooleanFlag enableFlag; private final SystemName system; public CuratorArchiveBucketDb(Controller controller) { this.archiveService = controller.serviceRegistry().archiveService(); this.curatorDb = controller.curator(); - this.enableFlag = Flags.ENABLE_ONPREM_TENANT_S3_ARCHIVE.bindTo(controller.flagSource()); this.system = controller.zoneRegistry().system(); } public Optional<URI> archiveUriFor(ZoneId zoneId, TenantName tenant, boolean createIfMissing) { - if ( ! enabled(zoneId, tenant)) return Optional.empty(); return getBucketNameFromCache(zoneId, tenant) .or(() -> findAndUpdateArchiveUriCache(zoneId, tenant, buckets(zoneId))) .or(() -> createIfMissing ? Optional.of(assignToBucket(zoneId, tenant)) : Optional.empty()) .map(bucketName -> URI.create(Text.format("s3://%s/%s/", bucketName, tenant.value()))); } - private boolean enabled(ZoneId zone, TenantName tenant) { - return system.isPublic() || - enableFlag - .with(FetchVector.Dimension.ZONE_ID, zone.value()) - .with(FetchVector.Dimension.TENANT_ID, tenant.value()) - .value(); - } - private String assignToBucket(ZoneId zoneId, TenantName tenant) { try (var lock = curatorDb.lockArchiveBuckets(zoneId)) { Set<ArchiveBucket> zoneBuckets = new HashSet<>(buckets(zoneId)); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java index 408b764afbe..88e9e56ad8d 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java @@ -101,6 +101,7 @@ import static com.yahoo.vespa.hosted.controller.deployment.Step.installTester; import static com.yahoo.vespa.hosted.controller.deployment.Step.report; import static java.nio.charset.StandardCharsets.UTF_8; import static java.util.Objects.requireNonNull; +import static java.util.logging.Level.FINE; import static java.util.logging.Level.INFO; import static java.util.logging.Level.WARNING; import static java.util.stream.Collectors.joining; @@ -260,7 +261,7 @@ public class InternalStepRunner implements StepRunner { return result; case LOAD_BALANCER_NOT_READY: case PARENT_HOST_NOT_READY: - logger.log(e.message()); + logger.log(e.message()); // Consider splitting these messages in summary and details, on config server. return result; case OUT_OF_CAPACITY: logger.log(e.message()); @@ -279,7 +280,7 @@ public class InternalStepRunner implements StepRunner { switch (e.type()) { case CERT_NOT_AVAILABLE: // Same as CERTIFICATE_NOT_READY above, only from the controller - logger.log("Waiting for certificate to become valid: New application, or old one has expired"); + logger.log("Waiting for certificate to become valid: new application, or old certificate has expired"); if (startTime.plus(timeouts.endpointCertificate()).isBefore(controller.clock().instant())) { logger.log(WARNING, "Controller could not validate certificate within " + timeouts.endpointCertificate() + ": " + Exceptions.toMessageString(e)); @@ -303,7 +304,7 @@ public class InternalStepRunner implements StepRunner { private Optional<RunStatus> installReal(RunId id, boolean setTheStage, DualLogger logger) { Optional<Deployment> deployment = deployment(id.application(), id.type()); if (deployment.isEmpty()) { - logger.log(INFO, "Deployment expired before installation was successful."); + logger.log("Deployment expired before installation was successful."); return Optional.of(installationFailed); } @@ -326,15 +327,32 @@ public class InternalStepRunner implements StepRunner { List<Node> parents = controller.serviceRegistry().configServer().nodeRepository().list(id.type().zone(controller.system()), NodeFilter.all() .hostnames(parentHostnames)); - NodeList nodeList = NodeList.of(nodes, parents, services.get()); boolean firstTick = run.convergenceSummary().isEmpty(); + NodeList nodeList = NodeList.of(nodes, parents, services.get()); + ConvergenceSummary summary = nodeList.summary(); if (firstTick) { // Run the first time (for each convergence step). logger.log("######## Details for all nodes ########"); logger.log(nodeList.asList().stream() .flatMap(node -> nodeDetails(node, true)) .collect(toList())); } - ConvergenceSummary summary = nodeList.summary(); + else if ( ! summary.converged()) { + logger.log("Waiting for convergence of " + summary.services() + " services across " + summary.nodes() + " nodes"); + if (summary.needPlatformUpgrade() > 0) + logger.log(summary.upgradingPlatform() + "/" + summary.needPlatformUpgrade() + " nodes upgrading platform"); + if (summary.needReboot() > 0) + logger.log(summary.rebooting() + "/" + summary.needReboot() + " nodes rebooting"); + if (summary.needRestart() > 0) + logger.log(summary.restarting() + "/" + summary.needRestart() + " nodes restarting"); + if (summary.retiring() > 0) + logger.log(summary.retiring() + " nodes retiring"); + if (summary.upgradingFirmware() > 0) + logger.log(summary.upgradingFirmware() + " nodes upgrading firmware"); + if (summary.upgradingOs() > 0) + logger.log(summary.upgradingOs() + " nodes upgrading OS"); + if (summary.needNewConfig() > 0) + logger.log(summary.needNewConfig() + " application services upgrading"); + } if (summary.converged()) { controller.jobController().locked(id, lockedRun -> lockedRun.withSummary(null)); if (endpointsAvailable(id.application(), id.type().zone(controller.system()), logger)) { @@ -398,10 +416,10 @@ public class InternalStepRunner implements StepRunner { } if ( ! firstTick) - logger.log(nodeList.expectedDown().and(nodeList.needsNewConfig()).asList().stream() - .distinct() - .flatMap(node -> nodeDetails(node, false)) - .collect(toList())); + logger.log(FINE, nodeList.expectedDown().and(nodeList.needsNewConfig()).asList().stream() + .distinct() + .flatMap(node -> nodeDetails(node, false)) + .collect(toList())); controller.jobController().locked(id, lockedRun -> { Instant noNodesDownSince = nodeList.allowedDown().size() == 0 ? lockedRun.noNodesDownSince().orElse(controller.clock().instant()) : null; @@ -1005,7 +1023,11 @@ public class InternalStepRunner implements StepRunner { } private void log(String... messages) { - log(List.of(messages)); + log(INFO, List.of(messages)); + } + + private void log(Level level, String... messages) { + log(level, List.of(messages)); } private void logAll(List<LogEntry> messages) { @@ -1013,7 +1035,11 @@ public class InternalStepRunner implements StepRunner { } private void log(List<String> messages) { - controller.jobController().log(id, step, INFO, messages); + log(INFO, messages); + } + + private void log(Level level, List<String> messages) { + controller.jobController().log(id, step, level, messages); } private void log(Level level, String message) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveAccessMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveAccessMaintainer.java index 8156a2e9f3b..e3f69f59d24 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveAccessMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveAccessMaintainer.java @@ -6,6 +6,7 @@ import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.jdisc.Metric; import com.yahoo.vespa.hosted.controller.Controller; +import com.yahoo.vespa.hosted.controller.api.integration.archive.ArchiveBucket; import com.yahoo.vespa.hosted.controller.api.integration.archive.ArchiveService; import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry; import com.yahoo.vespa.hosted.controller.archive.CuratorArchiveBucketDb; @@ -15,8 +16,11 @@ import com.yahoo.vespa.hosted.controller.tenant.Tenant; import java.time.Duration; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.stream.Collectors; +import static java.util.stream.Collectors.groupingBy; + /** * Update archive access permissions with roles from tenants * @@ -52,10 +56,20 @@ public class ArchiveAccessMaintainer extends ControllerMaintainer { try { var tenantArchiveAccessRoles = cloudTenantArchiveExternalAccessRoles(); archiveBucketDb.buckets(zoneId).forEach(archiveBucket -> - archiveService.updateBucketAndKeyPolicy(zoneId, archiveBucket, + archiveService.updateBucketPolicy(zoneId, archiveBucket, Maps.filterEntries(tenantArchiveAccessRoles, entry -> archiveBucket.tenants().contains(entry.getKey()))) ); + Map<String, List<ArchiveBucket>> bucketsPerKey = archiveBucketDb.buckets(zoneId).stream() + .collect(groupingBy(ArchiveBucket::keyArn)); + bucketsPerKey.forEach((keyArn, buckets) -> { + Set<String> authorizedIamRolesForKey = buckets.stream() + .flatMap(b -> b.tenants().stream()) + .filter(tenantArchiveAccessRoles::containsKey) + .map(tenantArchiveAccessRoles::get) + .collect(Collectors.toSet()); + archiveService.updateKeyPolicy(zoneId, keyArn, authorizedIamRolesForKey); + }); } catch (Exception e) { throw new RuntimeException("Failed to maintain archive access in " + zoneId.value(), e); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveAccessMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveAccessMaintainerTest.java index fe8dc0b1e29..df2b462914e 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveAccessMaintainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveAccessMaintainerTest.java @@ -15,6 +15,7 @@ import org.junit.Test; import java.time.Duration; import java.util.Map; import java.util.Optional; +import java.util.Set; import java.util.stream.Collectors; import static org.junit.Assert.assertEquals; @@ -39,10 +40,12 @@ public class ArchiveAccessMaintainerTest { var testBucket = new ArchiveBucket("bucketName", "keyArn").withTenant(tenant1); MockArchiveService archiveService = (MockArchiveService) tester.controller().serviceRegistry().archiveService(); - assertNull(archiveService.authorizedIamRoles.get(testBucket)); + assertNull(archiveService.authorizedIamRolesForBucket.get(testBucket)); + assertNull(archiveService.authorizedIamRolesForKey.get(testBucket.keyArn())); MockMetric metric = new MockMetric(); new ArchiveAccessMaintainer(tester.controller(), metric, Duration.ofMinutes(10)).maintain(); - assertEquals(Map.of(tenant1, tenant1role), archiveService.authorizedIamRoles.get(testBucket)); + assertEquals(Map.of(tenant1, tenant1role), archiveService.authorizedIamRolesForBucket.get(testBucket)); + assertEquals(Set.of(tenant1role), archiveService.authorizedIamRolesForKey.get(testBucket.keyArn())); var expected = Map.of("archive.bucketCount", tester.controller().zoneRegistry().zones().all().ids().stream() diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java index 2ae755ac8fe..4935ab22586 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java @@ -192,7 +192,7 @@ public class JobControllerApiHandlerHelperTest { private void compare(HttpResponse response, String expected) throws IOException { ByteArrayOutputStream baos = new ByteArrayOutputStream(); response.render(baos); - JsonTestHelper.assertJsonEquals(expected, baos.toString()); + JsonTestHelper.assertJsonEquals(baos.toString(), expected); } private void assertResponse(HttpResponse response, String fileName) { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/staging-test-log.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/staging-test-log.json index 0525f059dd0..6b1d48f4a08 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/staging-test-log.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/staging-test-log.json @@ -117,16 +117,26 @@ { "at": 14503000, "type": "info", - "message": "host-tenant:application:default-staging.us-east-3: unorchestrated" + "message": "Waiting for convergence of 1 services across 1 nodes" }, { "at": 14503000, "type": "info", + "message": "1 application services upgrading" + }, + { + "at": 14503000, + "type": "debug", + "message": "host-tenant:application:default-staging.us-east-3: unorchestrated" + }, + { + "at": 14503000, + "type": "debug", "message": "--- platform vespa/vespa:6.1" }, { "at": 14503000, - "type": "info", + "type": "debug", "message": "--- container on port 43 has config generation 1, wanted is 2" }, { @@ -150,7 +160,7 @@ } ] }, - "lastId": 27, + "lastId": 29, "steps": { "deployTester": { "status": "succeeded", diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/system-test-details.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/system-test-details.json index 7ee3952a8b5..065ca3c1020 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/system-test-details.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/system-test-details.json @@ -137,91 +137,151 @@ { "at": "(ignore)", "type": "info", - "message": "host-tenant1:application1:instance1-test.us-east-1: unorchestrated" + "message": "Waiting for convergence of 1 services across 1 nodes" }, { "at": "(ignore)", "type": "info", + "message": "1 application services upgrading" + }, + { + "at": "(ignore)", + "type": "debug", + "message": "host-tenant1:application1:instance1-test.us-east-1: unorchestrated" + }, + { + "at": "(ignore)", + "type": "debug", "message": "--- platform vespa/vespa:6.1" }, { "at": "(ignore)", - "type": "info", + "type": "debug", "message": "--- container on port 43 has config generation 1, wanted is 2" }, { "at": "(ignore)", "type": "info", - "message": "host-tenant1:application1:instance1-test.us-east-1: unorchestrated" + "message": "Waiting for convergence of 1 services across 1 nodes" }, { "at": "(ignore)", "type": "info", + "message": "1 application services upgrading" + }, + { + "at": "(ignore)", + "type": "debug", + "message": "host-tenant1:application1:instance1-test.us-east-1: unorchestrated" + }, + { + "at": "(ignore)", + "type": "debug", "message": "--- platform vespa/vespa:6.1" }, { "at": "(ignore)", - "type": "info", + "type": "debug", "message": "--- container on port 43 has config generation 1, wanted is 2" }, { "at": "(ignore)", "type": "info", - "message": "host-tenant1:application1:instance1-test.us-east-1: unorchestrated" + "message": "Waiting for convergence of 1 services across 1 nodes" }, { "at": "(ignore)", "type": "info", + "message": "1 application services upgrading" + }, + { + "at": "(ignore)", + "type": "debug", + "message": "host-tenant1:application1:instance1-test.us-east-1: unorchestrated" + }, + { + "at": "(ignore)", + "type": "debug", "message": "--- platform vespa/vespa:6.1" }, { "at": "(ignore)", - "type": "info", + "type": "debug", "message": "--- container on port 43 has config generation 1, wanted is 2" }, { "at": "(ignore)", "type": "info", - "message": "host-tenant1:application1:instance1-test.us-east-1: unorchestrated" + "message": "Waiting for convergence of 1 services across 1 nodes" }, { "at": "(ignore)", "type": "info", + "message": "1 application services upgrading" + }, + { + "at": "(ignore)", + "type": "debug", + "message": "host-tenant1:application1:instance1-test.us-east-1: unorchestrated" + }, + { + "at": "(ignore)", + "type": "debug", "message": "--- platform vespa/vespa:6.1" }, { "at": "(ignore)", - "type": "info", + "type": "debug", "message": "--- container on port 43 has config generation 1, wanted is 2" }, { "at": "(ignore)", "type": "info", - "message": "host-tenant1:application1:instance1-test.us-east-1: unorchestrated" + "message": "Waiting for convergence of 1 services across 1 nodes" }, { "at": "(ignore)", "type": "info", + "message": "1 application services upgrading" + }, + { + "at": "(ignore)", + "type": "debug", + "message": "host-tenant1:application1:instance1-test.us-east-1: unorchestrated" + }, + { + "at": "(ignore)", + "type": "debug", "message": "--- platform vespa/vespa:6.1" }, { "at": "(ignore)", - "type": "info", + "type": "debug", "message": "--- container on port 43 has config generation 1, wanted is 2" }, { "at": "(ignore)", "type": "info", - "message": "host-tenant1:application1:instance1-test.us-east-1: unorchestrated" + "message": "Waiting for convergence of 1 services across 1 nodes" }, { "at": "(ignore)", "type": "info", + "message": "1 application services upgrading" + }, + { + "at": "(ignore)", + "type": "debug", + "message": "host-tenant1:application1:instance1-test.us-east-1: unorchestrated" + }, + { + "at": "(ignore)", + "type": "debug", "message": "--- platform vespa/vespa:6.1" }, { "at": "(ignore)", - "type": "info", + "type": "debug", "message": "--- container on port 43 has config generation 1, wanted is 2" }, { @@ -294,7 +354,7 @@ } ] }, - "lastId": 54, + "lastId": 66, "steps": { "deployTester": { "status": "succeeded", diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/system-test-log.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/system-test-log.json index 6c6092cbd88..66173ec4976 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/system-test-log.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/system-test-log.json @@ -132,91 +132,151 @@ { "at": 0, "type": "info", - "message": "host-tenant:application:default-test.us-east-1: unorchestrated" + "message": "Waiting for convergence of 1 services across 1 nodes" }, { "at": 0, "type": "info", + "message": "1 application services upgrading" + }, + { + "at": 0, + "type": "debug", + "message": "host-tenant:application:default-test.us-east-1: unorchestrated" + }, + { + "at": 0, + "type": "debug", "message": "--- platform vespa/vespa:6.1" }, { "at": 0, - "type": "info", + "type": "debug", "message": "--- container on port 43 has config generation 1, wanted is 2" }, { "at": 0, "type": "info", - "message": "host-tenant:application:default-test.us-east-1: unorchestrated" + "message": "Waiting for convergence of 1 services across 1 nodes" }, { "at": 0, "type": "info", + "message": "1 application services upgrading" + }, + { + "at": 0, + "type": "debug", + "message": "host-tenant:application:default-test.us-east-1: unorchestrated" + }, + { + "at": 0, + "type": "debug", "message": "--- platform vespa/vespa:6.1" }, { "at": 0, - "type": "info", + "type": "debug", "message": "--- container on port 43 has config generation 1, wanted is 2" }, { "at": 0, "type": "info", - "message": "host-tenant:application:default-test.us-east-1: unorchestrated" + "message": "Waiting for convergence of 1 services across 1 nodes" }, { "at": 0, "type": "info", + "message": "1 application services upgrading" + }, + { + "at": 0, + "type": "debug", + "message": "host-tenant:application:default-test.us-east-1: unorchestrated" + }, + { + "at": 0, + "type": "debug", "message": "--- platform vespa/vespa:6.1" }, { "at": 0, - "type": "info", + "type": "debug", "message": "--- container on port 43 has config generation 1, wanted is 2" }, { "at": 0, "type": "info", - "message": "host-tenant:application:default-test.us-east-1: unorchestrated" + "message": "Waiting for convergence of 1 services across 1 nodes" }, { "at": 0, "type": "info", + "message": "1 application services upgrading" + }, + { + "at": 0, + "type": "debug", + "message": "host-tenant:application:default-test.us-east-1: unorchestrated" + }, + { + "at": 0, + "type": "debug", "message": "--- platform vespa/vespa:6.1" }, { "at": 0, - "type": "info", + "type": "debug", "message": "--- container on port 43 has config generation 1, wanted is 2" }, { "at": 0, "type": "info", - "message": "host-tenant:application:default-test.us-east-1: unorchestrated" + "message": "Waiting for convergence of 1 services across 1 nodes" }, { "at": 0, "type": "info", + "message": "1 application services upgrading" + }, + { + "at": 0, + "type": "debug", + "message": "host-tenant:application:default-test.us-east-1: unorchestrated" + }, + { + "at": 0, + "type": "debug", "message": "--- platform vespa/vespa:6.1" }, { "at": 0, - "type": "info", + "type": "debug", "message": "--- container on port 43 has config generation 1, wanted is 2" }, { "at": 0, "type": "info", - "message": "host-tenant:application:default-test.us-east-1: unorchestrated" + "message": "Waiting for convergence of 1 services across 1 nodes" }, { "at": 0, "type": "info", + "message": "1 application services upgrading" + }, + { + "at": 0, + "type": "debug", + "message": "host-tenant:application:default-test.us-east-1: unorchestrated" + }, + { + "at": 0, + "type": "debug", "message": "--- platform vespa/vespa:6.1" }, { "at": 0, - "type": "info", + "type": "debug", "message": "--- container on port 43 has config generation 1, wanted is 2" }, { @@ -289,7 +349,7 @@ } ] }, - "lastId": 54, + "lastId": 66, "steps": { "deployTester": { "status": "succeeded", |