diff options
61 files changed, 358 insertions, 600 deletions
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java index cd6a451ecf8..f9d5e9ba9b9 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java @@ -12,7 +12,6 @@ import com.yahoo.config.application.api.ApplicationMetaData; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Environment; -import com.yahoo.config.provision.Deployer; import com.yahoo.config.provision.HostFilter; import com.yahoo.config.provision.Provisioner; import com.yahoo.config.provision.TenantName; @@ -56,10 +55,15 @@ import java.net.URI; import java.time.Clock; import java.time.Duration; import java.time.Instant; +import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Optional; +import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; import java.util.logging.Level; import java.util.logging.Logger; @@ -413,15 +417,6 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye return session.getSessionId(); } - void redeployAllApplications(Deployer deployer) throws InterruptedException { - ExecutorService deploymentExecutor = Executors.newFixedThreadPool(configserverConfig.numParallelTenantLoaders(), - new DaemonThreadFactory("redeploy apps")); - tenants.getAllTenants().forEach(tenant -> listApplicationIds(tenant) - .forEach(applicationId -> redeployApplication(applicationId, deployer, deploymentExecutor))); - deploymentExecutor.shutdown(); - deploymentExecutor.awaitTermination(365, TimeUnit.DAYS); // Timeout should never happen - } - private static void cleanupApplicationDirectory(File tempDir, DeployLogger logger) { logger.log(LogLevel.DEBUG, "Deleting tmp dir '" + tempDir + "'"); if (!IOUtils.recursiveDeleteDir(tempDir)) { @@ -429,16 +424,24 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye } } - private void redeployApplication(ApplicationId applicationId, Deployer deployer, ExecutorService deploymentExecutor) { - log.log(LogLevel.DEBUG, () -> "Redeploying " + applicationId); - deployer.deployFromLocalActive(applicationId) - .ifPresent(deployment -> deploymentExecutor.execute(() -> { - try { - deployment.activate(); - } catch (RuntimeException e) { - log.log(LogLevel.ERROR, "Redeploying " + applicationId + " failed", e); - } - })); + void redeployAllApplications() throws InterruptedException { + ExecutorService executor = Executors.newFixedThreadPool(configserverConfig.numParallelTenantLoaders(), + new DaemonThreadFactory("redeploy apps")); + // Keep track of deployment per application + Map<ApplicationId, Future<?>> futures = new HashMap<>(); + tenants.getAllTenants() + .forEach(tenant -> listApplicationIds(tenant) + .forEach(appId -> deployFromLocalActive(appId).ifPresent( + deployment -> futures.put(appId,executor.submit(deployment::activate))))); + for (Map.Entry<ApplicationId, Future<?>> f : futures.entrySet()) { + try { + f.getValue().get(); + } catch (ExecutionException e) { + throw new RuntimeException("Redeploying of " + f.getKey() + " failed", e); + } + } + executor.shutdown(); + executor.awaitTermination(365, TimeUnit.DAYS); // Timeout should never happen } public ApplicationFile getApplicationFileFromSession(TenantName tenantName, long sessionId, String path, LocalSession.Mode mode) { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java index 6ab98f5af1c..9793a441355 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java @@ -3,7 +3,6 @@ package com.yahoo.vespa.config.server; import com.google.inject.Inject; import com.yahoo.component.AbstractComponent; -import com.yahoo.config.provision.Deployer; import com.yahoo.container.jdisc.state.StateMonitor; import com.yahoo.log.LogLevel; import com.yahoo.vespa.config.server.rpc.RpcServer; @@ -22,7 +21,6 @@ public class ConfigServerBootstrap extends AbstractComponent implements Runnable private final ApplicationRepository applicationRepository; private final RpcServer server; private final Thread serverThread; - private final Deployer deployer; private final VersionState versionState; private final StateMonitor stateMonitor; @@ -31,13 +29,23 @@ public class ConfigServerBootstrap extends AbstractComponent implements Runnable @SuppressWarnings("WeakerAccess") @Inject public ConfigServerBootstrap(ApplicationRepository applicationRepository, RpcServer server, - Deployer deployer, VersionState versionState, StateMonitor stateMonitor) { + VersionState versionState, StateMonitor stateMonitor) { + this(applicationRepository, server, versionState, stateMonitor, true); + } + + // For testing only + ConfigServerBootstrap(ApplicationRepository applicationRepository, RpcServer server, + VersionState versionState, StateMonitor stateMonitor, boolean startMainThread) { this.applicationRepository = applicationRepository; this.server = server; - this.deployer = deployer; this.versionState = versionState; this.stateMonitor = stateMonitor; this.serverThread = new Thread(this, "configserver main"); + if (startMainThread) + start(); + } + + private void start() { serverThread.start(); } @@ -55,22 +63,35 @@ public class ConfigServerBootstrap extends AbstractComponent implements Runnable @Override public void run() { if (versionState.isUpgraded()) { - log.log(LogLevel.INFO, "Configserver upgraded from " + versionState.storedVersion() + " to " + log.log(LogLevel.INFO, "Configserver upgrading from " + versionState.storedVersion() + " to " + versionState.currentVersion() + ". Redeploying all applications"); try { - applicationRepository.redeployAllApplications(deployer); - } catch (InterruptedException e) { - throw new RuntimeException("Redeploying applications failed", e); + applicationRepository.redeployAllApplications(); + versionState.saveNewVersion(); + log.log(LogLevel.INFO, "All applications redeployed successfully"); + } catch (Exception e) { + log.log(LogLevel.ERROR, "Redeployment of applications failed", e); + return; // Status will not be set to 'up' since we return here } - log.log(LogLevel.INFO, "All applications redeployed"); } - versionState.saveNewVersion(); stateMonitor.status(StateMonitor.Status.up); - log.log(LogLevel.DEBUG, "Starting RPC server"); + log.log(LogLevel.INFO, "Starting RPC server"); server.run(); - log.log(LogLevel.DEBUG, "RPC server stopped"); + do { + try { + Thread.sleep(1000); + } catch (InterruptedException e) { + log.log(LogLevel.ERROR, "Got interrupted", e); + break; + } + } while (server.isRunning()); + log.log(LogLevel.INFO, "RPC server stopped"); stateMonitor.status(StateMonitor.Status.down); } + StateMonitor.Status status() { + return stateMonitor.status(); + } + } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/RpcServer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/RpcServer.java index a5f288bf254..d42468ec8fd 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/RpcServer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/RpcServer.java @@ -100,6 +100,7 @@ public class RpcServer implements Runnable, ReloadListener, TenantListener { private final ThreadPoolExecutor executorService; private final FileDownloader downloader; private volatile boolean allTenantsLoaded = false; + private boolean isRunning = false; /** * Creates an RpcServer listening on the specified <code>port</code>. @@ -168,6 +169,7 @@ public class RpcServer implements Runnable, ReloadListener, TenantListener { log.log(LogLevel.INFO, "Rpc server listening on port " + spec.port()); try { Acceptor acceptor = supervisor.listen(spec); + isRunning = true; supervisor.transport().join(); acceptor.shutdown().join(); } catch (ListenFailedException e) { @@ -185,6 +187,11 @@ public class RpcServer implements Runnable, ReloadListener, TenantListener { } delayedConfigResponses.stop(); supervisor.transport().shutdown().join(); + isRunning = false; + } + + public boolean isRunning() { + return isRunning; } /** diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/ConfigServerBootstrapTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/ConfigServerBootstrapTest.java index 5d573323bb6..89b82058b6b 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/ConfigServerBootstrapTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/ConfigServerBootstrapTest.java @@ -1,132 +1,136 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server; +import com.google.common.io.Files; import com.yahoo.cloud.config.ConfigserverConfig; -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.ApplicationName; -import com.yahoo.config.provision.InstanceName; -import com.yahoo.config.provision.TenantName; import com.yahoo.container.jdisc.config.HealthMonitorConfig; import com.yahoo.container.jdisc.state.StateMonitor; -import com.yahoo.io.IOUtils; import com.yahoo.jdisc.core.SystemTimer; -import com.yahoo.vespa.config.server.deploy.MockDeployer; -import com.yahoo.vespa.config.server.host.HostRegistries; -import com.yahoo.vespa.config.server.http.SessionHandlerTest; -import com.yahoo.vespa.config.server.monitoring.Metrics; -import com.yahoo.vespa.config.server.rpc.UncompressedConfigResponseFactory; -import com.yahoo.vespa.config.server.tenant.Tenant; -import com.yahoo.vespa.config.server.tenant.TenantRequestHandler; -import com.yahoo.vespa.config.server.tenant.TestWithTenant; +import com.yahoo.vespa.config.server.deploy.DeployTester; +import com.yahoo.vespa.config.server.rpc.RpcServer; import com.yahoo.vespa.config.server.version.VersionState; -import org.hamcrest.core.Is; -import org.junit.Before; +import org.junit.Ignore; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; import java.io.File; -import java.io.FileReader; -import java.time.Clock; -import java.util.ArrayList; +import java.nio.file.Paths; +import java.time.Duration; +import java.time.Instant; +import java.util.function.BooleanSupplier; -import static org.hamcrest.Matchers.is; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; /** - * @author lulf - * @since 5.1 + * @author Ulf Lilleengen + * @author Harald Musum */ -public class ConfigServerBootstrapTest extends TestWithTenant { - private final TenantName tenant1 = TenantName.from("tenant1"); - private final TenantName tenant2 = TenantName.from("tenant2"); - - private ApplicationRepository applicationRepository; +public class ConfigServerBootstrapTest { @Rule public TemporaryFolder temporaryFolder = new TemporaryFolder(); - @Before - public void setup() throws Exception { - tenants.addTenant(tenant1); - tenants.addTenant(tenant2); - - applicationRepository = new ApplicationRepository(tenants, - new SessionHandlerTest.MockProvisioner(), - Clock.systemUTC()); - } - @Test - public void testConfigServerBootstrap() throws Exception { - File versionFile = temporaryFolder.newFile(); - ConfigserverConfig.Builder config = new ConfigserverConfig.Builder(); - MockTenantRequestHandler myServer = new MockTenantRequestHandler(Metrics.createTestMetrics()); - MockRpc rpc = new MockRpc(new ConfigserverConfig(config).rpcport()); + @Ignore // TODO: An issue with how MockCurator.MockLock is implemented make this not work (it will hang + // not being able to acquire activate lock in ConfigServerBootstrap + public void testBootStrap() throws Exception { + ConfigserverConfig configserverConfig = createConfigserverConfig(); + DeployTester tester = new DeployTester("src/test/apps/hosted/", configserverConfig); + tester.deployApp("myApp", "4.5.6", Instant.now()); - assertFalse(myServer.started); - assertFalse(myServer.stopped); + File versionFile = temporaryFolder.newFile(); VersionState versionState = new VersionState(versionFile); assertTrue(versionState.isUpgraded()); - ConfigServerBootstrap bootstrap = - new ConfigServerBootstrap(applicationRepository, rpc, new MockDeployer(), versionState, - new StateMonitor(new HealthMonitorConfig(new HealthMonitorConfig.Builder()), new SystemTimer())); - waitUntilStarted(rpc, 60000); - assertFalse(versionState.isUpgraded()); - assertThat(versionState.currentVersion(), is(versionState.storedVersion())); - assertThat(IOUtils.readAll(new FileReader(versionFile)), is(versionState.currentVersion().toSerializedForm())); - assertTrue(rpc.started); - assertFalse(rpc.stopped); + + RpcServer rpcServer = createRpcServer(configserverConfig); + ConfigServerBootstrap bootstrap = new ConfigServerBootstrap(tester.applicationRepository(), rpcServer, versionState, createStateMonitor()); + waitUntil(() -> bootstrap.status() == StateMonitor.Status.up, "failed waiting for status 'up'"); + waitUntil(rpcServer::isRunning, "failed waiting for Rpc server running"); + bootstrap.deconstruct(); - assertTrue(rpc.started); - assertTrue(rpc.stopped); + assertEquals(StateMonitor.Status.down, bootstrap.status()); + assertFalse(rpcServer.isRunning()); } @Test - public void testTenantRedeployment() throws Exception { - MockDeployer deployer = new MockDeployer(); - Tenant tenant = tenants.getTenant(tenant1); - ApplicationId id = ApplicationId.from(tenant1, ApplicationName.defaultName(), InstanceName.defaultName()); - tenant.getApplicationRepo().createPutApplicationTransaction(id, 3).commit(); - applicationRepository.redeployAllApplications(deployer); - assertThat(deployer.lastDeployed, Is.is(id)); + public void testBootStrapWhenRedeploymentFails() throws Exception { + ConfigserverConfig configserverConfig = createConfigserverConfig(); + DeployTester tester = new DeployTester("src/test/apps/hosted/", configserverConfig); + tester.deployApp("myApp", "4.5.6", Instant.now()); + + File versionFile = temporaryFolder.newFile(); + VersionState versionState = new VersionState(versionFile); + assertTrue(versionState.isUpgraded()); + + // Manipulate application package so that it will fail deployment when config server starts + java.nio.file.Files.delete(Paths.get(configserverConfig.configServerDBDir()) + .resolve("tenants/") + .resolve(tester.tenant().getName().value()) + .resolve("sessions/2/services.xml")); + + RpcServer rpcServer = createRpcServer(configserverConfig); + ConfigServerBootstrap bootstrap = new ConfigServerBootstrap(tester.applicationRepository(), rpcServer, versionState, + createStateMonitor(), false /* do not call run method */); + // Call method directly, to be sure that it is finished redeploying all applications and we can check status + bootstrap.run(); + // App is invalid, so bootstrapping was unsuccessful. Status should be 'initializing' and rpc server should not be running + assertEquals(StateMonitor.Status.initializing, bootstrap.status()); + assertFalse(rpcServer.isRunning()); } - private void waitUntilStarted(MockRpc server, long timeout) throws InterruptedException { - long start = System.currentTimeMillis(); - while ((System.currentTimeMillis() - start) < timeout) { - if (server.started) + private void waitUntil(BooleanSupplier booleanSupplier, String messageIfWaitingFails) throws InterruptedException { + Duration timeout = Duration.ofSeconds(60); + Instant endTime = Instant.now().plus(timeout); + while (Instant.now().isBefore(endTime)) { + if (booleanSupplier.getAsBoolean()) return; Thread.sleep(10); } + throw new RuntimeException(messageIfWaitingFails); + } + + private MockRpc createRpcServer(ConfigserverConfig configserverConfig) { + return new MockRpc(configserverConfig.rpcport()); } - public static class MockTenantRequestHandler extends TenantRequestHandler { - public volatile boolean started = false; - public volatile boolean stopped = false; + private StateMonitor createStateMonitor() { + return new StateMonitor(new HealthMonitorConfig(new HealthMonitorConfig.Builder().initialStatus("initializing")), + new SystemTimer()); + } - public MockTenantRequestHandler(Metrics statistics) { - super(statistics, TenantName.from("testTenant"), new ArrayList<>(), new UncompressedConfigResponseFactory(), new HostRegistries()); - } + private static ConfigserverConfig createConfigserverConfig() { + return new ConfigserverConfig(new ConfigserverConfig.Builder() + .configServerDBDir(Files.createTempDir().getAbsolutePath()) + .configDefinitionsDir(Files.createTempDir().getAbsolutePath()) + .hostedVespa(true) + .multitenant(true)); } public static class MockRpc extends com.yahoo.vespa.config.server.rpc.MockRpc { - public volatile boolean started = false; - public volatile boolean stopped = false; - public MockRpc(int port) { + volatile boolean isRunning = false; + + MockRpc(int port) { super(port); } @Override public void run() { - started = true; + isRunning = true; } @Override public void stop() { - stopped = true; + isRunning = false; + } + + @Override + public boolean isRunning() { + return isRunning; } } + } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/MockRpc.java b/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/MockRpc.java index 4c2a4b56751..5a9735f774a 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/MockRpc.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/MockRpc.java @@ -20,8 +20,7 @@ import java.util.concurrent.CompletionService; /** * Test utility mocking an RPC server. * - * @author lulf - * @since 5.25 + * @author Ulf Lilleengen */ public class MockRpc extends RpcServer { @@ -111,4 +110,9 @@ public class MockRpc extends RpcServer { @Override public boolean allTenantsLoaded() { return true; } + @Override + public boolean isRunning() { + return true; + } + } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/TenantResource.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/TenantResource.java index 8db6f982ef6..06a8f8371ab 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/TenantResource.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/TenantResource.java @@ -3,7 +3,6 @@ package com.yahoo.vespa.hosted.controller.api.application.v4; import com.yahoo.vespa.hosted.controller.api.application.v4.model.TenantCreateOptions; import com.yahoo.vespa.hosted.controller.api.application.v4.model.TenantInfo; -import com.yahoo.vespa.hosted.controller.api.application.v4.model.TenantMigrateOptions; import com.yahoo.vespa.hosted.controller.api.application.v4.model.TenantUpdateOptions; import com.yahoo.vespa.hosted.controller.api.application.v4.model.TenantWithApplications; @@ -41,7 +40,4 @@ public interface TenantResource { @Path(ApplicationResource.API_PATH) ApplicationResource application(); - @PUT - @Path("migrateTenantToAthens") - TenantInfo migrateTenantToAthens(TenantMigrateOptions tenantOptions); } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/TenantCreateOptions.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/TenantCreateOptions.java index aaa80ad73a2..b99c3182f03 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/TenantCreateOptions.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/TenantCreateOptions.java @@ -6,7 +6,6 @@ import com.fasterxml.jackson.annotation.JsonInclude; import com.yahoo.vespa.athenz.api.AthenzDomain; import com.yahoo.vespa.hosted.controller.api.identifiers.Property; import com.yahoo.vespa.hosted.controller.api.identifiers.PropertyId; -import com.yahoo.vespa.hosted.controller.api.identifiers.UserGroup; /** * @author bjorncs @@ -17,16 +16,9 @@ public class TenantCreateOptions { public AthenzDomain athensDomain; public Property property; public PropertyId propertyId; - public UserGroup userGroup; public TenantCreateOptions() {} - public TenantCreateOptions(UserGroup userGroup, Property property, PropertyId propertyId) { - this.userGroup = userGroup; - this.property = property; - this.propertyId = propertyId; - } - public TenantCreateOptions(AthenzDomain athensDomain, Property property, PropertyId propertyId) { this.athensDomain = athensDomain; this.property = property; diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/TenantMetaData.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/TenantMetaData.java index da088c76fda..a8bda2e5f10 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/TenantMetaData.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/TenantMetaData.java @@ -6,7 +6,6 @@ import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonInclude.Include; import com.yahoo.vespa.athenz.api.AthenzDomain; import com.yahoo.vespa.hosted.controller.api.identifiers.Property; -import com.yahoo.vespa.hosted.controller.api.identifiers.UserGroup; import java.util.Optional; @@ -19,18 +18,15 @@ public class TenantMetaData { public TenantType type; public Optional<AthenzDomain> athensDomain; public Optional<Property> property; - public Optional<UserGroup> userGroup; // Required for Jackson deserialization public TenantMetaData() {} public TenantMetaData(TenantType type, Optional<AthenzDomain> athensDomain, - Optional<Property> property, - Optional<UserGroup> userGroup) { + Optional<Property> property) { this.type = type; this.athensDomain = athensDomain; this.property = property; - this.userGroup = userGroup; } } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/TenantMigrateOptions.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/TenantMigrateOptions.java deleted file mode 100644 index 39561c31ed8..00000000000 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/TenantMigrateOptions.java +++ /dev/null @@ -1,22 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.api.application.v4.model; - -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.fasterxml.jackson.annotation.JsonInclude; -import com.yahoo.vespa.athenz.api.AthenzDomain; - -/** - * @author bjorncs - */ -@JsonIgnoreProperties(ignoreUnknown = true) -@JsonInclude(value = JsonInclude.Include.NON_NULL) -public class TenantMigrateOptions { - - public AthenzDomain athensDomain; - - public TenantMigrateOptions() {} - - public TenantMigrateOptions(AthenzDomain athensDomain) { - this.athensDomain = athensDomain; - } -} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/TenantType.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/TenantType.java index 2c543af7bf8..9f89da717be 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/TenantType.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/TenantType.java @@ -5,7 +5,6 @@ package com.yahoo.vespa.hosted.controller.api.application.v4.model; * @author bjorncs */ public enum TenantType { - OPSDB, USER, ATHENS } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/TenantUpdateOptions.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/TenantUpdateOptions.java index f441fdd6ec5..4e8c164a64d 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/TenantUpdateOptions.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/TenantUpdateOptions.java @@ -7,7 +7,6 @@ import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; import com.yahoo.vespa.athenz.api.AthenzDomain; import com.yahoo.vespa.hosted.controller.api.identifiers.Property; -import com.yahoo.vespa.hosted.controller.api.identifiers.UserGroup; import java.util.Objects; import java.util.Optional; @@ -20,14 +19,11 @@ import java.util.Optional; @JsonInclude(value = JsonInclude.Include.NON_ABSENT) public class TenantUpdateOptions { public final Property property; - public final Optional<UserGroup> userGroup; public final Optional<AthenzDomain> athensDomain; @JsonCreator public TenantUpdateOptions(@JsonProperty("property") Property property, - @JsonProperty("userGroup") Optional<UserGroup> userGroup, @JsonProperty("athensDomain") Optional<AthenzDomain> athensDomain) { - this.userGroup = userGroup; this.property = property; this.athensDomain = athensDomain; } @@ -38,20 +34,18 @@ public class TenantUpdateOptions { if (o == null || getClass() != o.getClass()) return false; TenantUpdateOptions that = (TenantUpdateOptions) o; return Objects.equals(property, that.property) && - Objects.equals(userGroup, that.userGroup) && Objects.equals(athensDomain, that.athensDomain); } @Override public int hashCode() { - return Objects.hash(property, userGroup, athensDomain); + return Objects.hash(property, athensDomain); } @Override public String toString() { return "TenantUpdateOptions{" + "property=" + property + - ", userGroup=" + userGroup + ", athensDomain=" + athensDomain + '}'; } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/TenantWithApplications.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/TenantWithApplications.java index 633547cfaca..0c40e8192d6 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/TenantWithApplications.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/TenantWithApplications.java @@ -5,7 +5,6 @@ import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonInclude; import com.yahoo.vespa.athenz.api.AthenzDomain; import com.yahoo.vespa.hosted.controller.api.identifiers.Property; -import com.yahoo.vespa.hosted.controller.api.identifiers.UserGroup; import java.util.List; @@ -19,7 +18,6 @@ public class TenantWithApplications { public TenantType type; public AthenzDomain athensDomain; public Property property; - public UserGroup userGroup; public List<ApplicationReference> applications; public TenantWithApplications() {} @@ -28,12 +26,10 @@ public class TenantWithApplications { TenantType type, AthenzDomain athensDomain, Property property, - UserGroup userGroup, List<ApplicationReference> applications) { this.type = type; this.athensDomain = athensDomain; this.property = property; - this.userGroup = userGroup; this.applications = applications; } } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/entity/EntityService.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/entity/EntityService.java index fc242a360f6..067686269b2 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/entity/EntityService.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/entity/EntityService.java @@ -3,11 +3,8 @@ package com.yahoo.vespa.hosted.controller.api.integration.entity; import com.yahoo.vespa.hosted.controller.api.identifiers.Property; import com.yahoo.vespa.hosted.controller.api.identifiers.PropertyId; -import com.yahoo.vespa.hosted.controller.api.identifiers.UserGroup; -import com.yahoo.vespa.hosted.controller.api.identifiers.UserId; import java.util.Map; -import java.util.Set; /** * A service which provides access to business-specific entities. @@ -19,10 +16,4 @@ public interface EntityService { /** List all properties known by the service */ Map<PropertyId, Property> listProperties(); - /** List all groups of which user is a member */ - Set<UserGroup> getUserGroups(UserId user); - - /** Whether user is a member of the group */ - boolean isGroupMember(UserId user, UserGroup group); - } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/entity/MemoryEntityService.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/entity/MemoryEntityService.java index e5c2bbedae4..c5400e2d8e7 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/entity/MemoryEntityService.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/entity/MemoryEntityService.java @@ -3,13 +3,10 @@ package com.yahoo.vespa.hosted.controller.api.integration.entity; import com.yahoo.vespa.hosted.controller.api.identifiers.Property; import com.yahoo.vespa.hosted.controller.api.identifiers.PropertyId; -import com.yahoo.vespa.hosted.controller.api.identifiers.UserGroup; -import com.yahoo.vespa.hosted.controller.api.identifiers.UserId; import java.util.Collections; import java.util.HashMap; import java.util.Map; -import java.util.Set; /** * @author mpolden @@ -24,14 +21,4 @@ public class MemoryEntityService implements EntityService { return Collections.unmodifiableMap(properties); } - @Override - public Set<UserGroup> getUserGroups(UserId userId) { - return Collections.singleton(new UserGroup("vespa")); - } - - @Override - public boolean isGroupMember(UserId userId, UserGroup userGroup) { - return true; - } - } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/zone/ZoneRegistry.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/zone/ZoneRegistry.java index 81283ce802f..25b286bf94c 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/zone/ZoneRegistry.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/zone/ZoneRegistry.java @@ -19,35 +19,31 @@ import java.util.Optional; */ public interface ZoneRegistry { - /** Returns whether the system of this registry contains the given zone. */ + /** Returns whether the system of this registry contains the given zone */ boolean hasZone(ZoneId zoneId); - /** Returns a list containing the id of all zones in this registry. */ + /** Returns a list containing the id of all zones in this registry */ ZoneFilter zones(); - /** Returns the default region for the given environment, if one is configured. */ + /** Returns the default region for the given environment, if one is configured */ Optional<RegionName> getDefaultRegion(Environment environment); - // TODO Rename to getConfigServerUris once port 4080 is removed from configservers - /** Returns a list with all known config servers in the given zone, with a secure connection URL. */ - List<URI> getConfigServerSecureUris(ZoneId zoneId); + /** Returns a list with all known config servers in the given zone, with a secure connection URL */ + List<URI> getConfigServerUris(ZoneId zoneId); - /** Returns a URL with the logs for the given deployment, if logging is configured for its zone. */ + /** Returns a URL with the logs for the given deployment, if logging is configured for its zone */ Optional<URI> getLogServerUri(DeploymentId deploymentId); - /** Returns the time to live for deployments in the given zone, or empty if this is infinite. */ + /** Returns the time to live for deployments in the given zone, or empty if this is infinite */ Optional<Duration> getDeploymentTimeToLive(ZoneId zoneId); - /** Returns a URL pointing at monitoring resources for the given deployment. */ + /** Returns the monitoring system URL for the given deployment */ URI getMonitoringSystemUri(DeploymentId deploymentId); - /** Returns the URL of the dashboard for the system of this registry. */ - URI getDashboardUri(); - - /** Returns the system of this registry. */ + /** Returns the system of this registry */ SystemName system(); /** Return the configserver's Athenz service identity */ - AthenzService getConfigserverAthenzService(ZoneId zoneId); + AthenzService getConfigServerAthenzService(ZoneId zoneId); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java index 303f5d5484b..b568b1ca6e4 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java @@ -24,7 +24,6 @@ import com.yahoo.vespa.hosted.controller.api.integration.organization.Organizati import com.yahoo.vespa.hosted.controller.api.integration.routing.GlobalRoutingService; import com.yahoo.vespa.hosted.controller.api.integration.routing.RotationStatus; import com.yahoo.vespa.hosted.controller.api.integration.routing.RoutingGenerator; -import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry; import com.yahoo.vespa.hosted.controller.persistence.ControllerDb; import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; @@ -33,7 +32,6 @@ import com.yahoo.vespa.hosted.controller.versions.VespaVersion; import com.yahoo.vespa.hosted.rotation.config.RotationsConfig; import com.yahoo.vespa.serviceview.bindings.ApplicationView; -import java.net.URI; import java.time.Clock; import java.util.LinkedHashMap; import java.util.List; @@ -138,7 +136,7 @@ public class Controller extends AbstractComponent { rotationsConfig, nameService, configServerClient, artifactRepository, routingGenerator, clock); - tenantController = new TenantController(this, db, curator, entityService, athenzClientFactory); + tenantController = new TenantController(this, db, curator, athenzClientFactory); } /** Returns the instance controlling tenants */ @@ -162,15 +160,6 @@ public class Controller extends AbstractComponent { public Clock clock() { return clock; } - public Optional<URI> getLogServerUrl(DeploymentId deploymentId) { - return zoneRegistry.getLogServerUri(deploymentId); - } - - // TODO Rename to getConfigServerUris once port 4080 is removed from configservers - public List<URI> getSecureConfigServerUris(ZoneId zoneId) { - return zoneRegistry.getConfigServerSecureUris(zoneId); - } - public ZoneRegistry zoneRegistry() { return zoneRegistry; } public Map<String, RotationStatus> getHealthStatus(String hostname) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/TenantController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/TenantController.java index c9f2ff6eb49..482b6a6f3a9 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/TenantController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/TenantController.java @@ -7,15 +7,10 @@ import com.yahoo.vespa.athenz.api.AthenzUser; import com.yahoo.vespa.athenz.api.NToken; import com.yahoo.vespa.curator.Lock; import com.yahoo.vespa.hosted.controller.api.Tenant; -import com.yahoo.vespa.hosted.controller.api.identifiers.ApplicationId; -import com.yahoo.vespa.hosted.controller.api.identifiers.Property; -import com.yahoo.vespa.hosted.controller.api.identifiers.PropertyId; import com.yahoo.vespa.hosted.controller.api.identifiers.TenantId; -import com.yahoo.vespa.hosted.controller.api.identifiers.UserGroup; import com.yahoo.vespa.hosted.controller.api.identifiers.UserId; import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzClientFactory; import com.yahoo.vespa.hosted.controller.api.integration.athenz.ZmsClient; -import com.yahoo.vespa.hosted.controller.api.integration.entity.EntityService; import com.yahoo.vespa.hosted.controller.persistence.ControllerDb; import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; import com.yahoo.vespa.hosted.controller.persistence.PersistenceException; @@ -48,15 +43,13 @@ public class TenantController { private final CuratorDb curator; private final AthenzClientFactory athenzClientFactory; - private final EntityService entityService; - public TenantController(Controller controller, ControllerDb db, CuratorDb curator, EntityService entityService, + public TenantController(Controller controller, ControllerDb db, CuratorDb curator, AthenzClientFactory athenzClientFactory) { this.controller = controller; this.db = db; this.curator = curator; this.athenzClientFactory = athenzClientFactory; - this.entityService = entityService; } public List<Tenant> asList() { @@ -64,16 +57,13 @@ public class TenantController { } public List<Tenant> asList(UserId user) { - Set<UserGroup> userGroups = entityService.getUserGroups(user); Set<AthenzDomain> userDomains = new HashSet<>(athenzClientFactory.createZtsClientWithServicePrincipal() .getTenantDomainsForUser(AthenzUser.fromUserId(user.id()))); - - Predicate<Tenant> hasUsersGroup = (tenant) -> tenant.getUserGroup().isPresent() && userGroups.contains(tenant.getUserGroup().get()); Predicate<Tenant> hasUsersDomain = (tenant) -> tenant.getAthensDomain().isPresent() && userDomains.contains(tenant.getAthensDomain().get()); Predicate<Tenant> isUserTenant = (tenant) -> tenant.getId().equals(user.toTenantId()); return asList().stream() - .filter(t -> hasUsersGroup.test(t) || hasUsersDomain.test(t) || isUserTenant.test(t)) + .filter(t -> hasUsersDomain.test(t) || isUserTenant.test(t)) .collect(Collectors.toList()); } @@ -86,11 +76,10 @@ public class TenantController { } } - /** Creates an Athens or OpsDb tenant. */ - // TODO: Rename to createAthensTenant and move creation here when opsDbTenant creation is removed */ - public void addTenant(Tenant tenant, Optional<NToken> token) { + /** Creates an Athens tenant. */ + public void createAthenzTenant(Tenant tenant, NToken token) { try (Lock lock = lock(tenant.getId())) { - internalCreateTenant(tenant, token); + internalCreateTenant(tenant, Optional.of(token)); } } @@ -190,44 +179,6 @@ public class TenantController { } } - public Tenant migrateTenantToAthenz(TenantId tenantId, - AthenzDomain tenantDomain, - PropertyId propertyId, - Property property, - NToken nToken) { - try (Lock lock = lock(tenantId)) { - Tenant existing = tenant(tenantId).orElseThrow(() -> new NotExistsException(tenantId)); - if (existing.isAthensTenant()) return existing; // nothing to do - log.info("Starting migration of " + existing + " to Athenz domain " + tenantDomain.getName()); - if (tenantHaving(tenantDomain).isPresent()) - throw new IllegalArgumentException("Could not migrate " + existing + " to " + tenantDomain + ": " + - "This domain is already used by " + tenantHaving(tenantDomain).get()); - if ( ! existing.isOpsDbTenant()) - throw new IllegalArgumentException("Could not migrate " + existing + " to " + tenantDomain + ": " + - "Tenant is not currently an OpsDb tenant"); - - ZmsClient zmsClient = athenzClientFactory.createZmsClientWithAuthorizedServiceToken(nToken); - zmsClient.createTenant(tenantDomain); - - // Create resource group in Athenz for each application name - controller.applications() - .asList(TenantName.from(existing.getId().id())) - .stream() - .map(name -> new ApplicationId(name.id().application().value())) - .distinct() - .forEach(appId -> zmsClient.addApplication(tenantDomain, appId)); - - db.deleteTenant(tenantId); - Tenant tenant = Tenant.createAthensTenant(tenantId, tenantDomain, property, Optional.of(propertyId)); - db.createTenant(tenant); - log.info("Migration of " + existing + " to Athenz completed."); - return tenant; - } - catch (PersistenceException e) { - throw new RuntimeException("Failed migrating " + tenantId + " to Athenz", e); - } - } - private TenantId dashToUnderscore(TenantId id) { return new TenantId(id.id().replaceAll("-", "_")); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/api/Tenant.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/api/Tenant.java index 9b0cf96bb89..0edc63c69f5 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/api/Tenant.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/api/Tenant.java @@ -1,12 +1,11 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.api; -import com.yahoo.vespa.hosted.controller.api.application.v4.model.TenantType; import com.yahoo.vespa.athenz.api.AthenzDomain; +import com.yahoo.vespa.hosted.controller.api.application.v4.model.TenantType; import com.yahoo.vespa.hosted.controller.api.identifiers.Property; import com.yahoo.vespa.hosted.controller.api.identifiers.PropertyId; import com.yahoo.vespa.hosted.controller.api.identifiers.TenantId; -import com.yahoo.vespa.hosted.controller.api.identifiers.UserGroup; import java.util.Optional; @@ -14,36 +13,29 @@ import java.util.Optional; * @author smorgrav */ // TODO: Move this and everything it owns to com.yahoo.hosted.controller.Tenant and com.yahoo.hosted.controller.tenant.* +// TODO: Use polymorphism to represent the two tenant types public class Tenant { private final TenantId id; - private final Optional<UserGroup> userGroup; private final Optional<Property> property; private final Optional<AthenzDomain> athenzDomain; private final Optional<PropertyId> propertyId; // TODO: Use factory methods. They're down at the bottom! - public Tenant(TenantId id, Optional<UserGroup> userGroup, Optional<Property> property, Optional<AthenzDomain> athenzDomain) { - this(id, userGroup, property, athenzDomain, Optional.empty()); + public Tenant(TenantId id, Optional<Property> property, Optional<AthenzDomain> athenzDomain) { + this(id, property, athenzDomain, Optional.empty()); } - public Tenant(TenantId id, Optional<UserGroup> userGroup, Optional<Property> property, Optional<AthenzDomain> athenzDomain, Optional<PropertyId> propertyId) { + public Tenant(TenantId id, Optional<Property> property, Optional<AthenzDomain> athenzDomain, Optional<PropertyId> propertyId) { if (id.isUser()) { - require( ! userGroup.isPresent(), "User tenant '%s' cannot have a user group.", id); require( ! property.isPresent(), "User tenant '%s' cannot have a property.", id); require( ! propertyId.isPresent(), "User tenant '%s' cannot have a property ID.", id); require( ! athenzDomain.isPresent(), "User tenant '%s' cannot have an athens domain.", id); - } else if (athenzDomain.isPresent()) { + } else { require( property.isPresent(), "Athens tenant '%s' must have a property.", id); - require( ! userGroup.isPresent(), "Athens tenant '%s' cannot have a user group.", id); require( athenzDomain.isPresent(), "Athens tenant '%s' must have an athens domain.", id); - } else { - require( property.isPresent(), "OpsDB tenant '%s' must have a property.", id); - require( userGroup.isPresent(), "OpsDb tenant '%s' must have a user group.", id); - require( ! athenzDomain.isPresent(), "OpsDb tenant '%s' cannot have an athens domain.", id); } this.id = id; - this.userGroup = userGroup; this.property = property; this.athenzDomain = athenzDomain; this.propertyId = propertyId; // TODO: Check validity after TODO@14. OpsDb tenants have this set in Sherpa, while athens tenants do not. @@ -51,15 +43,12 @@ public class Tenant { } public boolean isAthensTenant() { return athenzDomain.isPresent(); } - public boolean isOpsDbTenant() { return userGroup.isPresent();} public TenantType tenantType() { if (athenzDomain.isPresent()) { return TenantType.ATHENS; - } else if (id.isUser()) { - return TenantType.USER; } else { - return TenantType.OPSDB; + return TenantType.USER; } } @@ -67,10 +56,6 @@ public class Tenant { return id; } - public Optional<UserGroup> getUserGroup() { - return userGroup; - } - /** OpsDB property name of the tenant, or Optional.empty() if none is stored. */ public Optional<Property> getProperty() { return property; @@ -93,19 +78,7 @@ public class Tenant { if (id.isUser()) { throw new IllegalArgumentException("Invalid id for non-user tenant: " + id); } - return new Tenant(id, Optional.empty(), Optional.ofNullable(property), - Optional.ofNullable(athensDomain), propertyId); - } - - public static Tenant createOpsDbTenant(TenantId id, UserGroup userGroup, Property property, Optional<PropertyId> propertyId) { - if (id.isUser()) { - throw new IllegalArgumentException("Invalid id for non-user tenant: " + id); - } - return new Tenant(id, Optional.ofNullable(userGroup), Optional.ofNullable(property), Optional.empty(), propertyId); - } - - public static Tenant createOpsDbTenant(TenantId id, UserGroup userGroup, Property property) { - return createOpsDbTenant(id, userGroup, property, Optional.empty()); + return new Tenant(id, Optional.ofNullable(property), Optional.ofNullable(athensDomain), propertyId); } public static Tenant createUserTenant(TenantId id) { @@ -123,7 +96,6 @@ public class Tenant { Tenant tenant = (Tenant) o; if (!id.equals(tenant.id)) return false; - if (!userGroup.equals(tenant.userGroup)) return false; if (!property.equals(tenant.property)) return false; if (!athenzDomain.equals(tenant.athenzDomain)) return false; if (!propertyId.equals(tenant.propertyId)) return false; @@ -133,7 +105,6 @@ public class Tenant { @Override public int hashCode() { int result = id.hashCode(); - result = 31 * result + userGroup.hashCode(); result = 31 * result + property.hashCode(); result = 31 * result + athenzDomain.hashCode(); result = 31 * result + propertyId.hashCode(); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutorImpl.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutorImpl.java index 99b738a6a38..06dc29b3b70 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutorImpl.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutorImpl.java @@ -80,7 +80,7 @@ public class ConfigServerRestExecutorImpl implements ConfigServerRestExecutor { ZoneId zoneId = ZoneId.from(proxyRequest.getEnvironment(), proxyRequest.getRegion()); // Make a local copy of the list as we want to manipulate it in case of ping problems. - List<URI> allServers = new ArrayList<>(zoneRegistry.getConfigServerSecureUris(zoneId)); + List<URI> allServers = new ArrayList<>(zoneRegistry.getConfigServerUris(zoneId)); StringBuilder errorBuilder = new StringBuilder(); if (queueFirstServerIfDown(allServers, proxyRequest)) { @@ -263,7 +263,7 @@ public class ConfigServerRestExecutorImpl implements ConfigServerRestExecutor { AthenzIdentityVerifier hostnameVerifier = new AthenzIdentityVerifier( singleton( - zoneRegistry.getConfigserverAthenzService( + zoneRegistry.getConfigServerAthenzService( ZoneId.from(proxyRequest.getEnvironment(), proxyRequest.getRegion())))); return HttpClientBuilder.create() .setUserAgent("config-server-proxy-client") diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index 368fd323fb0..337ad934d41 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -50,7 +50,6 @@ import com.yahoo.vespa.hosted.controller.api.identifiers.Property; import com.yahoo.vespa.hosted.controller.api.identifiers.PropertyId; import com.yahoo.vespa.hosted.controller.api.identifiers.ScrewdriverId; import com.yahoo.vespa.hosted.controller.api.identifiers.TenantId; -import com.yahoo.vespa.hosted.controller.api.identifiers.UserGroup; import com.yahoo.vespa.hosted.controller.api.identifiers.UserId; import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzClientFactory; import com.yahoo.vespa.hosted.controller.api.integration.athenz.ZmsException; @@ -185,7 +184,6 @@ public class ApplicationApiHandler extends LoggingRequestHandler { Path path = new Path(request.getUri().getPath()); if (path.matches("/application/v4/user")) return createUser(request); if (path.matches("/application/v4/tenant/{tenant}")) return updateTenant(path.get("tenant"), request); - if (path.matches("/application/v4/tenant/{tenant}/migrateTenantToAthens")) return migrateTenant(path.get("tenant"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/global-rotation/override")) return setGlobalRotationOverride(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), false, request); return ErrorResponse.notFoundError("Nothing at " + path); @@ -462,7 +460,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { response.setString("nodes", withPath("/zone/v2/" + deploymentId.zoneId().environment() + "/" + deploymentId.zoneId().region() + "/nodes/v2/node/?&recursive=true&application=" + deploymentId.applicationId().tenant() + "." + deploymentId.applicationId().application() + "." + deploymentId.applicationId().instance(), request.getUri()).toString()); - controller.getLogServerUrl(deploymentId) + controller.zoneRegistry().getLogServerUri(deploymentId) .ifPresent(elkUrl -> response.setString("elkUrl", elkUrl.toString())); response.setString("yamasUrl", monitoringSystemUri(deploymentId).toString()); @@ -593,7 +591,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { ApplicationView applicationView = controller.getApplicationView(tenantName, applicationName, instanceName, environment, region); ServiceApiResponse response = new ServiceApiResponse(ZoneId.from(environment, region), new ApplicationId.Builder().tenant(tenantName).applicationName(applicationName).instanceName(instanceName).build(), - controller.getSecureConfigServerUris(ZoneId.from(environment, region)), + controller.zoneRegistry().getConfigServerUris(ZoneId.from(environment, region)), request.getUri()); response.setResponse(applicationView); return response; @@ -603,7 +601,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { Map<?,?> result = controller.getServiceApiResponse(tenantName, applicationName, instanceName, environment, region, serviceName, restPath); ServiceApiResponse response = new ServiceApiResponse(ZoneId.from(environment, region), new ApplicationId.Builder().tenant(tenantName).applicationName(applicationName).instanceName(instanceName).build(), - controller.getSecureConfigServerUris(ZoneId.from(environment, region)), + controller.zoneRegistry().getConfigServerUris(ZoneId.from(environment, region)), request.getUri()); response.setResponse(result, serviceName, restPath); return response; @@ -633,18 +631,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { case USER: { throw new BadRequestException("Cannot set property or OpsDB user group for user tenant"); } - case OPSDB: { - UserGroup userGroup = new UserGroup(mandatory("userGroup", requestData).asString()); - updatedTenant = Tenant.createOpsDbTenant(new TenantId(tenantName), - userGroup, - new Property(mandatory("property", requestData).asString()), - optional("propertyId", requestData).map(PropertyId::new)); - controller.tenants().updateTenant(updatedTenant, getUserPrincipal(request).getNToken()); - break; - } case ATHENS: { - if (requestData.field("userGroup").valid()) - throw new BadRequestException("Cannot set OpsDB user group to Athens tenant"); updatedTenant = Tenant.createAthensTenant(new TenantId(tenantName), new AthenzDomain(mandatory("athensDomain", requestData).asString()), new Property(mandatory("property", requestData).asString()), @@ -666,29 +653,15 @@ public class ApplicationApiHandler extends LoggingRequestHandler { Inspector requestData = toSlime(request.getData()).get(); Tenant tenant = new Tenant(new TenantId(tenantName), - optional("userGroup", requestData).map(UserGroup::new), optional("property", requestData).map(Property::new), optional("athensDomain", requestData).map(AthenzDomain::new), optional("propertyId", requestData).map(PropertyId::new)); if (tenant.isAthensTenant()) throwIfNotAthenzDomainAdmin(new AthenzDomain(mandatory("athensDomain", requestData).asString()), request); - controller.tenants().addTenant(tenant, getUserPrincipal(request).getNToken()); - return tenant(tenant, request, true); - } - - private HttpResponse migrateTenant(String tenantName, HttpRequest request) { - TenantId tenantid = new TenantId(tenantName); - Inspector requestData = toSlime(request.getData()).get(); - AthenzDomain tenantDomain = new AthenzDomain(mandatory("athensDomain", requestData).asString()); - Property property = new Property(mandatory("property", requestData).asString()); - PropertyId propertyId = new PropertyId(mandatory("propertyId", requestData).asString()); - - throwIfNotAthenzDomainAdmin(tenantDomain, request); - NToken nToken = getUserPrincipal(request).getNToken() - .orElseThrow(() -> - new BadRequestException("The NToken for a domain admin is required to migrate tenant to Athens")); - Tenant tenant = controller.tenants().migrateTenantToAthenz(tenantid, tenantDomain, propertyId, property, nToken); + NToken token = getUserPrincipal(request).getNToken() + .orElseThrow(() -> new IllegalArgumentException("Could not create " + tenant + ": No NToken provided")); + controller.tenants().createAthenzTenant(tenant, token); return tenant(tenant, request, true); } @@ -938,7 +911,6 @@ public class ApplicationApiHandler extends LoggingRequestHandler { tenant.getAthensDomain().ifPresent(a -> object.setString("athensDomain", a.getName())); tenant.getProperty().ifPresent(p -> object.setString("property", p.id())); tenant.getPropertyId().ifPresent(p -> object.setString("propertyId", p.toString())); - tenant.getUserGroup().ifPresent(g -> object.setString("userGroup", g.id())); Cursor applicationArray = object.setArray("applications"); if (listApplications) { // This cludge is needed because we call this after deleting the tenant. As this call makes another tenant lookup it will fail. TODO is to support lookup on tenant for (Application application : controller.applications().asList(TenantName.from(tenant.getId().id()))) { @@ -976,7 +948,6 @@ public class ApplicationApiHandler extends LoggingRequestHandler { metaData.setString("type", tenant.tenantType().name()); tenant.getAthensDomain().ifPresent(a -> metaData.setString("athensDomain", a.getName())); tenant.getProperty().ifPresent(p -> metaData.setString("property", p.id())); - tenant.getUserGroup().ifPresent(g -> metaData.setString("userGroup", g.id())); object.setString("url", withPath("/application/v4/tenant/" + tenant.getId().id(), requestURI).toString()); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/ControllerAuthorizationFilter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/ControllerAuthorizationFilter.java index 8a539beb83a..614ef960155 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/ControllerAuthorizationFilter.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/ControllerAuthorizationFilter.java @@ -14,13 +14,13 @@ import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.athenz.api.AthenzPrincipal; import com.yahoo.vespa.athenz.api.AthenzUser; import com.yahoo.vespa.hosted.controller.Controller; +import com.yahoo.vespa.hosted.controller.TenantController; import com.yahoo.vespa.hosted.controller.api.Tenant; import com.yahoo.vespa.hosted.controller.api.identifiers.TenantId; import com.yahoo.vespa.hosted.controller.api.identifiers.UserId; import com.yahoo.vespa.hosted.controller.api.integration.athenz.ApplicationAction; import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzClientFactory; import com.yahoo.vespa.hosted.controller.api.integration.athenz.ZmsException; -import com.yahoo.vespa.hosted.controller.api.integration.entity.EntityService; import com.yahoo.vespa.hosted.controller.restapi.Path; import com.yahoo.yolean.chain.After; @@ -54,8 +54,7 @@ public class ControllerAuthorizationFilter implements SecurityRequestFilter { private static final Logger log = Logger.getLogger(ControllerAuthorizationFilter.class.getName()); private final AthenzClientFactory clientFactory; - private final Controller controller; - private final EntityService entityService; + private final TenantController tenantController; private final AuthorizationResponseHandler authorizationResponseHandler; public interface AuthorizationResponseHandler { @@ -63,19 +62,15 @@ public class ControllerAuthorizationFilter implements SecurityRequestFilter { } @Inject - public ControllerAuthorizationFilter(AthenzClientFactory clientFactory, - Controller controller, - EntityService entityService) { - this(clientFactory, controller, entityService, new DefaultAuthorizationResponseHandler()); + public ControllerAuthorizationFilter(AthenzClientFactory clientFactory, Controller controller) { + this(clientFactory, controller.tenants(), new DefaultAuthorizationResponseHandler()); } ControllerAuthorizationFilter(AthenzClientFactory clientFactory, - Controller controller, - EntityService entityService, + TenantController tenantController, AuthorizationResponseHandler authorizationResponseHandler) { this.clientFactory = clientFactory; - this.controller = controller; - this.entityService = entityService; + this.tenantController = tenantController; this.authorizationResponseHandler = authorizationResponseHandler; } @@ -127,7 +122,6 @@ public class ControllerAuthorizationFilter implements SecurityRequestFilter { private static boolean isTenantAdminOperation(Path path, Method method) { if (isHostedOperatorOperation(path, method)) return false; return path.matches("/application/v4/tenant/{tenant}") || - path.matches("/application/v4/tenant/{tenant}/migrateTenantToAthens") || path.matches("/application/v4/tenant/{tenant}/application/{application}") || path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/dev/{*}") || path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/perf/{*}") || @@ -155,7 +149,7 @@ public class ControllerAuthorizationFilter implements SecurityRequestFilter { } private void verifyIsTenantAdmin(AthenzPrincipal principal, TenantId tenantId) { - controller.tenants().tenant(tenantId) + tenantController.tenant(tenantId) .ifPresent(tenant -> { if (!isTenantAdmin(principal.getIdentity(), tenant)) { throw new ForbiddenException("Tenant admin or Vespa operator role required"); @@ -168,13 +162,6 @@ public class ControllerAuthorizationFilter implements SecurityRequestFilter { case ATHENS: return clientFactory.createZmsClientWithServicePrincipal() .hasTenantAdminAccess(identity, tenant.getAthensDomain().get()); - case OPSDB: { - if (!(identity instanceof AthenzUser)) { - return false; - } - AthenzUser user = (AthenzUser) identity; - return entityService.isGroupMember(new UserId(user.getName()), tenant.getUserGroup().get()); - } case USER: { if (!(identity instanceof AthenzUser)) { return false; @@ -190,7 +177,7 @@ public class ControllerAuthorizationFilter implements SecurityRequestFilter { private void verifyIsTenantPipelineOperator(AthenzPrincipal principal, TenantId tenantId, ApplicationName application) { - controller.tenants().tenant(tenantId) + tenantController.tenant(tenantId) .ifPresent(tenant -> verifyIsTenantPipelineOperator(principal.getIdentity(), tenant, application)); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/ZoneApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/ZoneApiHandler.java index 7900413b485..63de0ac8f69 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/ZoneApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/ZoneApiHandler.java @@ -7,6 +7,7 @@ import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.container.jdisc.LoggingRequestHandler; import com.yahoo.slime.Cursor; import com.yahoo.slime.Slime; +import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneList; import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry; import com.yahoo.vespa.hosted.controller.proxy.ConfigServerRestExecutor; import com.yahoo.vespa.hosted.controller.proxy.ProxyException; @@ -89,13 +90,14 @@ public class ZoneApiHandler extends LoggingRequestHandler { Slime slime = new Slime(); Cursor root = slime.setObject(); Cursor uris = root.setArray("uris"); - zoneRegistry.zones().controllerManaged().ids().forEach(zoneId -> uris.addString(request.getUri() + ZoneList zoneList = zoneRegistry.zones().controllerManaged(); + zoneList.ids().forEach(zoneId -> uris.addString(request.getUri() .resolve("/zone/v2/") .resolve(zoneId.environment().value() + "/") .resolve(zoneId.region().value()) .toString())); Cursor zones = root.setArray("zones"); - zoneRegistry.zones().all().ids().forEach(zoneId -> { + zoneList.ids().forEach(zoneId -> { Cursor object = zones.addObject(); object.setString("environment", zoneId.environment().value()); object.setString("region", zoneId.region().value()); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java index d628489bc29..eb932d42e79 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java @@ -119,7 +119,7 @@ public class VersionStatus { List<URI> configServers = controller.zoneRegistry().zones() .controllerManaged() .ids().stream() - .flatMap(zoneId -> controller.getSecureConfigServerUris(zoneId).stream()) + .flatMap(zoneId -> controller.zoneRegistry().getConfigServerUris(zoneId).stream()) .collect(Collectors.toList()); ListMap<Version, String> versions = new ListMap<>(); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java index 22730cd2fb2..a898c3eec68 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java @@ -415,44 +415,6 @@ public class ControllerTest { } @Test - public void testMigratingTenantToAthenzWillModifyAthenzDomainsCorrectly() { - ControllerTester tester = new ControllerTester(); - - // Create Athens domain mock - AthenzDomain athensDomain = new AthenzDomain("vespa.john"); - AthenzDbMock.Domain mockDomain = new AthenzDbMock.Domain(athensDomain); - tester.athenzDb().addDomain(mockDomain); - - // Create OpsDb tenant - TenantId tenantId = new TenantId("mytenant"); - Tenant existingTenant = Tenant.createOpsDbTenant(tenantId, new UserGroup("myusergroup"), new Property("myproperty")); - tester.controller().tenants().addTenant(existingTenant, Optional.empty()); - - // Create an application without instance - String applicationName = "myapplication"; - ApplicationId applicationId = ApplicationId.from(tenantId.id(), applicationName, "default"); - tester.controller().applications().createApplication(applicationId, Optional.empty()); - - // Verify that Athens domain does not have any relations to tenant/application yet - assertTrue(mockDomain.applications.keySet().isEmpty()); - assertFalse(mockDomain.isVespaTenant); - - // Migrate tenant to Athens - NToken nToken = TestIdentities.userNToken; - tester.controller().tenants().migrateTenantToAthenz( - tenantId, athensDomain, new PropertyId("1567"), new Property("vespa_dev.no"), nToken); - - // Verify that tenant is migrated - Tenant tenant = tester.controller().tenants().tenant(tenantId).get(); - assertTrue(tenant.isAthensTenant()); - assertEquals(athensDomain, tenant.getAthensDomain().get()); - // Verify that domain knows about tenant and application - assertTrue(mockDomain.isVespaTenant); - assertTrue(mockDomain.applications.keySet().contains( - new com.yahoo.vespa.hosted.controller.api.identifiers.ApplicationId(applicationName))); - } - - @Test public void requeueOutOfCapacityStagingJob() { DeploymentTester tester = new DeploymentTester(); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java index 691a5ef223d..b4bbc0f68e7 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java @@ -196,7 +196,7 @@ public final class ControllerTester { Tenant tenant = Tenant.createAthensTenant(id, createDomain(domainName), new Property("app1Property"), propertyId == null ? Optional.empty() : Optional.of(new PropertyId(propertyId.toString()))); - controller().tenants().addTenant(tenant, Optional.of(TestIdentities.userNToken)); + controller().tenants().createAthenzTenant(tenant, TestIdentities.userNToken); assertNotNull(controller().tenants().tenant(id)); return id; } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/TestIdentities.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/TestIdentities.java index b7549364b73..6c74756ca33 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/TestIdentities.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/TestIdentities.java @@ -1,7 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller; -import com.yahoo.vespa.hosted.controller.api.Tenant; +import com.yahoo.vespa.athenz.api.NToken; import com.yahoo.vespa.hosted.controller.api.identifiers.EnvironmentId; import com.yahoo.vespa.hosted.controller.api.identifiers.InstanceId; import com.yahoo.vespa.hosted.controller.api.identifiers.Property; @@ -9,7 +9,6 @@ import com.yahoo.vespa.hosted.controller.api.identifiers.RegionId; import com.yahoo.vespa.hosted.controller.api.identifiers.TenantId; import com.yahoo.vespa.hosted.controller.api.identifiers.UserGroup; import com.yahoo.vespa.hosted.controller.api.identifiers.UserId; -import com.yahoo.vespa.athenz.api.NToken; /** * @author Tony Vaagenes @@ -30,8 +29,6 @@ public class TestIdentities { public static Property property = new Property("property"); - public static Tenant tenant = Tenant.createOpsDbTenant(tenantId, userGroup1, property); - public static NToken userNToken = new NToken("dummy"); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ZoneRegistryMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ZoneRegistryMock.java index 63751cfaa98..d7504edb342 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ZoneRegistryMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ZoneRegistryMock.java @@ -76,7 +76,7 @@ public class ZoneRegistryMock extends AbstractComponent implements ZoneRegistry return ZoneFilterMock.from(Collections.unmodifiableList(zones)); } - public AthenzService getConfigserverAthenzService(ZoneId zone) { + public AthenzService getConfigServerAthenzService(ZoneId zone) { return new AthenzService("vespadomain", "provider-" + zone.environment().value() + "-" + zone.region().value()); } @@ -86,7 +86,7 @@ public class ZoneRegistryMock extends AbstractComponent implements ZoneRegistry } @Override - public List<URI> getConfigServerSecureUris(ZoneId zoneId) { + public List<URI> getConfigServerUris(ZoneId zoneId) { return Collections.singletonList(URI.create(String.format("https://cfg.%s.test:4443", zoneId.value()))); } @@ -125,9 +125,4 @@ public class ZoneRegistryMock extends AbstractComponent implements ZoneRegistry + deploymentId.zoneId().region().value() + "&application=" + deploymentId.applicationId().toShortString()); } - @Override - public URI getDashboardUri() { - return URI.create("http://dashboard.test"); - } - } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ApplicationOwnershipConfirmerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ApplicationOwnershipConfirmerTest.java index c775dd3fd7c..0309eaf7d25 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ApplicationOwnershipConfirmerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ApplicationOwnershipConfirmerTest.java @@ -3,7 +3,6 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.config.provision.ApplicationId; import com.yahoo.vespa.hosted.controller.Application; -import com.yahoo.vespa.hosted.controller.api.Tenant; import com.yahoo.vespa.hosted.controller.api.identifiers.PropertyId; import com.yahoo.vespa.hosted.controller.api.identifiers.TenantId; import com.yahoo.vespa.hosted.controller.api.integration.organization.IssueId; @@ -45,7 +44,7 @@ public class ApplicationOwnershipConfirmerTest { Supplier<Application> propertyApp = () -> tester.controller().applications().require(ApplicationId.from("property", "application", "default")); TenantId user = new TenantId("by-user"); - tester.controller().tenants().addTenant(Tenant.createUserTenant(user), Optional.empty()); + tester.controller().tenants().createUserTenant("user"); tester.createAndDeploy(user, "application", 2, "default"); Supplier<Application> userApp = () -> tester.controller().applications().require(ApplicationId.from("by-user", "application", "default")); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java index 7b0ca7cb618..4d3f26d247e 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java @@ -21,6 +21,8 @@ import org.junit.Test; import org.mockito.Mockito; import java.io.IOException; +import java.nio.file.Path; +import java.nio.file.Paths; import java.time.Clock; import java.time.Instant; import java.time.ZoneId; @@ -40,6 +42,8 @@ import static org.mockito.Mockito.when; */ public class MetricsReporterTest { + private static final Path testData = Paths.get("src/test/resources/"); + @Test public void test_chef_metrics() throws IOException { ControllerTester tester = new ControllerTester(); @@ -108,7 +112,7 @@ public class MetricsReporterTest { Chef client = Mockito.mock(Chef.class); PartialNodeResult result = new ObjectMapper() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false) - .readValue(getClass().getClassLoader().getResource("chef_output.json"), PartialNodeResult.class); + .readValue(testData.resolve("chef_output.json").toFile(), PartialNodeResult.class); when(client.partialSearchNodes(anyString(), anyListOf(AttributeMapping.class))).thenReturn(result); Clock clock = Clock.fixed(Instant.ofEpochSecond(1475497913), ZoneId.systemDefault()); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerControllerTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerControllerTester.java index 256dcca7a05..b810c3adeb5 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerControllerTester.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerControllerTester.java @@ -75,10 +75,10 @@ public class ContainerControllerTester { public Application createApplication(String athensDomain, String tenant, String application) { AthenzDomain domain1 = addTenantAthenzDomain(athensDomain, "mytenant"); - controller().tenants().addTenant(Tenant.createAthensTenant(new TenantId(tenant), domain1, - new Property("property1"), - Optional.of(new PropertyId("1234"))), - Optional.of(TestIdentities.userNToken)); + controller().tenants().createAthenzTenant(Tenant.createAthensTenant(new TenantId(tenant), domain1, + new Property("property1"), + Optional.of(new PropertyId("1234"))), + TestIdentities.userNToken); ApplicationId app = ApplicationId.from(tenant, application, "default"); return controller().applications().createApplication(app, Optional.of(TestIdentities.userNToken)); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java index 4b59e57fd5a..0ae9cf767d0 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java @@ -158,18 +158,6 @@ public class ApplicationApiTest extends ControllerContainerTest { tester.assertResponse(request("/application/v4/tenant/tenant2", GET).userIdentity(USER_ID), new File("tenant-without-applications-with-id.json")); - // Test legacy OpsDB tenants - // POST (add) an OpsDB tenant with property ID - tester.assertResponse(request("/application/v4/tenant/tenant3", POST) - .userIdentity(USER_ID) - .data("{\"userGroup\":\"group1\",\"property\":\"property1\",\"propertyId\":\"1234\"}"), - new File("opsdb-tenant-with-id-without-applications.json")); - // PUT (modify) the OpsDB tenant to set another property - tester.assertResponse(request("/application/v4/tenant/tenant3", PUT) - .userIdentity(USER_ID) - .data("{\"userGroup\":\"group1\",\"property\":\"property2\",\"propertyId\":\"4321\"}"), - new File("opsdb-tenant-with-new-id-without-applications.json")); - // POST (create) an application tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1", POST) .userIdentity(USER_ID) @@ -558,6 +546,7 @@ public class ApplicationApiTest extends ControllerContainerTest { // Add the same tenant again tester.assertResponse(request("/application/v4/tenant/tenant1", POST) .userIdentity(USER_ID) + .nToken(N_TOKEN) .data("{\"athensDomain\":\"domain1\", \"property\":\"property1\"}"), "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Tenant 'tenant1' already exists\"}", 400); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/opsdb-tenant-with-id-without-applications.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/opsdb-tenant-with-id-without-applications.json deleted file mode 100644 index a2e70d9c1eb..00000000000 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/opsdb-tenant-with-id-without-applications.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "tenant": "tenant3", - "type": "OPSDB", - "property": "property1", - "propertyId": "1234", - "userGroup": "group1", - "applications": [ - - ], - "propertyUrl": "www.properties.tld/1234", - "contactsUrl": "www.contacts.tld/1234", - "issueCreationUrl": "www.issues.tld/1234", - "contacts": [ - [ - "alice" - ], - [ - "bob" - ] - ] -} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/opsdb-tenant-with-new-id-without-applications.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/opsdb-tenant-with-new-id-without-applications.json deleted file mode 100644 index f9161ea49b1..00000000000 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/opsdb-tenant-with-new-id-without-applications.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "tenant": "tenant3", - "type": "OPSDB", - "property": "property2", - "propertyId": "4321", - "userGroup": "group1", - "applications": [ - - ], - "propertyUrl": "www.properties.tld/4321", - "contactsUrl": "www.contacts.tld/4321", - "issueCreationUrl": "www.issues.tld/4321", - "contacts": [ - - ] -} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/recursive-root.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/recursive-root.json index a4395faede4..cea35b764b2 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/recursive-root.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/recursive-root.json @@ -1,5 +1,4 @@ [ @include(tenant2.json), - @include(tenant3.json), @include(tenant1-recursive.json) ] diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/recursive-until-tenant-root.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/recursive-until-tenant-root.json index 35ed8181fac..8b6b1ff7cf5 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/recursive-until-tenant-root.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/recursive-until-tenant-root.json @@ -1,6 +1,5 @@ [ @include(tenant2.json), - @include(tenant3.json), @include(tenant-with-application.json) ] diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant3.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant3.json deleted file mode 100644 index fdf3ca490f4..00000000000 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant3.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "tenant": "tenant3", - "type": "OPSDB", - "property": "property2", - "propertyId": "4321", - "userGroup": "group1", - "applications": [], - "propertyUrl": "www.properties.tld/4321", - "contactsUrl": "www.contacts.tld/4321", - "issueCreationUrl": "www.issues.tld/4321", - "contacts": [] -}
\ No newline at end of file diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/ControllerAuthorizationFilterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/ControllerAuthorizationFilterTest.java index c4863a0eb79..46e23715139 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/ControllerAuthorizationFilterTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/ControllerAuthorizationFilterTest.java @@ -144,8 +144,7 @@ public class ControllerAuthorizationFilterTest { private static ControllerAuthorizationFilter createFilter(ControllerTester controllerTester) { return new ControllerAuthorizationFilter(new AthenzClientFactoryMock(controllerTester.athenzDb()), - controllerTester.controller(), - controllerTester.entityService()); + controllerTester.controller()); } private static Optional<AuthorizationResponse> invokeFilter(ControllerAuthorizationFilter filter, diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/filter/ServletFilterRequest.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/filter/ServletFilterRequest.java index 5921f0b8e0a..3dccb09c971 100644 --- a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/filter/ServletFilterRequest.java +++ b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/filter/ServletFilterRequest.java @@ -143,7 +143,7 @@ class ServletFilterRequest extends DiscFilterRequest { @Override public List<X509Certificate> getClientCertificateChain() { - return Optional.ofNullable(parent.context().get(ServletRequest.SERVLET_REQUEST_X509CERT)) + return Optional.ofNullable(parent.getRequest().getAttribute(ServletRequest.SERVLET_REQUEST_X509CERT)) .map(X509Certificate[].class::cast) .map(Arrays::asList) .orElse(Collections.emptyList()); diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/Environment.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/Environment.java index 4fb3049c39d..5fb619ee6e2 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/Environment.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/Environment.java @@ -117,13 +117,11 @@ public class Environment { this.keyStoreOptions = createKeyStoreOptions( configServerConfig.keyStoreConfig().path(), configServerConfig.keyStoreConfig().password().toCharArray(), - configServerConfig.keyStoreConfig().type().name(), - "BC"); + configServerConfig.keyStoreConfig().type().name()); this.trustStoreOptions = createKeyStoreOptions( configServerConfig.trustStoreConfig().path(), configServerConfig.trustStoreConfig().password().toCharArray(), - configServerConfig.trustStoreConfig().type().name(), - null); + configServerConfig.trustStoreConfig().type().name()); this.athenzIdentity = createAthenzIdentity( configServerConfig.athenzDomain(), configServerConfig.serviceName()); @@ -184,10 +182,10 @@ public class Environment { return Arrays.asList(logstashNodes.split("[,\\s]+")); } - private static Optional<KeyStoreOptions> createKeyStoreOptions(String pathToKeyStore, char[] password, String type, String provider) { + private static Optional<KeyStoreOptions> createKeyStoreOptions(String pathToKeyStore, char[] password, String type) { return Optional.ofNullable(pathToKeyStore) .filter(path -> !Strings.isNullOrEmpty(path)) - .map(path -> new KeyStoreOptions(Paths.get(path), password, type, provider)); + .map(path -> new KeyStoreOptions(Paths.get(path), password, type)); } private static Optional<AthenzIdentity> createAthenzIdentity(String athenzDomain, String serviceName) { diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/SslConfigServerApiImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/SslConfigServerApiImpl.java index 04b222875c3..110dbe9c9b3 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/SslConfigServerApiImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/SslConfigServerApiImpl.java @@ -3,7 +3,6 @@ package com.yahoo.vespa.hosted.node.admin.configserver; import com.yahoo.vespa.athenz.tls.AthenzIdentityVerifier; import com.yahoo.vespa.athenz.tls.AthenzSslContextBuilder; -import com.yahoo.vespa.athenz.tls.KeyStoreType; import com.yahoo.vespa.hosted.node.admin.component.Environment; import com.yahoo.vespa.hosted.node.admin.configserver.certificate.ConfigServerKeyStoreRefresher; import com.yahoo.vespa.hosted.node.admin.util.KeyStoreOptions; @@ -18,7 +17,7 @@ import java.util.Optional; /** * ConfigServerApi with proper keystore, truststore and hostname verifier to communicate with the - * configserver(s). The keystore is refreshed automatically. + * config server(s). The keystore is refreshed automatically. * * @author freva */ @@ -99,16 +98,8 @@ public class SslConfigServerApiImpl implements ConfigServerApi { private SSLContext makeSslContext(Optional<KeyStoreOptions> keyStoreOptions) { AthenzSslContextBuilder sslContextBuilder = new AthenzSslContextBuilder(); - environment.getTrustStoreOptions().ifPresent( - options -> sslContextBuilder.withTrustStore(options.path.toFile(), KeyStoreType.valueOf(options.type))); - - keyStoreOptions.ifPresent(options -> { - try { - sslContextBuilder.withKeyStore(options.path.toFile(), options.password, KeyStoreType.valueOf(options.type)); - } catch (Exception e) { - throw new RuntimeException("Failed to read key store", e); - } - }); + environment.getTrustStoreOptions().map(KeyStoreOptions::loadKeyStore).ifPresent(sslContextBuilder::withTrustStore); + keyStoreOptions.ifPresent(options -> sslContextBuilder.withKeyStore(options.loadKeyStore(), options.password)); return sslContextBuilder.build(); } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/certificate/ConfigServerKeyStoreRefresher.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/certificate/ConfigServerKeyStoreRefresher.java index ae725769bdb..a9db96c2a77 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/certificate/ConfigServerKeyStoreRefresher.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/certificate/ConfigServerKeyStoreRefresher.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.hosted.node.admin.configserver.certificate; import com.yahoo.log.LogLevel; import com.yahoo.net.HostName; +import com.yahoo.vespa.athenz.tls.KeyStoreBuilder; import com.yahoo.vespa.hosted.node.admin.configserver.ConfigServerApi; import com.yahoo.vespa.hosted.node.admin.util.KeyStoreOptions; import org.bouncycastle.asn1.x500.X500Name; @@ -12,7 +13,6 @@ import org.bouncycastle.operator.jcajce.JcaContentSignerBuilder; import org.bouncycastle.pkcs.PKCS10CertificationRequest; import org.bouncycastle.pkcs.jcajce.JcaPKCS10CertificationRequestBuilder; -import java.io.FileOutputStream; import java.io.IOException; import java.security.KeyPair; import java.security.KeyPairGenerator; @@ -159,14 +159,12 @@ public class ConfigServerKeyStoreRefresher { private void storeCertificate(KeyPair keyPair, X509Certificate certificate) throws KeyStoreException, CertificateException, NoSuchAlgorithmException, IOException, NoSuchProviderException { keyStoreOptions.path.getParent().toFile().mkdirs(); - X509Certificate[] certificateChain = {certificate}; - try (FileOutputStream fos = new FileOutputStream(keyStoreOptions.path.toFile())) { - KeyStore keyStore = keyStoreOptions.getKeyStoreInstance(); - keyStore.load(null, null); - keyStore.setKeyEntry(KEY_STORE_ALIAS, keyPair.getPrivate(), keyStoreOptions.password, certificateChain); - keyStore.store(fos, keyStoreOptions.password); - } + KeyStore keyStore = KeyStoreBuilder.withType(keyStoreOptions.keyStoreType) + .withKeyEntry(KEY_STORE_ALIAS, keyPair.getPrivate(), keyStoreOptions.password, certificate) + .build(); + + keyStoreOptions.storeKeyStore(keyStore); } private X509Certificate sendCsr(PKCS10CertificationRequest csr) { diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/containerdata/PromptContainerData.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/containerdata/PromptContainerData.java index ea9b2312b77..8b6be32f649 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/containerdata/PromptContainerData.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/containerdata/PromptContainerData.java @@ -24,7 +24,7 @@ public class PromptContainerData { "color_off='\\[\\e[0m\\]' # Text Reset\n" + "color_bold='\\[\\e[1m\\]' # Bold text\n" + "\n" + - "env_colour=#if($zone.getSystem() == \"main\")#if($zone.getEnvironment() == \"prod\")'\\e[0;91m'#else'\\e[0;33m'#end#else$green#end\n" + + "env_colour=#if($zone.getSystem() == \"main\")#if($zone.getEnvironment() == \"prod\")'\\[\\e[0;91m\\]'#else'\\[\\e[0;33m\\]'#end#else'\\[\\e[0;32m\\]'#end\n" + "\n" + "\n" + "PS1=\"${env_colour}$zone.getRegion().toUpperCase()${color_off} [\\u@${color_bold}\\h${color_off}:\\w]\\$ \"\n" + diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java index 26e7f3846ac..da23e1cb3c9 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java @@ -92,7 +92,8 @@ public class DockerOperationsImpl implements DockerOperations { .withAddCapability("SYS_PTRACE") // Needed for gcore, pstack etc. .withAddCapability("SYS_ADMIN"); // Needed for perf - if (environment.getNodeType() == NodeType.confighost) { + if (environment.getNodeType() == NodeType.confighost || + environment.getNodeType() == NodeType.proxyhost) { command.withVolume("/var/lib/sia", "/var/lib/sia"); } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java index 336e947d7b4..f08c91ec9c0 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java @@ -29,6 +29,7 @@ import com.yahoo.vespa.hosted.node.admin.component.Environment; import com.yahoo.vespa.hosted.node.admin.util.PrefixLogger; import com.yahoo.vespa.hosted.provision.Node; +import java.io.UncheckedIOException; import java.text.SimpleDateFormat; import java.time.Clock; import java.time.Duration; @@ -688,18 +689,21 @@ public class NodeAgentImpl implements NodeAgent { ContainerName.fromHostname(nodeSpec.hostname)); // ContainerData only works when root, which is the case only for HostAdmin so far -- config nodes are only used under HostAdmin. + // If this fails, however, we should fail the start-up, as the config server won't work without it. Thus, no catch here. if (nodeSpec.nodeType.equals(NodeType.config.name())) { logger.info("Creating files needed by config server"); new ConfigServerContainerData(environment, nodeSpec.hostname).writeTo(containerData); } - // ContainerData only works when root, which is the case only for HostAdmin so far -- only AWS uses HostAdmin now. - if (environment.getRegion().startsWith("aws-")) { + // ContainerData only works when root, which is the case only for HostAdmin so far. Allow this to fail, since it's not critical. + try { logger.info("Creating files for message of the day and the bash prompt"); new MotdContainerData(nodeSpec, environment).writeTo(containerData); new PromptContainerData(environment).writeTo(containerData); } - + catch (UncheckedIOException e) { + logger.info("Failed creating files for message of the day and the bash prompt", e); + } } } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/util/KeyStoreOptions.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/util/KeyStoreOptions.java index 1115f6dca91..03aff7f22d8 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/util/KeyStoreOptions.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/util/KeyStoreOptions.java @@ -1,45 +1,32 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.node.admin.util; -import java.io.FileInputStream; -import java.io.IOException; +import com.yahoo.vespa.athenz.tls.KeyStoreBuilder; +import com.yahoo.vespa.athenz.tls.KeyStoreType; +import com.yahoo.vespa.athenz.tls.KeyStoreUtils; + import java.nio.file.Path; import java.security.KeyStore; -import java.security.KeyStoreException; -import java.security.NoSuchAlgorithmException; -import java.security.NoSuchProviderException; -import java.security.cert.CertificateException; -import java.util.Optional; public class KeyStoreOptions { public final Path path; public final char[] password; - public final String type; - private final Optional<String> provider; + public final KeyStoreType keyStoreType; public KeyStoreOptions(Path path, char[] password, String type) { - this(path, password, type, null); - } - - public KeyStoreOptions(Path path, char[] password, String type, String provider) { this.path = path; this.password = password; - this.type = type; - this.provider = Optional.ofNullable(provider); + this.keyStoreType = KeyStoreType.valueOf(type); } - public KeyStore loadKeyStore() - throws IOException, NoSuchProviderException, KeyStoreException, CertificateException, NoSuchAlgorithmException { - try (FileInputStream in = new FileInputStream(path.toFile())) { - KeyStore keyStore = getKeyStoreInstance(); - keyStore.load(in, password); - return keyStore; - } + public KeyStore loadKeyStore() { + return KeyStoreBuilder + .withType(keyStoreType) + .fromFile(path.toFile(), password) + .build(); } - public KeyStore getKeyStoreInstance() throws NoSuchProviderException, KeyStoreException { - return provider.isPresent() ? - KeyStore.getInstance(type, provider.get()) : - KeyStore.getInstance(type); + public void storeKeyStore(KeyStore keyStore) { + KeyStoreUtils.writeKeyStoreToFile(keyStore, path.toFile(), password); } } diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/certificate/ConfigServerKeyStoreRefresherTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/certificate/ConfigServerKeyStoreRefresherTest.java index f9f8b230154..85684ea3bd4 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/certificate/ConfigServerKeyStoreRefresherTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/certificate/ConfigServerKeyStoreRefresherTest.java @@ -54,7 +54,7 @@ public class ConfigServerKeyStoreRefresherTest { @Before public void setup() { keyStoreOptions = new KeyStoreOptions( - tempFolder.getRoot().toPath().resolve("some/path/keystore.p12"), new char[0], "PKCS12", null); + tempFolder.getRoot().toPath().resolve("some/path/keystore.p12"), new char[0], "PKCS12"); } @Test diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/Authorizer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/Authorizer.java index 29e98a4b0cb..e69d551c86a 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/Authorizer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/Authorizer.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.hosted.provision.restapi.v2; import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.SystemName; +import com.yahoo.net.HostName; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeRepository; import org.apache.http.NameValuePair; @@ -16,6 +17,7 @@ import java.util.Collections; import java.util.List; import java.util.Optional; import java.util.function.BiPredicate; +import java.util.function.Supplier; import java.util.stream.Collectors; /** @@ -28,10 +30,16 @@ public class Authorizer implements BiPredicate<Principal, URI> { private final SystemName system; private final NodeRepository nodeRepository; + private final Supplier<String> hostnameSupplier; public Authorizer(SystemName system, NodeRepository nodeRepository) { + this(system, nodeRepository, HostName::getLocalhost); + } + + Authorizer(SystemName system, NodeRepository nodeRepository, Supplier<String> hostnameSupplier) { this.system = system; this.nodeRepository = nodeRepository; + this.hostnameSupplier = hostnameSupplier; } /** Returns whether principal is authorized to access given URI */ @@ -42,7 +50,7 @@ public class Authorizer implements BiPredicate<Principal, URI> { return true; } - // Nodes can only access its own resources + // Individual nodes can only access their own resources if (canAccess(hostnamesFrom(uri), principal, this::isSelfOrParent)) { return true; } @@ -52,6 +60,11 @@ public class Authorizer implements BiPredicate<Principal, URI> { return true; } + // The host itself can access all resources + if (isLocalhost(principal)) { + return true; + } + return false; } @@ -74,6 +87,11 @@ public class Authorizer implements BiPredicate<Principal, URI> { .orElse(false); } + /** Returns whether given principal is the hostname of this node */ + private boolean isLocalhost(Principal principal) { + return principal.getName().equals(hostnameSupplier.get()); + } + /** Returns whether principal can access all given resources */ private <T> boolean canAccess(List<T> resources, Principal principal, BiPredicate<T, Principal> predicate) { return !resources.isEmpty() && resources.stream().allMatch(resource -> predicate.test(resource, principal)); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/filter/AuthorizationFilter.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/filter/AuthorizationFilter.java index 64af3d5c0ca..41f6d0ef4bc 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/filter/AuthorizationFilter.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/filter/AuthorizationFilter.java @@ -3,8 +3,6 @@ package com.yahoo.vespa.hosted.provision.restapi.v2.filter; import com.google.inject.Inject; import com.yahoo.config.provision.Zone; -import com.yahoo.jdisc.handler.FastContentWriter; -import com.yahoo.jdisc.handler.ResponseDispatch; import com.yahoo.jdisc.handler.ResponseHandler; import com.yahoo.jdisc.http.filter.DiscFilterRequest; import com.yahoo.jdisc.http.filter.SecurityRequestFilter; @@ -17,9 +15,6 @@ import org.bouncycastle.asn1.x500.style.BCStyle; import org.bouncycastle.asn1.x500.style.IETFUtils; import org.bouncycastle.cert.jcajce.JcaX509CertificateHolder; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.UncheckedIOException; import java.net.URI; import java.security.Principal; import java.security.cert.CertificateEncodingException; @@ -70,20 +65,6 @@ public class AuthorizationFilter implements SecurityRequestFilter { } } - /** Write error response */ - static void write(ErrorResponse response, ResponseHandler handler) { - try (FastContentWriter writer = ResponseDispatch.newInstance(response.getJdiscResponse()) - .connectFastWriter(handler)) { - ByteArrayOutputStream out = new ByteArrayOutputStream(); - try { - response.render(out); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - writer.write(out.toByteArray()); - } - } - /** Log error response without writing anything */ private static void log(ErrorResponse response, @SuppressWarnings("unused") ResponseHandler handler) { log.warning("Would reject request: " + response.getStatus() + " - " + response.message()); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/filter/FilterUtils.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/filter/FilterUtils.java new file mode 100644 index 00000000000..82d82f4694e --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/filter/FilterUtils.java @@ -0,0 +1,35 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.restapi.v2.filter; + +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.jdisc.handler.FastContentWriter; +import com.yahoo.jdisc.handler.ResponseDispatch; +import com.yahoo.jdisc.handler.ResponseHandler; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.UncheckedIOException; + +/** + * @author mpolden + */ +public class FilterUtils { + + private FilterUtils() {} + + /** Write HTTP response using given handler */ + public static void write(HttpResponse response, ResponseHandler handler) { + response.headers().put("Content-Type", response.getContentType()); + try (FastContentWriter writer = ResponseDispatch.newInstance(response.getJdiscResponse()) + .connectFastWriter(handler)) { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + try { + response.render(out); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + writer.write(out.toByteArray()); + } + } + +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/filter/LocalhostFilter.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/filter/LocalhostFilter.java index 00e89582dff..8276299a225 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/filter/LocalhostFilter.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/filter/LocalhostFilter.java @@ -1,17 +1,11 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.restapi.v2.filter; -import com.yahoo.jdisc.handler.FastContentWriter; -import com.yahoo.jdisc.handler.ResponseDispatch; import com.yahoo.jdisc.handler.ResponseHandler; import com.yahoo.jdisc.http.filter.DiscFilterRequest; import com.yahoo.jdisc.http.filter.SecurityRequestFilter; import com.yahoo.vespa.hosted.provision.restapi.v2.ErrorResponse; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.UncheckedIOException; - /** * A security filter that rejects requests not originating from localhost. * @@ -21,7 +15,7 @@ import java.io.UncheckedIOException; public class LocalhostFilter implements SecurityRequestFilter { private static final String inet4Loopback = "127.0.0.1"; - private static final String inet6Loopback = "::1"; + private static final String inet6Loopback = "0:0:0:0:0:0:0:1"; @Override public void filter(DiscFilterRequest request, ResponseHandler handler) { @@ -30,22 +24,10 @@ public class LocalhostFilter implements SecurityRequestFilter { case inet6Loopback: break; // Allowed default: - write(ErrorResponse.unauthorized(String.format("%s %s denied for %s: Unauthorized host", - request.getMethod(), request.getUri().getPath(), - request.getRemoteAddr())), handler); + FilterUtils.write(ErrorResponse.unauthorized( + String.format("%s %s denied for %s: Unauthorized host", request.getMethod(), + request.getUri().getPath(), request.getRemoteAddr())), handler); } } - private static void write(ErrorResponse response, ResponseHandler handler) { - try (FastContentWriter writer = ResponseDispatch.newInstance(response.getJdiscResponse()) - .connectFastWriter(handler)) { - ByteArrayOutputStream out = new ByteArrayOutputStream(); - try { - response.render(out); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - writer.write(out.toByteArray()); - } - } } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/AuthorizerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/AuthorizerTest.java index d6bdc88da8a..9aa23436d45 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/AuthorizerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/AuthorizerTest.java @@ -35,7 +35,7 @@ public class AuthorizerTest { public void before() { NodeFlavors flavors = new MockNodeFlavors(); nodeRepository = new MockNodeRepository(new MockCurator(), flavors); - authorizer = new Authorizer(SystemName.main, nodeRepository); + authorizer = new Authorizer(SystemName.main, nodeRepository, () -> "cfg-host"); { // Populate with nodes used in this test. Note that only nodes requiring node repository lookup are added here Set<String> ipAddresses = new HashSet<>(Arrays.asList("127.0.0.1", "::1")); Flavor flavor = flavors.getFlavorOrThrow("default"); @@ -136,6 +136,12 @@ public class AuthorizerTest { assertTrue(authorized("proxy1", "/routing/v1/status")); } + @Test + public void host_authorization() { + assertTrue(authorized("cfg-host", "/")); + assertTrue(authorized("cfg-host", "/application/v2")); + } + private boolean authorized(String principal, String path) { return authorizer.test(() -> principal, uri(path)); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/filter/AuthorizationFilterTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/filter/AuthorizationFilterTest.java index b14cc570a75..b245f2b29d0 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/filter/AuthorizationFilterTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/filter/AuthorizationFilterTest.java @@ -23,7 +23,7 @@ public class AuthorizationFilterTest { tester = new FilterTester(new AuthorizationFilter(new Authorizer(SystemName.main, new MockNodeRepository(new MockCurator(), new MockNodeFlavors())), - AuthorizationFilter::write)); + FilterUtils::write)); } @Test diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/filter/FilterTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/filter/FilterTester.java index 3916be9d826..d3e98ae2c84 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/filter/FilterTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/filter/FilterTester.java @@ -28,6 +28,8 @@ import java.time.Duration; import java.time.Instant; import java.util.Collections; import java.util.Date; +import java.util.List; +import java.util.Map; import java.util.Optional; import static org.junit.Assert.assertEquals; @@ -55,6 +57,8 @@ public class FilterTester { Optional<Response> response = getResponse(request); assertTrue("Expected response from filter", response.isPresent()); assertEquals("Response body", body, response.get().body); + assertEquals("Content type", "application/json", + response.get().headers.get("Content-Type").get(0)); assertEquals("Status code", status, response.get().status); } @@ -62,7 +66,7 @@ public class FilterTester { RequestHandlerTestDriver.MockResponseHandler handler = new RequestHandlerTestDriver.MockResponseHandler(); filter.filter(toDiscFilterRequest(request), handler); return Optional.ofNullable(handler.getResponse()) - .map(response -> new Response(response.getStatus(), handler.readAll())); + .map(response -> new Response(response.getStatus(), response.headers(), handler.readAll())); } private static DiscFilterRequest toDiscFilterRequest(Request request) { @@ -113,10 +117,12 @@ public class FilterTester { private static class Response { private final int status; + private final Map<String, List<String>> headers; private final String body; - private Response(int status, String body) { + private Response(int status, Map<String, List<String>> headers, String body) { this.status = status; + this.headers = headers; this.body = body; } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/filter/LocalhostFilterTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/filter/LocalhostFilterTest.java index b65cf31aca4..d7a6c96be1a 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/filter/LocalhostFilterTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/filter/LocalhostFilterTest.java @@ -25,7 +25,7 @@ public class LocalhostFilterTest { "1.2.3.4: Unauthorized host\"}"); tester.assertSuccess(new Request(Method.GET, "/").remoteAddr("127.0.0.1")); - tester.assertSuccess(new Request(Method.GET, "/").remoteAddr("::1")); + tester.assertSuccess(new Request(Method.GET, "/").remoteAddr("0:0:0:0:0:0:0:1")); } } diff --git a/searchcore/src/apps/tests/persistenceconformance_test.cpp b/searchcore/src/apps/tests/persistenceconformance_test.cpp index 5b50ec49844..3840cfb7869 100644 --- a/searchcore/src/apps/tests/persistenceconformance_test.cpp +++ b/searchcore/src/apps/tests/persistenceconformance_test.cpp @@ -286,7 +286,7 @@ protected: class MyPersistenceEngineOwner : public IPersistenceEngineOwner { virtual void - setClusterState(const storage::spi::ClusterState &calc) override + setClusterState(BucketSpace, const storage::spi::ClusterState &calc) override { (void) calc; } diff --git a/searchcore/src/tests/proton/persistenceengine/persistenceengine_test.cpp b/searchcore/src/tests/proton/persistenceengine/persistenceengine_test.cpp index 4b73e4ca115..a016a56edf7 100644 --- a/searchcore/src/tests/proton/persistenceengine/persistenceengine_test.cpp +++ b/searchcore/src/tests/proton/persistenceengine/persistenceengine_test.cpp @@ -393,7 +393,7 @@ HandlerSet::prepareGetModifiedBuckets() class SimplePersistenceEngineOwner : public IPersistenceEngineOwner { - void setClusterState(const storage::spi::ClusterState &calc) override { (void) calc; } + void setClusterState(BucketSpace, const storage::spi::ClusterState &calc) override { (void) calc; } }; struct SimpleResourceWriteFilter : public IResourceWriteFilter diff --git a/searchcore/src/vespa/searchcore/proton/persistenceengine/ipersistenceengineowner.h b/searchcore/src/vespa/searchcore/proton/persistenceengine/ipersistenceengineowner.h index e8ca0bdce9b..bf07aaa82d8 100644 --- a/searchcore/src/vespa/searchcore/proton/persistenceengine/ipersistenceengineowner.h +++ b/searchcore/src/vespa/searchcore/proton/persistenceengine/ipersistenceengineowner.h @@ -16,7 +16,7 @@ public: } virtual void - setClusterState(const storage::spi::ClusterState &calc) = 0; + setClusterState(document::BucketSpace bucketSpace, const storage::spi::ClusterState &calc) = 0; }; } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.cpp b/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.cpp index e2b389fb898..0950a596f5b 100644 --- a/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.cpp +++ b/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.cpp @@ -285,7 +285,7 @@ PersistenceEngine::setClusterState(BucketSpace bucketSpace, const ClusterState & handler->handleSetClusterState(calc, resultHandler); } resultHandler.await(); - _owner.setClusterState(calc); + _owner.setClusterState(bucketSpace, calc); return resultHandler.getResult(); } diff --git a/searchcore/src/vespa/searchcore/proton/server/proton.cpp b/searchcore/src/vespa/searchcore/proton/server/proton.cpp index 533d0737ca2..a6c2fb730a9 100644 --- a/searchcore/src/vespa/searchcore/proton/server/proton.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/proton.cpp @@ -201,7 +201,9 @@ Proton::Proton(const config::ConfigUri & configUri, _initStarted(false), _initComplete(false), _initDocumentDbsInSequence(false), - _documentDBReferenceRegistry() + _documentDBReferenceRegistry(), + _nodeUpLock(), + _nodeUp() { _documentDBReferenceRegistry = std::make_shared<DocumentDBReferenceRegistry>(); } @@ -763,15 +765,27 @@ Proton::getConfigGeneration() return _protonConfigurer.getActiveConfigSnapshot()->getBootstrapConfig()->getGeneration(); } +bool +Proton::updateNodeUp(BucketSpace bucketSpace, bool nodeUpInBucketSpace) +{ + std::lock_guard guard(_nodeUpLock); + if (nodeUpInBucketSpace) { + _nodeUp.insert(bucketSpace); + } else { + _nodeUp.erase(bucketSpace); + } + return !_nodeUp.empty(); +} void -Proton::setClusterState(const storage::spi::ClusterState &calc) +Proton::setClusterState(BucketSpace bucketSpace, const storage::spi::ClusterState &calc) { // forward info sent by cluster controller to persistence engine // about whether node is supposed to be up or not. Match engine // needs to know this in order to stop serving queries. - bool nodeUp(calc.nodeUp()); + bool nodeUpInBucketSpace(calc.nodeUp()); bool nodeRetired(calc.nodeRetired()); + bool nodeUp = updateNodeUp(bucketSpace, nodeUpInBucketSpace); _matchEngine->setNodeUp(nodeUp); if (_memoryFlushConfigUpdater) { _memoryFlushConfigUpdater->setNodeRetired(nodeRetired); diff --git a/searchcore/src/vespa/searchcore/proton/server/proton.h b/searchcore/src/vespa/searchcore/proton/server/proton.h index 501b5ab8c62..e7e7153be8b 100644 --- a/searchcore/src/vespa/searchcore/proton/server/proton.h +++ b/searchcore/src/vespa/searchcore/proton/server/proton.h @@ -56,7 +56,6 @@ private: typedef std::map<DocTypeName, DocumentDB::SP> DocumentDBMap; typedef BootstrapConfig::ProtonConfigSP ProtonConfigSP; using InitializeThreads = std::shared_ptr<vespalib::ThreadStackExecutorBase>; - using lock_guard = std::lock_guard<std::mutex>; using BucketSpace = document::BucketSpace; struct MetricsUpdateHook : metrics::UpdateHook @@ -123,6 +122,8 @@ private: bool _initComplete; bool _initDocumentDbsInSequence; std::shared_ptr<IDocumentDBReferenceRegistry> _documentDBReferenceRegistry; + std::mutex _nodeUpLock; + std::set<BucketSpace> _nodeUp; // bucketspaces where node is up IDocumentDBConfigOwner * addDocumentDB(const DocTypeName & docTypeName, BucketSpace bucketSpace, const vespalib::string & configid, @@ -145,6 +146,7 @@ private: uint32_t getDistributionKey() const override { return _distributionKey; } BootstrapConfig::SP getActiveConfigSnapshot() const; std::shared_ptr<IDocumentDBReferenceRegistry> getDocumentDBReferenceRegistry() const override; + bool updateNodeUp(BucketSpace bucketSpace, bool nodeUpInBucketSpace); public: typedef std::unique_ptr<Proton> UP; typedef std::shared_ptr<Proton> SP; @@ -177,7 +179,7 @@ public: bool prepareRestart(); void getComponentConfig(Consumer &consumer) override; - void setClusterState(const storage::spi::ClusterState &calc) override; + void setClusterState(BucketSpace bucketSpace, const storage::spi::ClusterState &calc) override; // Return the oldest active config generation used by proton. int64_t getConfigGeneration(); diff --git a/vespaclient-java/src/main/java/com/yahoo/dummyreceiver/DummyReceiver.java b/vespaclient-java/src/main/java/com/yahoo/dummyreceiver/DummyReceiver.java index e6c644353ad..f8b785fd4e3 100755 --- a/vespaclient-java/src/main/java/com/yahoo/dummyreceiver/DummyReceiver.java +++ b/vespaclient-java/src/main/java/com/yahoo/dummyreceiver/DummyReceiver.java @@ -44,6 +44,7 @@ public class DummyReceiver implements MessageHandler { long maxQueueTime = -1; BlockingQueue<Runnable> queue; boolean verbose = false; + private boolean helpOption = false; DummyReceiver() { } @@ -144,6 +145,7 @@ public class DummyReceiver implements MessageHandler { if (arg.equals("-h") || arg.equals("--help")) { help(); + helpOption = true; return false; } else if ("--name".equals(arg)) { name = getParam(args, arg); @@ -161,6 +163,7 @@ public class DummyReceiver implements MessageHandler { verbose = true; } else { help(); + helpOption = true; return false; } } @@ -180,9 +183,12 @@ public class DummyReceiver implements MessageHandler { for (String arg : args) { l.add(arg); } - if (!rcv.parseArgs(l)) { + if (!rcv.parseArgs(l) && !rcv.helpOption) { System.exit(1); } + if (rcv.helpOption) { + System.exit(0); // exit with success instead of returning + } rcv.init(); while (true) { |