// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.model.container.xml;
import com.yahoo.cloud.config.ZookeeperServerConfig;
import com.yahoo.component.ComponentId;
import com.yahoo.config.application.api.ApplicationPackage;
import com.yahoo.config.model.NullConfigModelRegistry;
import com.yahoo.config.model.api.ApplicationClusterEndpoint;
import com.yahoo.config.model.api.ContainerEndpoint;
import com.yahoo.config.model.api.EndpointCertificateSecrets;
import com.yahoo.config.model.api.ModelContext;
import com.yahoo.config.model.api.TenantSecretStore;
import com.yahoo.config.model.builder.xml.test.DomBuilderTest;
import com.yahoo.config.model.deploy.DeployState;
import com.yahoo.config.model.deploy.TestProperties;
import com.yahoo.config.model.producer.AbstractConfigProducerRoot;
import com.yahoo.config.model.provision.InMemoryProvisioner;
import com.yahoo.config.model.provision.SingleNodeProvisioner;
import com.yahoo.config.model.test.MockApplicationPackage;
import com.yahoo.config.model.test.MockRoot;
import com.yahoo.config.provision.Environment;
import com.yahoo.config.provision.Flavor;
import com.yahoo.config.provision.RegionName;
import com.yahoo.config.provision.SystemName;
import com.yahoo.config.provision.Zone;
import com.yahoo.config.provisioning.FlavorsConfig;
import com.yahoo.container.ComponentsConfig;
import com.yahoo.container.QrConfig;
import com.yahoo.container.core.ChainsConfig;
import com.yahoo.container.core.VipStatusConfig;
import com.yahoo.container.di.config.PlatformBundlesConfig;
import com.yahoo.container.handler.VipStatusHandler;
import com.yahoo.container.handler.metrics.MetricsV2Handler;
import com.yahoo.container.handler.observability.ApplicationStatusHandler;
import com.yahoo.container.jdisc.JdiscBindingsConfig;
import com.yahoo.container.jdisc.secretstore.SecretStoreConfig;
import com.yahoo.container.usability.BindingsOverviewHandler;
import com.yahoo.jdisc.http.ConnectorConfig;
import com.yahoo.net.HostName;
import com.yahoo.path.Path;
import com.yahoo.prelude.cluster.QrMonitorConfig;
import com.yahoo.search.config.QrStartConfig;
import com.yahoo.security.X509CertificateUtils;
import com.yahoo.security.tls.TlsContext;
import com.yahoo.vespa.defaults.Defaults;
import com.yahoo.vespa.model.AbstractService;
import com.yahoo.vespa.model.VespaModel;
import com.yahoo.vespa.model.container.ApplicationContainer;
import com.yahoo.vespa.model.container.ApplicationContainerCluster;
import com.yahoo.vespa.model.container.ContainerCluster;
import com.yahoo.vespa.model.container.ContainerModelEvaluation;
import com.yahoo.vespa.model.container.SecretStore;
import com.yahoo.vespa.model.container.component.Component;
import com.yahoo.vespa.model.container.http.ConnectorFactory;
import com.yahoo.vespa.model.content.utils.ContentClusterUtils;
import com.yahoo.vespa.model.test.VespaModelTester;
import com.yahoo.vespa.model.test.utils.VespaModelCreatorWithFilePkg;
import org.hamcrest.Matchers;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
import org.w3c.dom.Element;
import org.xml.sax.SAXException;
import java.io.IOException;
import java.io.StringReader;
import java.time.Duration;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.function.Function;
import java.util.logging.Level;
import java.util.stream.Collectors;
import static com.yahoo.config.model.test.TestUtil.joinLines;
import static com.yahoo.test.LinePatternMatcher.containsLineWithPattern;
import static com.yahoo.vespa.defaults.Defaults.getDefaults;
import static com.yahoo.vespa.model.container.ContainerCluster.ROOT_HANDLER_BINDING;
import static com.yahoo.vespa.model.container.ContainerCluster.STATE_HANDLER_BINDING_1;
import static org.hamcrest.CoreMatchers.is;
import static org.hamcrest.CoreMatchers.not;
import static org.hamcrest.Matchers.contains;
import static org.hamcrest.Matchers.containsInAnyOrder;
import static org.hamcrest.Matchers.containsString;
import static org.hamcrest.Matchers.hasItem;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
/**
* Tests for "core functionality" of the container model, e.g. ports, or the 'components' and 'bundles' configs.
*
* Before adding a new test to this class, check if the test fits into one of the other existing subclasses
* of {@link ContainerModelBuilderTestBase}. If not, consider creating a new subclass.
*
* @author gjoranv
*/
public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase {
@Rule
public TemporaryFolder applicationFolder = new TemporaryFolder();
@Test
public void model_evaluation_bundles_are_deployed() {
createBasicContainerModel();
PlatformBundlesConfig config = root.getConfig(PlatformBundlesConfig.class, "default");
assertTrue(config.bundlePaths().contains(ContainerModelEvaluation.MODEL_EVALUATION_BUNDLE_FILE.toString()));
assertTrue(config.bundlePaths().contains(ContainerModelEvaluation.MODEL_INTEGRATION_BUNDLE_FILE.toString()));
}
@Test
public void deprecated_jdisc_tag_is_allowed() {
Element clusterElem = DomBuilderTest.parse(
"",
nodesXml,
"" );
TestLogger logger = new TestLogger();
createModel(root, logger, clusterElem);
AbstractService container = (AbstractService)root.getProducer("jdisc/container.0");
assertNotNull(container);
assertFalse(logger.msgs.isEmpty());
assertEquals(Level.WARNING, logger.msgs.get(0).getFirst());
assertEquals("'jdisc' is deprecated as tag name. Use 'container' instead.", logger.msgs.get(0).getSecond());
}
@Test
public void default_port_is_4080() {
Element clusterElem = DomBuilderTest.parse(
"",
nodesXml,
"" );
createModel(root, clusterElem);
AbstractService container = (AbstractService)root.getProducer("container/container.0");
assertEquals(getDefaults().vespaWebServicePort(), container.getRelativePort(0));
}
@Test
public void http_server_port_is_configurable_and_does_not_affect_other_ports() {
Element clusterElem = DomBuilderTest.parse(
"",
" ",
" ",
" ",
nodesXml,
"" );
createModel(root, clusterElem);
AbstractService container = (AbstractService)root.getProducer("container/container.0");
assertEquals(9000, container.getRelativePort(0));
assertNotEquals(9001, container.getRelativePort(1));
}
@Test
public void omitting_http_server_port_gives_default() {
Element clusterElem = DomBuilderTest.parse(
"",
" ",
" ",
" ",
nodesXml,
"" );
createModel(root, clusterElem);
AbstractService container = (AbstractService)root.getProducer("container/container.0");
assertEquals(Defaults.getDefaults().vespaWebServicePort(), container.getRelativePort(0));
}
@Test
public void fail_if_http_port_is_not_default_in_hosted_vespa() throws Exception {
try {
String servicesXml =
"" +
"" +
" " +
"" +
"" +
" " +
" " +
" " +
nodesXml +
"" +
"";
ApplicationPackage applicationPackage = new MockApplicationPackage.Builder().withServices(servicesXml).build();
// Need to create VespaModel to make deploy properties have effect
TestLogger logger = new TestLogger();
new VespaModel(new NullConfigModelRegistry(), new DeployState.Builder()
.applicationPackage(applicationPackage)
.deployLogger(logger)
.properties(new TestProperties().setHostedVespa(true))
.build());
fail("Expected exception");
}
catch (IllegalArgumentException e) {
// Success
assertEquals("Illegal port 9000 in http server 'foo': Port must be set to " + Defaults.getDefaults().vespaWebServicePort(),
e.getMessage());
}
}
@Test
public void one_cluster_with_explicit_port_and_one_without_is_ok() {
Element cluster1Elem = DomBuilderTest.parse(
"");
Element cluster2Elem = DomBuilderTest.parse(
"",
" ",
" ",
" ",
"");
createModel(root, cluster1Elem, cluster2Elem);
}
@Test
public void two_clusters_without_explicit_port_throws_exception() {
Element cluster1Elem = DomBuilderTest.parse(
"",
nodesXml,
"" );
Element cluster2Elem = DomBuilderTest.parse(
"",
nodesXml,
"" );
try {
createModel(root, cluster1Elem, cluster2Elem);
fail("Expected exception");
} catch (RuntimeException e) {
assertThat(e.getMessage(), containsString("cannot reserve port"));
}
}
@Test
public void verify_bindings_for_builtin_handlers() {
createBasicContainerModel();
JdiscBindingsConfig config = root.getConfig(JdiscBindingsConfig.class, "default/container.0");
JdiscBindingsConfig.Handlers defaultRootHandler = config.handlers(BindingsOverviewHandler.class.getName());
assertThat(defaultRootHandler.serverBindings(), contains("http://*/"));
JdiscBindingsConfig.Handlers applicationStatusHandler = config.handlers(ApplicationStatusHandler.class.getName());
assertThat(applicationStatusHandler.serverBindings(), contains("http://*/ApplicationStatus"));
JdiscBindingsConfig.Handlers fileRequestHandler = config.handlers(VipStatusHandler.class.getName());
assertThat(fileRequestHandler.serverBindings(), contains("http://*/status.html"));
JdiscBindingsConfig.Handlers metricsV2Handler = config.handlers(MetricsV2Handler.class.getName());
assertThat(metricsV2Handler.serverBindings(), contains("http://*/metrics/v2", "http://*/metrics/v2/*"));
}
@Test
public void default_root_handler_binding_can_be_stolen_by_user_configured_handler() {
Element clusterElem = DomBuilderTest.parse(
"" +
" " +
" " + ROOT_HANDLER_BINDING.patternString() + "" +
" " +
"");
createModel(root, clusterElem);
// The handler is still set up.
ComponentsConfig.Components userRootHandler = getComponent(componentsConfig(), BindingsOverviewHandler.class.getName());
assertNotNull(userRootHandler);
// .. but it has no bindings
var discBindingsConfig = root.getConfig(JdiscBindingsConfig.class, "default");
assertNull(discBindingsConfig.handlers(BindingsOverviewHandler.class.getName()));
}
@Test
public void reserved_binding_cannot_be_stolen_by_user_configured_handler() {
Element clusterElem = DomBuilderTest.parse(
"" +
" " +
" " + STATE_HANDLER_BINDING_1.patternString() + "" +
" " +
"");
try {
createModel(root, clusterElem);
fail("Expected exception when stealing a reserved binding.");
} catch (IllegalArgumentException e) {
assertThat(e.getMessage(), is("Binding 'http://*/state/v1' is a reserved Vespa binding " +
"and cannot be used by handler: userHandler"));
}
}
@Test
public void handler_bindings_are_included_in_discBindings_config() {
createClusterWithJDiscHandler();
String discBindingsConfig = root.getConfig(JdiscBindingsConfig.class, "default").toString();
assertThat(discBindingsConfig, containsString("{discHandler}"));
assertThat(discBindingsConfig, containsString(".serverBindings[0] \"http://*/binding0\""));
assertThat(discBindingsConfig, containsString(".serverBindings[1] \"http://*/binding1\""));
assertThat(discBindingsConfig, containsString(".clientBindings[0] \"http://*/clientBinding\""));
}
@Test
public void handlers_are_included_in_components_config() {
createClusterWithJDiscHandler();
assertThat(componentsConfig().toString(), containsString(".id \"discHandler\""));
}
private void createClusterWithJDiscHandler() {
Element clusterElem = DomBuilderTest.parse(
"",
" ",
" http://*/binding0",
" http://*/binding1",
" http://*/clientBinding",
" ",
"");
createModel(root, clusterElem);
}
@Test
public void processing_handler_bindings_can_be_overridden() {
Element clusterElem = DomBuilderTest.parse(
"",
" ",
" http://*/binding0",
" http://*/binding1",
" ",
"");
createModel(root, clusterElem);
String discBindingsConfig = root.getConfig(JdiscBindingsConfig.class, "default").toString();
assertThat(discBindingsConfig, containsString(".serverBindings[0] \"http://*/binding0\""));
assertThat(discBindingsConfig, containsString(".serverBindings[1] \"http://*/binding1\""));
assertThat(discBindingsConfig, not(containsString("/processing/*")));
}
@Test
public void clientProvider_bindings_are_included_in_discBindings_config() {
createModelWithClientProvider();
String discBindingsConfig = root.getConfig(JdiscBindingsConfig.class, "default").toString();
assertThat(discBindingsConfig, containsString("{discClient}"));
assertThat(discBindingsConfig, containsString(".clientBindings[0] \"http://*/binding0\""));
assertThat(discBindingsConfig, containsString(".clientBindings[1] \"http://*/binding1\""));
assertThat(discBindingsConfig, containsString(".serverBindings[0] \"http://*/serverBinding\""));
}
@Test
public void clientProviders_are_included_in_components_config() {
createModelWithClientProvider();
assertThat(componentsConfig().toString(), containsString(".id \"discClient\""));
}
private void createModelWithClientProvider() {
Element clusterElem = DomBuilderTest.parse(
"" +
" " +
" http://*/binding0" +
" http://*/binding1" +
" http://*/serverBinding" +
" " +
"" );
createModel(root, clusterElem);
}
@Test
public void serverProviders_are_included_in_components_config() {
Element clusterElem = DomBuilderTest.parse(
"" +
" " +
"" );
createModel(root, clusterElem);
String componentsConfig = componentsConfig().toString();
assertThat(componentsConfig, containsString(".id \"discServer\""));
}
private String getChainsConfig(String configId) {
return root.getConfig(ChainsConfig.class, configId).toString();
}
@Test
public void searchHandler_gets_only_search_chains_in_chains_config() {
createClusterWithProcessingAndSearchChains();
String searchHandlerConfigId = "default/component/com.yahoo.search.handler.SearchHandler";
String chainsConfig = getChainsConfig(searchHandlerConfigId);
assertThat(chainsConfig, containsLineWithPattern(".*\\.id \"testSearcher@default\"$"));
assertThat(chainsConfig, not(containsLineWithPattern(".*\\.id \"testProcessor@default\"$")));
}
@Test
public void processingHandler_gets_only_processing_chains_in_chains_config() {
createClusterWithProcessingAndSearchChains();
String processingHandlerConfigId = "default/component/com.yahoo.processing.handler.ProcessingHandler";
String chainsConfig = getChainsConfig(processingHandlerConfigId);
assertThat(chainsConfig, containsLineWithPattern(".*\\.id \"testProcessor@default\"$"));
assertThat(chainsConfig, not(containsLineWithPattern(".*\\.id \"testSearcher@default\"$")));
}
private void createClusterWithProcessingAndSearchChains() {
Element clusterElem = DomBuilderTest.parse(
"" +
" " +
" " +
" " +
" " +
" " +
" " +
" " +
" " +
" " +
" " +
nodesXml +
" ");
createModel(root, clusterElem);
}
@Test
public void user_config_can_be_overridden_on_node() {
Element containerElem = DomBuilderTest.parse(
"",
" " +
" 111",
" " +
" ",
" ",
" ",
" ",
" 222",
" ",
" ",
" ",
"");
root = ContentClusterUtils.createMockRoot(new String[]{"host1", "host2"});
createModel(root, containerElem);
ContainerCluster cluster = (ContainerCluster)root.getChildren().get("default");
assertEquals(2, cluster.getContainers().size());
assertEquals(root.getConfig(QrMonitorConfig.class, "default/container.0").requesttimeout(), 111);
assertEquals(root.getConfig(QrMonitorConfig.class, "default/container.1").requesttimeout(), 222);
}
@Test
public void nested_components_are_injected_to_handlers() {
Element clusterElem = DomBuilderTest.parse(
"",
" ",
" ",
" ",
" ", // remember, a client is also a request handler
" ",
" ",
"");
createModel(root, clusterElem);
Component,?> handler = getContainerComponent("default", "myHandler");
assertThat(handler.getInjectedComponentIds(), hasItem("injected@myHandler"));
Component,?> client = getContainerComponent("default", "myClient");
assertThat(client.getInjectedComponentIds(), hasItem("injected@myClient"));
}
@Test
public void component_includes_are_added() {
VespaModelCreatorWithFilePkg creator = new VespaModelCreatorWithFilePkg("src/test/cfg/application/include_dirs");
VespaModel model = creator.create(true);
ContainerCluster cluster = model.getContainerClusters().get("default");
Map> componentsMap = cluster.getComponentsMap();
Component,?> example = componentsMap.get(
ComponentId.fromString("test.Exampledocproc"));
assertEquals("test.Exampledocproc", example.getComponentId().getName());
}
@Test
public void affinity_is_set() {
Element clusterElem = DomBuilderTest.parse(
"",
" ",
" ",
" ",
" ",
" ",
" " +
"");
createModel(root, clusterElem);
assertTrue(getContainerCluster("default").getContainers().get(0).getAffinity().isPresent());
assertEquals(0, getContainerCluster("default").getContainers().get(0).getAffinity().get().cpuSocket());
}
@Test
public void singlenode_servicespec_is_used_with_hosts_xml() throws IOException, SAXException {
String servicesXml = "";
String hostsXml = "\n" +
" \n" +
" node1\n" +
" \n" +
"";
ApplicationPackage applicationPackage = new MockApplicationPackage.Builder()
.withHosts(hostsXml)
.withServices(servicesXml)
.build();
VespaModel model = new VespaModel(applicationPackage);
assertEquals(1, model.hostSystem().getHosts().size());
}
@Test
public void http_aliases_are_stored_on_cluster_and_on_service_properties() {
Element clusterElem = DomBuilderTest.parse(
"",
" ",
" service1",
" service2",
" foo1.bar1.com",
" foo2.bar2.com",
" ",
" ",
" ",
" ",
"");
createModel(root, clusterElem);
assertEquals(getContainerCluster("default").serviceAliases().get(0), "service1");
assertEquals(getContainerCluster("default").endpointAliases().get(0), "foo1.bar1.com");
assertEquals(getContainerCluster("default").serviceAliases().get(1), "service2");
assertEquals(getContainerCluster("default").endpointAliases().get(1), "foo2.bar2.com");
assertEquals(getContainerCluster("default").getContainers().get(0).getServicePropertyString("servicealiases"), "service1,service2");
assertEquals(getContainerCluster("default").getContainers().get(0).getServicePropertyString("endpointaliases"), "foo1.bar1.com,foo2.bar2.com");
}
@Test
public void http_aliases_are_only_honored_in_prod_environment() {
Element clusterElem = DomBuilderTest.parse(
"",
" ",
" service1",
" foo1.bar1.com",
" ",
" ",
" ",
" ",
"");
DeployState deployState = new DeployState.Builder().zone(new Zone(Environment.dev, RegionName.from("us-east-1"))).build();
createModel(root, deployState, null, clusterElem);
assertEquals(0, getContainerCluster("default").serviceAliases().size());
assertEquals(0, getContainerCluster("default").endpointAliases().size());
assertNull(getContainerCluster("default").getContainers().get(0).getServicePropertyString("servicealiases"));
assertNull(getContainerCluster("default").getContainers().get(0).getServicePropertyString("endpointaliases"));
}
@Test
public void endpoints_are_added_to_containers() throws IOException, SAXException {
final var servicesXml = joinLines("",
"",
" ",
" ",
" ",
""
);
final var deploymentXml = joinLines("",
"",
" ",
""
);
final var applicationPackage = new MockApplicationPackage.Builder()
.withServices(servicesXml)
.withDeploymentSpec(deploymentXml)
.build();
final var deployState = new DeployState.Builder()
.applicationPackage(applicationPackage)
.zone(new Zone(Environment.prod, RegionName.from("us-east-1")))
.endpoints(Set.of(new ContainerEndpoint("comics-search", ApplicationClusterEndpoint.Scope.global, List.of("nalle", "balle"))))
.properties(new TestProperties().setHostedVespa(true))
.build();
final var model = new VespaModel(new NullConfigModelRegistry(), deployState);
final var containers = model.getContainerClusters().values().stream()
.flatMap(cluster -> cluster.getContainers().stream())
.collect(Collectors.toList());
assertFalse("Missing container objects based on configuration", containers.isEmpty());
containers.forEach(container -> {
final var rotations = container.getServicePropertyString("rotations").split(",");
final var rotationsSet = Set.of(rotations);
assertEquals(Set.of("balle", "nalle"), rotationsSet);
});
}
@Test
public void singlenode_servicespec_is_used_with_hosted_vespa() throws IOException, SAXException {
String servicesXml = "";
ApplicationPackage applicationPackage = new MockApplicationPackage.Builder().withServices(servicesXml).build();
VespaModel model = new VespaModel(new NullConfigModelRegistry(), new DeployState.Builder()
.modelHostProvisioner(new InMemoryProvisioner(true, false, "host1.yahoo.com", "host2.yahoo.com"))
.applicationPackage(applicationPackage)
.properties(new TestProperties()
.setMultitenant(true)
.setHostedVespa(true))
.build());
assertEquals(2, model.hostSystem().getHosts().size());
}
@Test(expected = IllegalArgumentException.class)
public void renderers_named_JsonRenderer_are_not_allowed() {
createModel(root, generateContainerElementWithRenderer("JsonRenderer"));
}
@Test(expected = IllegalArgumentException.class)
public void renderers_named_DefaultRenderer_are_not_allowed() {
createModel(root, generateContainerElementWithRenderer("XmlRenderer"));
}
@Test
public void renderers_named_something_else_are_allowed() {
createModel(root, generateContainerElementWithRenderer("my-little-renderer"));
}
@Test
public void vip_status_handler_uses_file_for_hosted_vespa() throws Exception {
String servicesXml = "" +
"" +
nodesXml +
"" +
"";
ApplicationPackage applicationPackage = new MockApplicationPackage.Builder().withServices(servicesXml).build();
VespaModel model = new VespaModel(new NullConfigModelRegistry(), new DeployState.Builder()
.applicationPackage(applicationPackage)
.properties(new TestProperties().setHostedVespa(true))
.build());
AbstractConfigProducerRoot modelRoot = model.getRoot();
VipStatusConfig vipStatusConfig = modelRoot.getConfig(VipStatusConfig.class, "container/component/status.html-status-handler");
assertTrue(vipStatusConfig.accessdisk());
assertEquals(ContainerModelBuilder.HOSTED_VESPA_STATUS_FILE, vipStatusConfig.statusfile());
}
@Test
public void qrconfig_is_produced() throws IOException, SAXException {
QrConfig qr = getQrConfig(new TestProperties());
String hostname = HostName.getLocalhost(); // Using the same way of getting hostname as filedistribution model
assertEquals("default.container.0", qr.discriminator());
assertEquals(19102, qr.rpc().port());
assertEquals("vespa/service/default/container.0", qr.rpc().slobrokId());
assertTrue(qr.rpc().enabled());
assertEquals("", qr.rpc().host());
assertFalse(qr.restartOnDeploy());
assertEquals("filedistribution/" + hostname, qr.filedistributor().configid());
assertEquals(50.0, qr.shutdown().timeout(), 0.00000000000001);
assertFalse(qr.shutdown().dumpHeapOnTimeout());
}
private QrConfig getQrConfig(ModelContext.Properties properties) throws IOException, SAXException {
String servicesXml =
"" +
" " +
" " +
" " +
" " +
" " +
" " +
" " +
" " +
"";
ApplicationPackage applicationPackage = new MockApplicationPackage.Builder()
.withServices(servicesXml)
.build();
VespaModel model = new VespaModel(new NullConfigModelRegistry(), new DeployState.Builder()
.applicationPackage(applicationPackage)
.properties(properties)
.build());
return model.getConfig(QrConfig.class, "default/container.0");
}
@Test
public void control_container_shutdown() throws IOException, SAXException {
QrConfig qr = getQrConfig(new TestProperties().containerShutdownTimeout(133).containerDumpHeapOnShutdownTimeout(true));
assertEquals(133.0, qr.shutdown().timeout(), 0.00000000000001);
assertTrue(qr.shutdown().dumpHeapOnTimeout());
}
@Test
public void secret_store_can_be_set_up() {
Element clusterElem = DomBuilderTest.parse(
"",
" ",
" ",
" ",
"");
createModel(root, clusterElem);
SecretStore secretStore = getContainerCluster("container").getSecretStore().get();
assertEquals("group1", secretStore.getGroups().get(0).name);
assertEquals("env1", secretStore.getGroups().get(0).environment);
}
@Test
public void cloud_secret_store_requires_configured_secret_store() {
Element clusterElem = DomBuilderTest.parse(
"",
" ",
" ",
" ",
" ",
" ",
"");
try {
DeployState state = new DeployState.Builder()
.properties(new TestProperties().setHostedVespa(true))
.zone(new Zone(SystemName.Public, Environment.prod, RegionName.defaultName()))
.build();
createModel(root, state, null, clusterElem);
fail("secret store not defined");
} catch (RuntimeException e) {
assertEquals("No configured secret store named store1", e.getMessage());
}
}
@Test
public void cloud_secret_store_can_be_set_up() {
Element clusterElem = DomBuilderTest.parse(
"",
" ",
" ",
" ",
" ",
" ",
"");
DeployState state = new DeployState.Builder()
.properties(
new TestProperties()
.setHostedVespa(true)
.setTenantSecretStores(List.of(new TenantSecretStore("store1", "1234", "role", Optional.of("externalid")))))
.zone(new Zone(SystemName.Public, Environment.prod, RegionName.defaultName()))
.build();
createModel(root, state, null, clusterElem);
ApplicationContainerCluster container = getContainerCluster("container");
assertComponentConfigured(container, "com.yahoo.jdisc.cloud.aws.AwsParameterStore");
CloudSecretStore secretStore = (CloudSecretStore) container.getComponentsMap().get(ComponentId.fromString("com.yahoo.jdisc.cloud.aws.AwsParameterStore"));
SecretStoreConfig.Builder configBuilder = new SecretStoreConfig.Builder();
secretStore.getConfig(configBuilder);
SecretStoreConfig secretStoreConfig = configBuilder.build();
assertEquals(1, secretStoreConfig.awsParameterStores().size());
assertEquals("store1", secretStoreConfig.awsParameterStores().get(0).name());
}
@Test
public void cloud_secret_store_fails_to_set_up_in_non_public_zone() {
try {
Element clusterElem = DomBuilderTest.parse(
"",
" ",
" ",
" ",
" ",
" ",
"");
DeployState state = new DeployState.Builder()
.properties(
new TestProperties()
.setHostedVespa(true)
.setTenantSecretStores(List.of(new TenantSecretStore("store1", "1234", "role", Optional.of("externalid")))))
.zone(new Zone(SystemName.main, Environment.prod, RegionName.defaultName()))
.build();
createModel(root, state, null, clusterElem);
} catch (RuntimeException e) {
assertEquals("Cloud secret store is not supported in non-public system, see the documentation",
e.getMessage());
return;
}
fail();
}
@Test
public void missing_security_clients_pem_fails_in_public() {
Element clusterElem = DomBuilderTest.parse("");
try {
DeployState state = new DeployState.Builder()
.properties(
new TestProperties()
.setHostedVespa(true)
.setEndpointCertificateSecrets(Optional.of(new EndpointCertificateSecrets("CERT", "KEY"))))
.zone(new Zone(SystemName.Public, Environment.prod, RegionName.defaultName()))
.build();
createModel(root, state, null, clusterElem);
} catch (RuntimeException e) {
assertEquals("Client certificate authority security/clients.pem is missing - see: https://cloud.vespa.ai/en/security-model#data-plane",
e.getMessage());
return;
}
fail();
}
@Test
public void security_clients_pem_is_picked_up() {
var applicationPackage = new MockApplicationPackage.Builder()
.withRoot(applicationFolder.getRoot())
.build();
applicationPackage.getFile(Path.fromString("security")).createDirectory();
applicationPackage.getFile(Path.fromString("security/clients.pem")).writeFile(new StringReader("I am a very nice certificate"));
var deployState = DeployState.createTestState(applicationPackage);
Element clusterElem = DomBuilderTest.parse("");
createModel(root, deployState, null, clusterElem);
assertEquals(Optional.of("I am a very nice certificate"), getContainerCluster("container").getTlsClientAuthority());
}
@Test
public void operator_certificates_are_joined_with_clients_pem() {
var applicationPackage = new MockApplicationPackage.Builder()
.withRoot(applicationFolder.getRoot())
.build();
var applicationTrustCert = X509CertificateUtils.toPem(
X509CertificateUtils.createSelfSigned("CN=application", Duration.ofDays(1)).certificate());
var operatorCert = X509CertificateUtils.createSelfSigned("CN=operator", Duration.ofDays(1)).certificate();
applicationPackage.getFile(Path.fromString("security")).createDirectory();
applicationPackage.getFile(Path.fromString("security/clients.pem")).writeFile(new StringReader(applicationTrustCert));
var deployState = new DeployState.Builder().properties(
new TestProperties()
.setOperatorCertificates(List.of(operatorCert))
.setHostedVespa(true)
.setEndpointCertificateSecrets(Optional.of(new EndpointCertificateSecrets("CERT", "KEY"))))
.zone(new Zone(SystemName.PublicCd, Environment.dev, RegionName.defaultName()))
.applicationPackage(applicationPackage)
.build();
Element clusterElem = DomBuilderTest.parse("");
createModel(root, deployState, null, clusterElem);
ApplicationContainer container = (ApplicationContainer)root.getProducer("container/container.0");
List connectorFactories = container.getHttp().getHttpServer().get().getConnectorFactories();
ConnectorFactory tlsPort = connectorFactories.stream().filter(connectorFactory -> connectorFactory.getListenPort() == 4443).findFirst().orElseThrow();
ConnectorConfig.Builder builder = new ConnectorConfig.Builder();
tlsPort.getConfig(builder);
ConnectorConfig connectorConfig = new ConnectorConfig(builder);
var caCerts = X509CertificateUtils.certificateListFromPem(connectorConfig.ssl().caCertificate());
assertEquals(2, caCerts.size());
List certnames = caCerts.stream()
.map(cert -> cert.getSubjectX500Principal().getName())
.collect(Collectors.toList());
assertThat(certnames, containsInAnyOrder("CN=operator", "CN=application"));
}
@Test
public void environment_vars_are_honoured() {
Element clusterElem = DomBuilderTest.parse(
"",
" ",
" ",
" 1",
" granularity=fine,verbose,compact,1,0",
" ",
" ",
" ",
"" );
createModel(root, clusterElem);
QrStartConfig.Builder qrStartBuilder = new QrStartConfig.Builder();
root.getConfig(qrStartBuilder, "container/container.0");
QrStartConfig qrStartConfig = new QrStartConfig(qrStartBuilder);
assertEquals("KMP_SETTING=1 KMP_AFFINITY=granularity=fine,verbose,compact,1,0 ", qrStartConfig.qrs().env());
}
private void verifyAvailableprocessors(boolean isHosted, Flavor flavor, int expectProcessors) {
DeployState deployState = new DeployState.Builder()
.modelHostProvisioner(flavor != null ? new SingleNodeProvisioner(flavor) : new SingleNodeProvisioner())
.properties(new TestProperties()
.setMultitenant(isHosted)
.setHostedVespa(isHosted))
.build();
MockRoot myRoot = new MockRoot("root", deployState);
Element clusterElem = DomBuilderTest.parse(
"",
" ",
" ",
" ",
""
);
createModel(myRoot, clusterElem);
QrStartConfig.Builder qsB = new QrStartConfig.Builder();
myRoot.getConfig(qsB, "container/container.0");
QrStartConfig qsC= new QrStartConfig(qsB);
assertEquals(expectProcessors, qsC.jvm().availableProcessors());
}
@Test
public void requireThatAvailableProcessorsFollowFlavor() {
verifyAvailableprocessors(false, null,0);
verifyAvailableprocessors(true, null,0);
verifyAvailableprocessors(true, new Flavor(new FlavorsConfig.Flavor.Builder().name("test-flavor").minCpuCores(9).build()), 9);
verifyAvailableprocessors(true, new Flavor(new FlavorsConfig.Flavor.Builder().name("test-flavor").minCpuCores(1).build()), 2);
}
@Test
public void requireThatProvidingEndpointCertificateSecretsOpensPort4443() {
Element clusterElem = DomBuilderTest.parse(
"",
nodesXml,
"" );
DeployState state = new DeployState.Builder().properties(new TestProperties().setHostedVespa(true).setEndpointCertificateSecrets(Optional.of(new EndpointCertificateSecrets("CERT", "KEY")))).build();
createModel(root, state, null, clusterElem);
ApplicationContainer container = (ApplicationContainer)root.getProducer("container/container.0");
// Verify that there are two connectors
List connectorFactories = container.getHttp().getHttpServer().get().getConnectorFactories();
assertEquals(2, connectorFactories.size());
List ports = connectorFactories.stream()
.map(ConnectorFactory::getListenPort)
.collect(Collectors.toList());
assertThat(ports, Matchers.containsInAnyOrder(8080, 4443));
ConnectorFactory tlsPort = connectorFactories.stream().filter(connectorFactory -> connectorFactory.getListenPort() == 4443).findFirst().orElseThrow();
ConnectorConfig.Builder builder = new ConnectorConfig.Builder();
tlsPort.getConfig(builder);
ConnectorConfig connectorConfig = new ConnectorConfig(builder);
assertTrue(connectorConfig.ssl().enabled());
assertEquals(ConnectorConfig.Ssl.ClientAuth.Enum.WANT_AUTH, connectorConfig.ssl().clientAuth());
assertEquals("CERT", connectorConfig.ssl().certificate());
assertEquals("KEY", connectorConfig.ssl().privateKey());
assertEquals(4443, connectorConfig.listenPort());
assertEquals("Connector must use Athenz truststore in a non-public system.",
"/opt/yahoo/share/ssl/certs/athenz_certificate_bundle.pem",
connectorConfig.ssl().caCertificateFile());
assertTrue(connectorConfig.ssl().caCertificate().isEmpty());
}
@Test
public void requireThatClientAuthenticationIsEnforced() {
Element clusterElem = DomBuilderTest.parse(
"",
nodesXml,
" " +
" " +
" " +
"" );
DeployState state = new DeployState.Builder().properties(
new TestProperties()
.setHostedVespa(true)
.setEndpointCertificateSecrets(Optional.of(new EndpointCertificateSecrets("CERT", "KEY"))))
.build();
createModel(root, state, null, clusterElem);
ApplicationContainer container = (ApplicationContainer)root.getProducer("container/container.0");
List connectorFactories = container.getHttp().getHttpServer().get().getConnectorFactories();
ConnectorFactory tlsPort = connectorFactories.stream().filter(connectorFactory -> connectorFactory.getListenPort() == 4443).findFirst().orElseThrow();
ConnectorConfig.Builder builder = new ConnectorConfig.Builder();
tlsPort.getConfig(builder);
ConnectorConfig connectorConfig = new ConnectorConfig(builder);
assertTrue(connectorConfig.ssl().enabled());
assertEquals(ConnectorConfig.Ssl.ClientAuth.Enum.NEED_AUTH, connectorConfig.ssl().clientAuth());
assertEquals("CERT", connectorConfig.ssl().certificate());
assertEquals("KEY", connectorConfig.ssl().privateKey());
assertEquals(4443, connectorConfig.listenPort());
assertEquals("Connector must use Athenz truststore in a non-public system.",
"/opt/yahoo/share/ssl/certs/athenz_certificate_bundle.pem",
connectorConfig.ssl().caCertificateFile());
assertTrue(connectorConfig.ssl().caCertificate().isEmpty());
}
@Test
public void require_allowed_ciphers() {
Element clusterElem = DomBuilderTest.parse(
"",
nodesXml,
"" );
DeployState state = new DeployState.Builder().properties(new TestProperties().setHostedVespa(true).setEndpointCertificateSecrets(Optional.of(new EndpointCertificateSecrets("CERT", "KEY")))).build();
createModel(root, state, null, clusterElem);
ApplicationContainer container = (ApplicationContainer)root.getProducer("container/container.0");
List connectorFactories = container.getHttp().getHttpServer().get().getConnectorFactories();
ConnectorFactory tlsPort = connectorFactories.stream().filter(connectorFactory -> connectorFactory.getListenPort() == 4443).findFirst().orElseThrow();
ConnectorConfig.Builder builder = new ConnectorConfig.Builder();
tlsPort.getConfig(builder);
ConnectorConfig connectorConfig = new ConnectorConfig(builder);
assertThat(connectorConfig.ssl().enabledCipherSuites(), containsInAnyOrder(TlsContext.ALLOWED_CIPHER_SUITES.toArray()));
}
@Test
public void cluster_with_zookeeper() {
Function servicesXml = (nodeCount) -> "" +
"" +
"" +
"";
VespaModelTester tester = new VespaModelTester();
tester.addHosts(3);
{
VespaModel model = tester.createModel(servicesXml.apply(3), true);
ApplicationContainerCluster cluster = model.getContainerClusters().get("default");
assertNotNull(cluster);
assertComponentConfigured(cluster,"com.yahoo.vespa.curator.Curator");
cluster.getContainers().forEach(container -> {
assertComponentConfigured(container, "com.yahoo.vespa.zookeeper.ReconfigurableVespaZooKeeperServer");
assertComponentConfigured(container, "com.yahoo.vespa.zookeeper.Reconfigurer");
assertComponentConfigured(container, "com.yahoo.vespa.zookeeper.VespaZooKeeperAdminImpl");
ZookeeperServerConfig config = model.getConfig(ZookeeperServerConfig.class, container.getConfigId());
assertEquals(container.index(), config.myid());
assertEquals(3, config.server().size());
});
}
{
try {
tester.createModel(servicesXml.apply(2), true);
fail("Expected exception");
} catch (IllegalArgumentException ignored) {}
}
{
String xmlWithNodes =
"" +
"" +
" " +
" " +
" " +
" " +
" " +
" " +
" " +
"";
try {
tester.createModel(xmlWithNodes, true);
fail("Expected exception");
} catch (IllegalArgumentException ignored) {}
}
}
@Test
public void logs_deployment_spec_deprecations() throws Exception {
String containerService = joinLines("",
" ",
" ",
" ",
"");
String deploymentXml = joinLines("",
" ",
" us-east-1",
" ",
"");
ApplicationPackage applicationPackage = new MockApplicationPackage.Builder()
.withServices(containerService)
.withDeploymentSpec(deploymentXml)
.build();
TestLogger logger = new TestLogger();
DeployState deployState = new DeployState.Builder()
.applicationPackage(applicationPackage)
.zone(new Zone(Environment.prod, RegionName.from("us-east-1")))
.properties(new TestProperties().setHostedVespa(true))
.deployLogger(logger)
.build();
createModel(root, deployState, null, DomBuilderTest.parse(containerService));
assertFalse(logger.msgs.isEmpty());
assertEquals(Level.WARNING, logger.msgs.get(0).getFirst());
assertEquals(Level.WARNING, logger.msgs.get(1).getFirst());
assertEquals("Element 'prod' contains deprecated attribute: 'global-service-id'. See https://cloud.vespa.ai/en/reference/routing#deprecated-syntax",
logger.msgs.get(0).getSecond());
assertEquals("Element 'region' contains deprecated attribute: 'active'. See https://cloud.vespa.ai/en/reference/routing#deprecated-syntax",
logger.msgs.get(1).getSecond());
}
private void assertComponentConfigured(ApplicationContainerCluster cluster, String componentId) {
Component, ?> component = cluster.getComponentsMap().get(ComponentId.fromString(componentId));
assertNotNull(component);
}
private void assertComponentConfigured(ApplicationContainer container, String id) {
assertTrue(container.getComponents().getComponents().stream().anyMatch(component -> id.equals(component.getComponentId().getName())));
}
private Element generateContainerElementWithRenderer(String rendererId) {
return DomBuilderTest.parse(
"",
" ",
String.format(" ", rendererId),
" ",
"");
}
}