// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.container.xml; import com.yahoo.component.ComponentId; import com.yahoo.component.ComponentSpecification; import com.yahoo.component.Version; import com.yahoo.component.chain.dependencies.Dependencies; import com.yahoo.component.chain.model.ChainedComponentModel; import com.yahoo.config.application.Xml; import com.yahoo.config.application.api.ApplicationFile; import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.application.api.DeploymentInstanceSpec; import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.model.ConfigModelContext; import com.yahoo.config.model.ConfigModelContext.ApplicationType; import com.yahoo.config.model.api.ApplicationClusterEndpoint; import com.yahoo.config.model.api.ConfigServerSpec; import com.yahoo.config.model.api.ContainerEndpoint; import com.yahoo.config.model.api.EndpointCertificateSecrets; import com.yahoo.config.model.api.TenantSecretStore; import com.yahoo.config.model.application.provider.IncludeDirs; import com.yahoo.config.model.builder.xml.ConfigModelBuilder; import com.yahoo.config.model.builder.xml.ConfigModelId; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.producer.AbstractConfigProducer; import com.yahoo.config.provision.AthenzDomain; import com.yahoo.config.provision.AthenzService; import com.yahoo.config.provision.Capacity; import com.yahoo.config.provision.ClusterMembership; import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.LoadBalancerSettings; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.Zone; import com.yahoo.container.bundle.BundleInstantiationSpecification; import com.yahoo.container.logging.FileConnectionLog; import com.yahoo.io.IOUtils; import com.yahoo.osgi.provider.model.ComponentModel; import com.yahoo.path.Path; import com.yahoo.schema.OnnxModel; import com.yahoo.schema.derived.RankProfileList; import com.yahoo.search.rendering.RendererRegistry; import com.yahoo.security.X509CertificateUtils; import com.yahoo.text.XML; import com.yahoo.vespa.defaults.Defaults; import com.yahoo.vespa.model.AbstractService; import com.yahoo.vespa.model.HostResource; import com.yahoo.vespa.model.HostSystem; import com.yahoo.vespa.model.builder.xml.dom.DomComponentBuilder; import com.yahoo.vespa.model.builder.xml.dom.DomHandlerBuilder; import com.yahoo.vespa.model.builder.xml.dom.ModelElement; import com.yahoo.vespa.model.builder.xml.dom.NodesSpecification; import com.yahoo.vespa.model.builder.xml.dom.VespaDomBuilder; import com.yahoo.vespa.model.builder.xml.dom.chains.docproc.DomDocprocChainsBuilder; import com.yahoo.vespa.model.builder.xml.dom.chains.processing.DomProcessingBuilder; import com.yahoo.vespa.model.builder.xml.dom.chains.search.DomSearchChainsBuilder; import com.yahoo.vespa.model.clients.ContainerDocumentApi; import com.yahoo.vespa.model.container.ApplicationContainer; import com.yahoo.vespa.model.container.ApplicationContainerCluster; import com.yahoo.vespa.model.container.Container; import com.yahoo.vespa.model.container.ContainerCluster; import com.yahoo.vespa.model.container.ContainerModel; import com.yahoo.vespa.model.container.ContainerModelEvaluation; import com.yahoo.vespa.model.container.ContainerThreadpool; import com.yahoo.vespa.model.container.IdentityProvider; import com.yahoo.vespa.model.container.PlatformBundles; import com.yahoo.vespa.model.container.SecretStore; import com.yahoo.vespa.model.container.component.AccessLogComponent; import com.yahoo.vespa.model.container.component.BindingPattern; import com.yahoo.vespa.model.container.component.Component; import com.yahoo.vespa.model.container.component.ConnectionLogComponent; import com.yahoo.vespa.model.container.component.FileStatusHandlerComponent; import com.yahoo.vespa.model.container.component.Handler; import com.yahoo.vespa.model.container.component.SimpleComponent; import com.yahoo.vespa.model.container.component.SystemBindingPattern; import com.yahoo.vespa.model.container.component.UserBindingPattern; import com.yahoo.vespa.model.container.component.chain.Chain; import com.yahoo.vespa.model.container.docproc.ContainerDocproc; import com.yahoo.vespa.model.container.docproc.DocprocChains; import com.yahoo.vespa.model.container.http.AccessControl; import com.yahoo.vespa.model.container.http.Client; import com.yahoo.vespa.model.container.http.ConnectorFactory; import com.yahoo.vespa.model.container.http.Filter; import com.yahoo.vespa.model.container.http.FilterBinding; import com.yahoo.vespa.model.container.http.FilterChains; import com.yahoo.vespa.model.container.http.Http; import com.yahoo.vespa.model.container.http.JettyHttpServer; import com.yahoo.vespa.model.container.http.ssl.HostedSslConnectorFactory; import com.yahoo.vespa.model.container.http.xml.HttpBuilder; import com.yahoo.vespa.model.container.processing.ProcessingChains; import com.yahoo.vespa.model.container.search.ContainerSearch; import com.yahoo.vespa.model.container.search.PageTemplates; import com.yahoo.vespa.model.container.search.searchchain.SearchChains; import com.yahoo.vespa.model.container.xml.document.DocumentFactoryBuilder; import com.yahoo.vespa.model.content.StorageGroup; import org.w3c.dom.Element; import org.w3c.dom.Node; import java.io.IOException; import java.io.Reader; import java.net.URI; import java.security.cert.X509Certificate; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.LinkedHashMap; import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.OptionalInt; import java.util.Set; import java.util.function.Consumer; import java.util.logging.Level; import java.util.regex.Pattern; import java.util.stream.Collectors; import static com.yahoo.vespa.model.container.ContainerCluster.VIP_HANDLER_BINDING; import static java.util.logging.Level.WARNING; /** * @author Tony Vaagenes * @author gjoranv */ public class ContainerModelBuilder extends ConfigModelBuilder { // Default path to vip status file for container in Hosted Vespa. static final String HOSTED_VESPA_STATUS_FILE = Defaults.getDefaults().underVespaHome("var/vespa/load-balancer/status.html"); // Data plane port for hosted Vespa static final int HOSTED_VESPA_DATAPLANE_PORT = 4443; //Path to vip status file for container in Hosted Vespa. Only used if set, else use HOSTED_VESPA_STATUS_FILE private static final String HOSTED_VESPA_STATUS_FILE_SETTING = "VESPA_LB_STATUS_FILE"; private static final String CONTAINER_TAG = "container"; private static final String ENVIRONMENT_VARIABLES_ELEMENT = "environment-variables"; // The node count to enforce in a cluster running ZooKeeper private static final int MIN_ZOOKEEPER_NODE_COUNT = 1; private static final int MAX_ZOOKEEPER_NODE_COUNT = 7; public enum Networking { disable, enable } private ApplicationPackage app; private final boolean standaloneBuilder; private final Networking networking; private final boolean rpcServerEnabled; private final boolean httpServerEnabled; protected DeployLogger log; public static final List configModelIds = List.of(ConfigModelId.fromName(CONTAINER_TAG)); private static final String xmlRendererId = RendererRegistry.xmlRendererId.getName(); private static final String jsonRendererId = RendererRegistry.jsonRendererId.getName(); public ContainerModelBuilder(boolean standaloneBuilder, Networking networking) { super(ContainerModel.class); this.standaloneBuilder = standaloneBuilder; this.networking = networking; // Always disable rpc server for standalone container this.rpcServerEnabled = !standaloneBuilder; this.httpServerEnabled = networking == Networking.enable; } @Override public List handlesElements() { return configModelIds; } @Override public void doBuild(ContainerModel model, Element spec, ConfigModelContext modelContext) { log = modelContext.getDeployLogger(); app = modelContext.getApplicationPackage(); checkVersion(spec); ApplicationContainerCluster cluster = createContainerCluster(spec, modelContext); addClusterContent(cluster, spec, modelContext); cluster.setMessageBusEnabled(rpcServerEnabled); cluster.setRpcServerEnabled(rpcServerEnabled); cluster.setHttpServerEnabled(httpServerEnabled); model.setCluster(cluster); } private ApplicationContainerCluster createContainerCluster(Element spec, ConfigModelContext modelContext) { return new VespaDomBuilder.DomConfigProducerBuilder() { @Override protected ApplicationContainerCluster doBuild(DeployState deployState, AbstractConfigProducer ancestor, Element producerSpec) { return new ApplicationContainerCluster(ancestor, modelContext.getProducerId(), modelContext.getProducerId(), deployState); } }.build(modelContext.getDeployState(), modelContext.getParentProducer(), spec); } private void addClusterContent(ApplicationContainerCluster cluster, Element spec, ConfigModelContext context) { DeployState deployState = context.getDeployState(); DocumentFactoryBuilder.buildDocumentFactories(cluster, spec); addConfiguredComponents(deployState, cluster, spec); addSecretStore(cluster, spec, deployState); addModelEvaluation(spec, cluster, context); addModelEvaluationBundles(cluster); addProcessing(deployState, spec, cluster, context); addSearch(deployState, spec, cluster, context); addDocproc(deployState, spec, cluster); addDocumentApi(deployState, spec, cluster, context); // NOTE: Must be done after addSearch cluster.addDefaultHandlersExceptStatus(); addStatusHandlers(cluster, context.getDeployState().isHosted()); addUserHandlers(deployState, cluster, spec, context); addClients(deployState, spec, cluster); addHttp(deployState, spec, cluster, context); addAccessLogs(deployState, cluster, spec); addNodes(cluster, spec, context); addServerProviders(deployState, spec, cluster); if (!standaloneBuilder) cluster.addAllPlatformBundles(); // Must be added after nodes: addDeploymentSpecConfig(cluster, context, deployState.getDeployLogger()); addZooKeeper(cluster, spec); addParameterStoreValidationHandler(cluster, deployState); } private void addParameterStoreValidationHandler(ApplicationContainerCluster cluster, DeployState deployState) { // Always add platform bundle. Cannot be controlled by a feature flag as platform bundle cannot change. if(deployState.isHosted()) { cluster.addPlatformBundle(PlatformBundles.absoluteBundlePath("jdisc-cloud-aws")); } if (deployState.zone().system().isPublic()) { BindingPattern bindingPattern = SystemBindingPattern.fromHttpPath("/validate-secret-store"); Handler handler = new Handler( new ComponentModel("com.yahoo.jdisc.cloud.aws.AwsParameterStoreValidationHandler", null, "jdisc-cloud-aws", null)); handler.addServerBindings(bindingPattern); cluster.addComponent(handler); } } private void addZooKeeper(ApplicationContainerCluster cluster, Element spec) { Element zooKeeper = getZooKeeper(spec); if (zooKeeper == null) return; Element nodesElement = XML.getChild(spec, "nodes"); boolean isCombined = nodesElement != null && nodesElement.hasAttribute("of"); if (isCombined) { throw new IllegalArgumentException("A combined cluster cannot run ZooKeeper"); } long nonRetiredNodes = cluster.getContainers().stream().filter(c -> !c.isRetired()).count(); if (nonRetiredNodes < MIN_ZOOKEEPER_NODE_COUNT || nonRetiredNodes > MAX_ZOOKEEPER_NODE_COUNT || nonRetiredNodes % 2 == 0) { throw new IllegalArgumentException("Cluster with ZooKeeper needs an odd number of nodes, between " + MIN_ZOOKEEPER_NODE_COUNT + " and " + MAX_ZOOKEEPER_NODE_COUNT + ", have " + nonRetiredNodes + " non-retired"); } cluster.addSimpleComponent("com.yahoo.vespa.curator.Curator", null, "zkfacade"); cluster.addSimpleComponent("com.yahoo.vespa.curator.CuratorWrapper", null, "zkfacade"); String sessionTimeoutSeconds = zooKeeper.getAttribute("session-timeout-seconds"); if ( ! sessionTimeoutSeconds.isBlank()) { try { int timeoutSeconds = Integer.parseInt(sessionTimeoutSeconds); if (timeoutSeconds <= 0) throw new IllegalArgumentException("must be a positive value"); cluster.setZookeeperSessionTimeoutSeconds(timeoutSeconds); } catch (RuntimeException e) { throw new IllegalArgumentException("invalid zookeeper session-timeout-seconds '" + sessionTimeoutSeconds + "'", e); } } // These need to be setup so that they will use the container's config id, since each container // have different config (id of zookeeper server) cluster.getContainers().forEach(ContainerModelBuilder::addReconfigurableZooKeeperServerComponents); } public static void addReconfigurableZooKeeperServerComponents(Container container) { container.addComponent(zookeeperComponent("com.yahoo.vespa.zookeeper.ReconfigurableVespaZooKeeperServer", container)); container.addComponent(zookeeperComponent("com.yahoo.vespa.zookeeper.Reconfigurer", container)); container.addComponent(zookeeperComponent("com.yahoo.vespa.zookeeper.VespaZooKeeperAdminImpl", container)); } private static SimpleComponent zookeeperComponent(String idSpec, Container container) { String configId = container.getConfigId(); return new SimpleComponent(new ComponentModel(idSpec, null, "zookeeper-server", configId)); } private void addSecretStore(ApplicationContainerCluster cluster, Element spec, DeployState deployState) { Element secretStoreElement = XML.getChild(spec, "secret-store"); if (secretStoreElement != null) { String type = secretStoreElement.getAttribute("type"); if ("cloud".equals(type)) { addCloudSecretStore(cluster, secretStoreElement, deployState); } else { SecretStore secretStore = new SecretStore(); for (Element group : XML.getChildren(secretStoreElement, "group")) { secretStore.addGroup(group.getAttribute("name"), group.getAttribute("environment")); } cluster.setSecretStore(secretStore); } } } private void addCloudSecretStore(ApplicationContainerCluster cluster, Element secretStoreElement, DeployState deployState) { if ( ! deployState.isHosted()) return; if ( ! cluster.getZone().system().isPublic()) throw new IllegalArgumentException("Cloud secret store is not supported in non-public system, see the documentation"); CloudSecretStore cloudSecretStore = new CloudSecretStore(); Map secretStoresByName = deployState.getProperties().tenantSecretStores() .stream() .collect(Collectors.toMap( TenantSecretStore::getName, store -> store )); Element store = XML.getChild(secretStoreElement, "store"); for (Element group : XML.getChildren(store, "aws-parameter-store")) { String account = group.getAttribute("account"); String region = group.getAttribute("aws-region"); TenantSecretStore secretStore = secretStoresByName.get(account); if (secretStore == null) throw new IllegalArgumentException("No configured secret store named " + account); if (secretStore.getExternalId().isEmpty()) throw new IllegalArgumentException("No external ID has been set"); cloudSecretStore.addConfig(account, region, secretStore.getAwsId(), secretStore.getRole(), secretStore.getExternalId().get()); } cluster.addComponent(cloudSecretStore); } private void addDeploymentSpecConfig(ApplicationContainerCluster cluster, ConfigModelContext context, DeployLogger deployLogger) { if ( ! context.getDeployState().isHosted()) return; Optional deploymentSpec = app.getDeployment().map(DeploymentSpec::fromXml); if (deploymentSpec.isEmpty()) return; for (var deprecatedElement : deploymentSpec.get().deprecatedElements()) { deployLogger.logApplicationPackage(WARNING, deprecatedElement.humanReadableString()); } addIdentityProvider(cluster, context.getDeployState().getProperties().configServerSpecs(), context.getDeployState().getProperties().loadBalancerName(), context.getDeployState().getProperties().ztsUrl(), context.getDeployState().getProperties().athenzDnsSuffix(), context.getDeployState().zone(), deploymentSpec.get()); addRotationProperties(cluster, context.getDeployState().zone(), context.getDeployState().getEndpoints(), deploymentSpec.get()); } private void addRotationProperties(ApplicationContainerCluster cluster, Zone zone, Set endpoints, DeploymentSpec spec) { cluster.getContainers().forEach(container -> { setRotations(container, endpoints, cluster.getName()); container.setProp("activeRotation", Boolean.toString(zoneHasActiveRotation(zone, spec))); }); } private boolean zoneHasActiveRotation(Zone zone, DeploymentSpec spec) { Optional instance = spec.instance(app.getApplicationId().instance()); if (instance.isEmpty()) return false; return instance.get().zones().stream() .anyMatch(declaredZone -> declaredZone.concerns(zone.environment(), Optional.of(zone.region())) && declaredZone.active()); } private void setRotations(Container container, Set endpoints, String containerClusterName) { var rotationsProperty = endpoints.stream() .filter(endpoint -> endpoint.clusterId().equals(containerClusterName)) // Only consider global endpoints. .filter(endpoint -> endpoint.scope() == ApplicationClusterEndpoint.Scope.global) .flatMap(endpoint -> endpoint.names().stream()) .collect(Collectors.toCollection(LinkedHashSet::new)); // Build the comma delimited list of endpoints this container should be known as. // Confusingly called 'rotations' for legacy reasons. container.setProp("rotations", String.join(",", rotationsProperty)); } private void addConfiguredComponents(DeployState deployState, ApplicationContainerCluster cluster, Element parent) { for (Element components : XML.getChildren(parent, "components")) { addIncludes(components); addConfiguredComponents(deployState, cluster, components, "component"); } addConfiguredComponents(deployState, cluster, parent, "component"); } protected void addStatusHandlers(ApplicationContainerCluster cluster, boolean isHostedVespa) { if (isHostedVespa) { String name = "status.html"; Optional statusFile = Optional.ofNullable(System.getenv(HOSTED_VESPA_STATUS_FILE_SETTING)); cluster.addComponent( new FileStatusHandlerComponent( name + "-status-handler", statusFile.orElse(HOSTED_VESPA_STATUS_FILE), SystemBindingPattern.fromHttpPath("/" + name))); } else { cluster.addVipHandler(); } } private void addServerProviders(DeployState deployState, Element spec, ApplicationContainerCluster cluster) { addConfiguredComponents(deployState, cluster, spec, "server"); } protected void addAccessLogs(DeployState deployState, ApplicationContainerCluster cluster, Element spec) { List accessLogElements = getAccessLogElements(spec); if (cluster.isHostedVespa() && !accessLogElements.isEmpty()) { accessLogElements.clear(); log.logApplicationPackage( Level.WARNING, "Applications are not allowed to override the 'accesslog' element"); } else { for (Element accessLog : accessLogElements) { AccessLogBuilder.buildIfNotDisabled(deployState, cluster, accessLog).ifPresent(cluster::addComponent); } } if (accessLogElements.isEmpty() && deployState.getAccessLoggingEnabledByDefault()) cluster.addDefaultSearchAccessLog(); // Add connection log if access log is configured if (cluster.getAllComponents().stream().anyMatch(component -> component instanceof AccessLogComponent)) { // TODO: Vespa > 8: Clean up if (cluster.isHostedVespa() || deployState.getVespaVersion().getMajor() == 8) { cluster.addComponent(new ConnectionLogComponent(cluster, FileConnectionLog.class, "access")); } else { cluster.addComponent(new ConnectionLogComponent(cluster, FileConnectionLog.class, "qrs")); } } } private List getAccessLogElements(Element spec) { return XML.getChildren(spec, "accesslog"); } protected void addHttp(DeployState deployState, Element spec, ApplicationContainerCluster cluster, ConfigModelContext context) { Element httpElement = XML.getChild(spec, "http"); if (httpElement != null) { cluster.setHttp(buildHttp(deployState, cluster, httpElement)); } if (isHostedTenantApplication(context)) { addHostedImplicitHttpIfNotPresent(deployState, cluster); addHostedImplicitAccessControlIfNotPresent(deployState, cluster); addDefaultConnectorHostedFilterBinding(cluster); addAdditionalHostedConnector(deployState, cluster, context); addCloudDataPlaneFilter(deployState, cluster); } } private static void addCloudDataPlaneFilter(DeployState deployState, ApplicationContainerCluster cluster) { if (!deployState.isHosted() || !deployState.zone().system().isPublic() || !deployState.featureFlags().enableDataPlaneFilter()) return; // Setup secure filter chain var secureChain = new Chain(FilterChains.emptyChainSpec(ComponentId.fromString("cloud-data-plane-secure"))); secureChain.addInnerComponent(new CloudDataPlaneFilter(cluster, cluster.clientsLegacyMode())); cluster.getHttp().getFilterChains().add(secureChain); // Set cloud data plane filter as default request filter chain for data plane connector cluster.getHttp().getHttpServer().orElseThrow().getConnectorFactories().stream() .filter(c -> c.getListenPort() == HOSTED_VESPA_DATAPLANE_PORT).findAny().orElseThrow() .setDefaultRequestFilterChain(secureChain.getComponentId()); // Setup insecure filter chain var insecureChain = new Chain(FilterChains.emptyChainSpec(ComponentId.fromString("cloud-data-plane-insecure"))); insecureChain.addInnerComponent(new Filter( new ChainedComponentModel( new BundleInstantiationSpecification( new ComponentSpecification("com.yahoo.jdisc.http.filter.security.misc.NoopFilter"), null, new ComponentSpecification("jdisc-security-filters")), Dependencies.emptyDependencies()))); cluster.getHttp().getFilterChains().add(insecureChain); var insecureChainComponentSpec = new ComponentSpecification(insecureChain.getComponentId().toString()); FilterBinding insecureBinding = FilterBinding.create(FilterBinding.Type.REQUEST, insecureChainComponentSpec, VIP_HANDLER_BINDING); cluster.getHttp().getBindings().add(insecureBinding); // Set insecure filter as default request filter chain for default connector cluster.getHttp().getHttpServer().orElseThrow().getConnectorFactories().stream() .filter(c -> c.getListenPort() == Defaults.getDefaults().vespaWebServicePort()).findAny().orElseThrow() .setDefaultRequestFilterChain(insecureChain.getComponentId()); } protected void addClients(DeployState deployState, Element spec, ApplicationContainerCluster cluster) { if (!deployState.isHosted() || !deployState.zone().system().isPublic() || !deployState.featureFlags().enableDataPlaneFilter()) return; List clients; Element clientsElement = XML.getChild(spec, "clients"); boolean legacyMode = false; if (clientsElement == null) { Client defaultClient = new Client("default", List.of(), getCertificates(app.getFile(Path.fromString("security/clients.pem")))); clients = List.of(defaultClient); legacyMode = true; } else { clients = XML.getChildren(clientsElement, "client").stream() .map(this::getCLient) .toList(); } cluster.setClients(legacyMode, clients); } private Client getCLient(Element clientElement) { String id = XML.attribute("id", clientElement).orElseThrow(); List permissions = XML.attribute("permissions", clientElement) .map(p -> p.split(",")).stream() .flatMap(Arrays::stream) .toList(); List x509Certificates = XML.getChildren(clientElement, "certificate").stream() .map(certElem -> Path.fromString(certElem.getAttribute("file"))) .map(path -> app.getFile(path)) .map(this::getCertificates) .flatMap(Collection::stream) .toList(); return new Client(id, permissions, x509Certificates); } private List getCertificates(ApplicationFile file) { try { Reader reader = file.createReader(); String certPem = IOUtils.readAll(reader); reader.close(); List x509Certificates = X509CertificateUtils.certificateListFromPem(certPem); if (x509Certificates.isEmpty()) { throw new IllegalArgumentException("File %s does not contain any certificates.".formatted(file.getPath().getRelative())); } return x509Certificates; } catch (IOException e) { throw new RuntimeException(e); } } private void addDefaultConnectorHostedFilterBinding(ApplicationContainerCluster cluster) { cluster.getHttp().getAccessControl() .ifPresent(accessControl -> accessControl.configureDefaultHostedConnector(cluster.getHttp())); ; } private void addAdditionalHostedConnector(DeployState deployState, ApplicationContainerCluster cluster, ConfigModelContext context) { JettyHttpServer server = cluster.getHttp().getHttpServer().get(); String serverName = server.getComponentId().getName(); // If the deployment contains certificate/private key reference, setup TLS port HostedSslConnectorFactory connectorFactory; Collection tlsCiphersOverride = deployState.getProperties().tlsCiphersOverride(); boolean proxyProtocolMixedMode = deployState.getProperties().featureFlags().enableProxyProtocolMixedMode(); if (deployState.endpointCertificateSecrets().isPresent()) { boolean authorizeClient = deployState.zone().system().isPublic(); List clientCertificates = deployState.featureFlags().enableDataPlaneFilter() ? getClientCertificates(cluster) : deployState.tlsClientAuthority().map(X509CertificateUtils::certificateListFromPem).orElse(List.of()); if (authorizeClient && clientCertificates.isEmpty()) { throw new IllegalArgumentException("Client certificate authority security/clients.pem is missing - " + "see: https://cloud.vespa.ai/en/security-model#data-plane"); } EndpointCertificateSecrets endpointCertificateSecrets = deployState.endpointCertificateSecrets().get(); boolean enforceHandshakeClientAuth = cluster.getHttp().getAccessControl() .map(accessControl -> accessControl.clientAuthentication) .map(clientAuth -> clientAuth == AccessControl.ClientAuthentication.need) .orElse(false); connectorFactory = authorizeClient ? HostedSslConnectorFactory.withProvidedCertificateAndTruststore( serverName, endpointCertificateSecrets, getTlsClientAuthorities(clientCertificates, deployState), tlsCiphersOverride, proxyProtocolMixedMode, HOSTED_VESPA_DATAPLANE_PORT) : HostedSslConnectorFactory.withProvidedCertificate( serverName, endpointCertificateSecrets, enforceHandshakeClientAuth, tlsCiphersOverride, proxyProtocolMixedMode, HOSTED_VESPA_DATAPLANE_PORT); } else { connectorFactory = HostedSslConnectorFactory.withDefaultCertificateAndTruststore(serverName, tlsCiphersOverride, proxyProtocolMixedMode, HOSTED_VESPA_DATAPLANE_PORT); } cluster.getHttp().getAccessControl().ifPresent(accessControl -> accessControl.configureHostedConnector(connectorFactory)); server.addConnector(connectorFactory); } // Returns the client certificates defined in private List getClientCertificates(ApplicationContainerCluster cluster) { return cluster.getClients() .stream() .map(Client::certificates) .flatMap(Collection::stream) .toList(); } /* Return trusted certificates as a PEM encoded string containing the concatenation of trusted certs from the application package and all operator certificates. */ String getTlsClientAuthorities(List applicationCertificates, DeployState deployState) { ArrayList x509Certificates = new ArrayList<>(applicationCertificates); x509Certificates.addAll(deployState.getProperties().operatorCertificates()); return X509CertificateUtils.toPem(x509Certificates); } private static boolean isHostedTenantApplication(ConfigModelContext context) { var deployState = context.getDeployState(); boolean isTesterApplication = deployState.getProperties().applicationId().instance().isTester(); return deployState.isHosted() && context.getApplicationType() == ApplicationType.DEFAULT && !isTesterApplication; } private static void addHostedImplicitHttpIfNotPresent(DeployState deployState, ApplicationContainerCluster cluster) { if (cluster.getHttp() == null) { cluster.setHttp(new Http(new FilterChains(cluster))); } JettyHttpServer httpServer = cluster.getHttp().getHttpServer().orElse(null); if (httpServer == null) { httpServer = new JettyHttpServer("DefaultHttpServer", cluster, deployState); cluster.getHttp().setHttpServer(httpServer); } int defaultPort = Defaults.getDefaults().vespaWebServicePort(); boolean defaultConnectorPresent = httpServer.getConnectorFactories().stream().anyMatch(connector -> connector.getListenPort() == defaultPort); if (!defaultConnectorPresent) { httpServer.addConnector(new ConnectorFactory.Builder("SearchServer", defaultPort).build()); } } private void addHostedImplicitAccessControlIfNotPresent(DeployState deployState, ApplicationContainerCluster cluster) { Http http = cluster.getHttp(); if (http.getAccessControl().isPresent()) return; // access control added explicitly AthenzDomain tenantDomain = deployState.getProperties().athenzDomain().orElse(null); if (tenantDomain == null) return; // tenant domain not present, cannot add access control. this should eventually be a failure. new AccessControl.Builder(tenantDomain.value()) .setHandlers(cluster) .clientAuthentication(AccessControl.ClientAuthentication.need) .build() .configureHttpFilterChains(http); } private Http buildHttp(DeployState deployState, ApplicationContainerCluster cluster, Element httpElement) { Http http = new HttpBuilder().build(deployState, cluster, httpElement); if (networking == Networking.disable) http.removeAllServers(); return http; } private void addDocumentApi(DeployState deployState, Element spec, ApplicationContainerCluster cluster, ConfigModelContext context) { ContainerDocumentApi containerDocumentApi = buildDocumentApi(deployState, cluster, spec, context); if (containerDocumentApi == null) return; cluster.setDocumentApi(containerDocumentApi); } private void addDocproc(DeployState deployState, Element spec, ApplicationContainerCluster cluster) { ContainerDocproc containerDocproc = buildDocproc(deployState, cluster, spec); if (containerDocproc == null) return; cluster.setDocproc(containerDocproc); ContainerDocproc.Options docprocOptions = containerDocproc.options; cluster.setMbusParams(new ApplicationContainerCluster.MbusParams( docprocOptions.maxConcurrentFactor, docprocOptions.documentExpansionFactor, docprocOptions.containerCoreMemory)); } private void addSearch(DeployState deployState, Element spec, ApplicationContainerCluster cluster, ConfigModelContext context) { Element searchElement = XML.getChild(spec, "search"); if (searchElement == null) return; addIncludes(searchElement); cluster.setSearch(buildSearch(deployState, cluster, searchElement)); addSearchHandler(deployState, cluster, searchElement, context); validateAndAddConfiguredComponents(deployState, cluster, searchElement, "renderer", ContainerModelBuilder::validateRendererElement); } private void addModelEvaluation(Element spec, ApplicationContainerCluster cluster, ConfigModelContext context) { Element modelEvaluationElement = XML.getChild(spec, "model-evaluation"); if (modelEvaluationElement == null) return; RankProfileList profiles = context.vespaModel() != null ? context.vespaModel().rankProfileList() : RankProfileList.empty; Element onnxElement = XML.getChild(modelEvaluationElement, "onnx"); Element modelsElement = XML.getChild(onnxElement, "models"); for (Element modelElement : XML.getChildren(modelsElement, "model") ) { OnnxModel onnxModel = profiles.getOnnxModels().asMap().get(modelElement.getAttribute("name")); if (onnxModel == null) continue; // Skip if model is not found onnxModel.setStatelessExecutionMode(getStringValue(modelElement, "execution-mode", null)); onnxModel.setStatelessInterOpThreads(getIntValue(modelElement, "interop-threads", -1)); onnxModel.setStatelessIntraOpThreads(getIntValue(modelElement, "intraop-threads", -1)); } cluster.setModelEvaluation(new ContainerModelEvaluation(cluster, profiles)); } private String getStringValue(Element element, String name, String defaultValue) { Element child = XML.getChild(element, name); return (child != null) ? child.getTextContent() : defaultValue; } private int getIntValue(Element element, String name, int defaultValue) { Element child = XML.getChild(element, name); return (child != null) ? Integer.parseInt(child.getTextContent()) : defaultValue; } protected void addModelEvaluationBundles(ApplicationContainerCluster cluster) { /* These bundles are added to all application container clusters, even if they haven't * declared 'model-evaluation' in services.xml, because there are many public API packages * in the model-evaluation bundle that could be used by customer code. */ cluster.addPlatformBundle(ContainerModelEvaluation.MODEL_EVALUATION_BUNDLE_FILE); cluster.addPlatformBundle(ContainerModelEvaluation.MODEL_INTEGRATION_BUNDLE_FILE); } private void addProcessing(DeployState deployState, Element spec, ApplicationContainerCluster cluster, ConfigModelContext context) { Element processingElement = XML.getChild(spec, "processing"); if (processingElement == null) return; cluster.addSearchAndDocprocBundles(); addIncludes(processingElement); cluster.setProcessingChains(new DomProcessingBuilder(null).build(deployState, cluster, processingElement), serverBindings(deployState, context, processingElement, ProcessingChains.defaultBindings).toArray(BindingPattern[]::new)); validateAndAddConfiguredComponents(deployState, cluster, processingElement, "renderer", ContainerModelBuilder::validateRendererElement); } private ContainerSearch buildSearch(DeployState deployState, ApplicationContainerCluster containerCluster, Element producerSpec) { SearchChains searchChains = new DomSearchChainsBuilder() .build(deployState, containerCluster, producerSpec); ContainerSearch containerSearch = new ContainerSearch(containerCluster, searchChains, new ContainerSearch.Options()); applyApplicationPackageDirectoryConfigs(deployState.getApplicationPackage(), containerSearch); containerSearch.setQueryProfiles(deployState.getQueryProfiles()); containerSearch.setSemanticRules(deployState.getSemanticRules()); return containerSearch; } private void applyApplicationPackageDirectoryConfigs(ApplicationPackage applicationPackage,ContainerSearch containerSearch) { PageTemplates.validate(applicationPackage); containerSearch.setPageTemplates(PageTemplates.create(applicationPackage)); } private void addUserHandlers(DeployState deployState, ApplicationContainerCluster cluster, Element spec, ConfigModelContext context) { OptionalInt portBindingOverride = isHostedTenantApplication(context) ? OptionalInt.of(HOSTED_VESPA_DATAPLANE_PORT) : OptionalInt.empty(); for (Element component: XML.getChildren(spec, "handler")) { cluster.addComponent( new DomHandlerBuilder(cluster, portBindingOverride).build(deployState, cluster, component)); } } private void checkVersion(Element spec) { String version = spec.getAttribute("version"); if ( ! Version.fromString(version).equals(new Version(1))) throw new IllegalArgumentException("Expected container version to be 1.0, but got " + version); } private void addNodes(ApplicationContainerCluster cluster, Element spec, ConfigModelContext context) { if (standaloneBuilder) addStandaloneNode(cluster, context.getDeployState()); else addNodesFromXml(cluster, spec, context); } private void addStandaloneNode(ApplicationContainerCluster cluster, DeployState deployState) { ApplicationContainer container = new ApplicationContainer(cluster, "standalone", cluster.getContainers().size(), deployState); cluster.addContainers(Collections.singleton(container)); } private static String buildJvmGCOptions(ConfigModelContext context, String jvmGCOptions) { return new JvmGcOptions(context.getDeployState(), jvmGCOptions).build(); } private static String getJvmOptions(Element nodesElement, DeployState deployState, boolean legacyOptions) { return new JvmOptions(nodesElement, deployState, legacyOptions).build(); } private static String extractAttribute(Element element, String attrName) { return element.hasAttribute(attrName) ? element.getAttribute(attrName) : null; } private void extractJvmOptions(List nodes, ApplicationContainerCluster cluster, Element nodesElement, ConfigModelContext context) { Element jvmElement = XML.getChild(nodesElement, "jvm"); if (jvmElement == null) { extractJvmFromLegacyNodesTag(nodes, cluster, nodesElement, context); } else { extractJvmTag(nodes, cluster, nodesElement, jvmElement, context); } } private void extractJvmFromLegacyNodesTag(List nodes, ApplicationContainerCluster cluster, Element nodesElement, ConfigModelContext context) { applyNodesTagJvmArgs(nodes, getJvmOptions(nodesElement, context.getDeployState(), true)); if (cluster.getJvmGCOptions().isEmpty()) { String jvmGCOptions = extractAttribute(nodesElement, VespaDomBuilder.JVM_GC_OPTIONS); cluster.setJvmGCOptions(buildJvmGCOptions(context, jvmGCOptions)); } applyMemoryPercentage(cluster, nodesElement.getAttribute(VespaDomBuilder.Allocated_MEMORY_ATTRIB_NAME)); } private void extractJvmTag(List nodes, ApplicationContainerCluster cluster, Element nodesElement, Element jvmElement, ConfigModelContext context) { applyNodesTagJvmArgs(nodes, getJvmOptions(nodesElement, context.getDeployState(), false)); applyMemoryPercentage(cluster, jvmElement.getAttribute(VespaDomBuilder.Allocated_MEMORY_ATTRIB_NAME)); String jvmGCOptions = extractAttribute(jvmElement, VespaDomBuilder.GC_OPTIONS); cluster.setJvmGCOptions(buildJvmGCOptions(context, jvmGCOptions)); } /** * Add nodes to cluster according to the given containerElement. * * Note: DO NOT change allocation behaviour to allow version X and Y of the config-model to allocate a different set * of nodes. Such changes must be guarded by a common condition (e.g. feature flag) so the behaviour can be changed * simultaneously for all active config models. */ private void addNodesFromXml(ApplicationContainerCluster cluster, Element containerElement, ConfigModelContext context) { Element nodesElement = XML.getChild(containerElement, "nodes"); if (nodesElement == null) { cluster.addContainers(allocateWithoutNodesTag(cluster, context)); } else { List nodes = createNodes(cluster, containerElement, nodesElement, context); extractJvmOptions(nodes, cluster, nodesElement, context); applyDefaultPreload(nodes, nodesElement); var envVars = getEnvironmentVariables(XML.getChild(nodesElement, ENVIRONMENT_VARIABLES_ELEMENT)).entrySet(); for (var container : nodes) { for (var entry : envVars) { container.addEnvironmentVariable(entry.getKey(), entry.getValue()); } } if (useCpuSocketAffinity(nodesElement)) AbstractService.distributeCpuSocketAffinity(nodes); cluster.addContainers(nodes); } } private LoadBalancerSettings loadBalancerSettings(Element loadBalancerElement) { List allowedUrnElements = XML.getChildren(XML.getChild(loadBalancerElement, "private-access"), "allow-urn") .stream().map(XML::getValue).toList(); return new LoadBalancerSettings(allowedUrnElements); } private static Map getEnvironmentVariables(Element environmentVariables) { var map = new LinkedHashMap(); if (environmentVariables != null) { for (Element var: XML.getChildren(environmentVariables)) { var name = new com.yahoo.text.Identifier(var.getNodeName()); map.put(name.toString(), var.getTextContent()); } } return map; } private List createNodes(ApplicationContainerCluster cluster, Element containerElement, Element nodesElement, ConfigModelContext context) { if (nodesElement.hasAttribute("type")) // internal use for hosted system infrastructure nodes return createNodesFromNodeType(cluster, nodesElement, context); else if (nodesElement.hasAttribute("of")) {// hosted node spec referencing a content cluster // TODO: Remove support for combined clusters in Vespa 9 List containers = createNodesFromContentServiceReference(cluster, nodesElement, context); log.logApplicationPackage(WARNING, "Declaring combined cluster with is deprecated without " + "replacement, and the feature will be removed in Vespa 9. Use separate container and " + "content clusters instead"); return containers; } else if (nodesElement.hasAttribute("count")) // regular, hosted node spec return createNodesFromNodeCount(cluster, containerElement, nodesElement, context); else if (cluster.isHostedVespa() && cluster.getZone().environment().isManuallyDeployed()) // default to 1 in manual zones return createNodesFromNodeCount(cluster, containerElement, nodesElement, context); else // the non-hosted option return createNodesFromNodeList(context.getDeployState(), cluster, nodesElement); } private static void applyMemoryPercentage(ApplicationContainerCluster cluster, String memoryPercentage) { if (memoryPercentage == null || memoryPercentage.isEmpty()) return; memoryPercentage = memoryPercentage.trim(); if ( ! memoryPercentage.endsWith("%")) throw new IllegalArgumentException("The memory percentage given for nodes in " + cluster + " must be an integer percentage ending by the '%' sign"); memoryPercentage = memoryPercentage.substring(0, memoryPercentage.length()-1).trim(); try { cluster.setMemoryPercentage(Integer.parseInt(memoryPercentage)); } catch (NumberFormatException e) { throw new IllegalArgumentException("The memory percentage given for nodes in " + cluster + " must be an integer percentage ending by the '%' sign"); } } /** Allocate a container cluster without a nodes tag */ private List allocateWithoutNodesTag(ApplicationContainerCluster cluster, ConfigModelContext context) { DeployState deployState = context.getDeployState(); HostSystem hostSystem = cluster.hostSystem(); if (deployState.isHosted()) { // request just enough nodes to satisfy environment capacity requirement ClusterSpec clusterSpec = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from(cluster.getName())) .vespaVersion(deployState.getWantedNodeVespaVersion()) .dockerImageRepository(deployState.getWantedDockerImageRepo()) .build(); int nodeCount = deployState.zone().environment().isProduction() ? 2 : 1; deployState.getDeployLogger().logApplicationPackage(Level.INFO, "Using " + nodeCount + " nodes in " + cluster); ClusterResources resources = new ClusterResources(nodeCount, 1, NodeResources.unspecified()); Capacity capacity = Capacity.from(resources, resources, false, !deployState.getProperties().isBootstrap(), context.getDeployState().getProperties().cloudAccount()); var hosts = hostSystem.allocateHosts(clusterSpec, capacity, log); return createNodesFromHosts(hosts, cluster, context.getDeployState()); } else { return singleHostContainerCluster(cluster, hostSystem.getHost(Container.SINGLENODE_CONTAINER_SERVICESPEC), context); } } private List singleHostContainerCluster(ApplicationContainerCluster cluster, HostResource host, ConfigModelContext context) { ApplicationContainer node = new ApplicationContainer(cluster, "container.0", 0, context.getDeployState()); node.setHostResource(host); node.initService(context.getDeployState()); return List.of(node); } private List createNodesFromNodeCount(ApplicationContainerCluster cluster, Element containerElement, Element nodesElement, ConfigModelContext context) { NodesSpecification nodesSpecification = NodesSpecification.from(new ModelElement(nodesElement), context); LoadBalancerSettings loadBalancerSettings = loadBalancerSettings(XML.getChild(containerElement, "load-balancer")); Map hosts = nodesSpecification.provision(cluster.getRoot().hostSystem(), ClusterSpec.Type.container, ClusterSpec.Id.from(cluster.getName()), loadBalancerSettings, log, getZooKeeper(containerElement) != null); return createNodesFromHosts(hosts, cluster, context.getDeployState()); } private List createNodesFromNodeType(ApplicationContainerCluster cluster, Element nodesElement, ConfigModelContext context) { NodeType type = NodeType.valueOf(nodesElement.getAttribute("type")); ClusterSpec clusterSpec = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from(cluster.getName())) .vespaVersion(context.getDeployState().getWantedNodeVespaVersion()) .dockerImageRepository(context.getDeployState().getWantedDockerImageRepo()) .build(); Map hosts = cluster.getRoot().hostSystem().allocateHosts(clusterSpec, Capacity.fromRequiredNodeType(type), log); return createNodesFromHosts(hosts, cluster, context.getDeployState()); } private List createNodesFromContentServiceReference(ApplicationContainerCluster cluster, Element nodesElement, ConfigModelContext context) { NodesSpecification nodeSpecification; try { nodeSpecification = NodesSpecification.from(new ModelElement(nodesElement), context); } catch (IllegalArgumentException e) { throw new IllegalArgumentException(cluster + " contains an invalid reference", e); } String referenceId = nodesElement.getAttribute("of"); cluster.setHostClusterId(referenceId); Map hosts = StorageGroup.provisionHosts(nodeSpecification, referenceId, cluster.getRoot().hostSystem(), context.getDeployLogger()); return createNodesFromHosts(hosts, cluster, context.getDeployState()); } private List createNodesFromHosts(Map hosts, ApplicationContainerCluster cluster, DeployState deployState) { List nodes = new ArrayList<>(); for (Map.Entry entry : hosts.entrySet()) { String id = "container." + entry.getValue().index(); ApplicationContainer container = new ApplicationContainer(cluster, id, entry.getValue().retired(), entry.getValue().index(), deployState); container.setHostResource(entry.getKey()); container.initService(deployState); nodes.add(container); } return nodes; } private List createNodesFromNodeList(DeployState deployState, ApplicationContainerCluster cluster, Element nodesElement) { List nodes = new ArrayList<>(); int nodeIndex = 0; for (Element nodeElem: XML.getChildren(nodesElement, "node")) { nodes.add(new ContainerServiceBuilder("container." + nodeIndex, nodeIndex).build(deployState, cluster, nodeElem)); nodeIndex++; } return nodes; } private static boolean useCpuSocketAffinity(Element nodesElement) { if (nodesElement.hasAttribute(VespaDomBuilder.CPU_SOCKET_AFFINITY_ATTRIB_NAME)) return Boolean.parseBoolean(nodesElement.getAttribute(VespaDomBuilder.CPU_SOCKET_AFFINITY_ATTRIB_NAME)); else return false; } private static void applyNodesTagJvmArgs(List containers, String jvmArgs) { for (Container container: containers) { if (container.getAssignedJvmOptions().isEmpty()) container.prependJvmOptions(jvmArgs); } } private static void applyDefaultPreload(List containers, Element nodesElement) { if (! nodesElement.hasAttribute(VespaDomBuilder.PRELOAD_ATTRIB_NAME)) return; for (Container container: containers) container.setPreLoad(nodesElement.getAttribute(VespaDomBuilder.PRELOAD_ATTRIB_NAME)); } private void addSearchHandler(DeployState deployState, ApplicationContainerCluster cluster, Element searchElement, ConfigModelContext context) { BindingPattern bindingPattern = SearchHandler.DEFAULT_BINDING; if (isHostedTenantApplication(context) && deployState.featureFlags().useRestrictedDataPlaneBindings()) { bindingPattern = SearchHandler.bindingPattern(Optional.of(Integer.toString(HOSTED_VESPA_DATAPLANE_PORT))); } SearchHandler searchHandler = new SearchHandler(cluster, serverBindings(deployState, context, searchElement, bindingPattern), ContainerThreadpool.UserOptions.fromXml(searchElement).orElse(null)); cluster.addComponent(searchHandler); // Add as child to SearchHandler to get the correct chains config. searchHandler.addComponent(Component.fromClassAndBundle(SearchHandler.EXECUTION_FACTORY_CLASS, PlatformBundles.SEARCH_AND_DOCPROC_BUNDLE)); } private List serverBindings(DeployState deployState, ConfigModelContext context, Element searchElement, BindingPattern... defaultBindings) { List bindings = XML.getChildren(searchElement, "binding"); if (bindings.isEmpty()) return List.of(defaultBindings); return toBindingList(deployState, context, bindings); } private List toBindingList(DeployState deployState, ConfigModelContext context, List bindingElements) { List result = new ArrayList<>(); OptionalInt portOverride = isHostedTenantApplication(context) && deployState.featureFlags().useRestrictedDataPlaneBindings() ? OptionalInt.of(HOSTED_VESPA_DATAPLANE_PORT) : OptionalInt.empty(); for (Element element: bindingElements) { String text = element.getTextContent().trim(); if (!text.isEmpty()) result.add(userBindingPattern(text, portOverride)); } return result; } private static UserBindingPattern userBindingPattern(String path, OptionalInt portOverride) { UserBindingPattern bindingPattern = UserBindingPattern.fromPattern(path); return portOverride.isPresent() ? bindingPattern.withPort(portOverride.getAsInt()) : bindingPattern; } private ContainerDocumentApi buildDocumentApi(DeployState deployState, ApplicationContainerCluster cluster, Element spec, ConfigModelContext context) { Element documentApiElement = XML.getChild(spec, "document-api"); if (documentApiElement == null) return null; ContainerDocumentApi.HandlerOptions documentApiOptions = DocumentApiOptionsBuilder.build(documentApiElement); Element ignoreUndefinedFields = XML.getChild(documentApiElement, "ignore-undefined-fields"); OptionalInt portBindingOverride = deployState.featureFlags().useRestrictedDataPlaneBindings() && isHostedTenantApplication(context) ? OptionalInt.of(HOSTED_VESPA_DATAPLANE_PORT) : OptionalInt.empty(); return new ContainerDocumentApi(cluster, documentApiOptions, "true".equals(XML.getValue(ignoreUndefinedFields)), portBindingOverride); } private ContainerDocproc buildDocproc(DeployState deployState, ApplicationContainerCluster cluster, Element spec) { Element docprocElement = XML.getChild(spec, "document-processing"); if (docprocElement == null) return null; addIncludes(docprocElement); DocprocChains chains = new DomDocprocChainsBuilder(null, false).build(deployState, cluster, docprocElement); ContainerDocproc.Options docprocOptions = DocprocOptionsBuilder.build(docprocElement, deployState.getDeployLogger()); return new ContainerDocproc(cluster, chains, docprocOptions, !standaloneBuilder); } private void addIncludes(Element parentElement) { List includes = XML.getChildren(parentElement, IncludeDirs.INCLUDE); if (includes.isEmpty()) { return; } if (app == null) { throw new IllegalArgumentException("Element given in XML config, but no application package given."); } for (Element include : includes) { addInclude(parentElement, include); } } private void addInclude(Element parentElement, Element include) { String dirName = include.getAttribute(IncludeDirs.DIR); app.validateIncludeDir(dirName); List includedFiles = Xml.allElemsFromPath(app, dirName); for (Element includedFile : includedFiles) { List includedSubElements = XML.getChildren(includedFile); for (Element includedSubElement : includedSubElements) { Node copiedNode = parentElement.getOwnerDocument().importNode(includedSubElement, true); parentElement.appendChild(copiedNode); } } } private static void addConfiguredComponents(DeployState deployState, ContainerCluster cluster, Element parent, String componentName) { for (Element component : XML.getChildren(parent, componentName)) { ModelIdResolver.resolveModelIds(component, deployState.isHosted()); cluster.addComponent(new DomComponentBuilder().build(deployState, cluster, component)); } } private static void validateAndAddConfiguredComponents(DeployState deployState, ContainerCluster cluster, Element spec, String componentName, Consumer elementValidator) { for (Element node : XML.getChildren(spec, componentName)) { elementValidator.accept(node); // throws exception here if something is wrong cluster.addComponent(new DomComponentBuilder().build(deployState, cluster, node)); } } private void addIdentityProvider(ApplicationContainerCluster cluster, List configServerSpecs, HostName loadBalancerName, URI ztsUrl, String athenzDnsSuffix, Zone zone, DeploymentSpec spec) { spec.athenzDomain() .ifPresent(domain -> { AthenzService service = spec.instance(app.getApplicationId().instance()) .flatMap(instanceSpec -> instanceSpec.athenzService(zone.environment(), zone.region())) .or(spec::athenzService) .orElseThrow(() -> new IllegalArgumentException("Missing Athenz service configuration in instance '" + app.getApplicationId().instance() + "'")); String zoneDnsSuffix = zone.environment().value() + "-" + zone.region().value() + "." + athenzDnsSuffix; IdentityProvider identityProvider = new IdentityProvider(domain, service, getLoadBalancerName(loadBalancerName, configServerSpecs), ztsUrl, zoneDnsSuffix, zone); cluster.addComponent(identityProvider); cluster.getContainers().forEach(container -> { container.setProp("identity.domain", domain.value()); container.setProp("identity.service", service.value()); }); }); } private HostName getLoadBalancerName(HostName loadbalancerName, List configServerSpecs) { // Set lbaddress, or use first hostname if not specified. // TODO: Remove this method and use the loadbalancerName directly return Optional.ofNullable(loadbalancerName) .orElseGet( () -> HostName.of(configServerSpecs.stream() .findFirst() .map(ConfigServerSpec::getHostName) .orElse("unknown") // Currently unable to test this, hence the unknown )); } private static Element getZooKeeper(Element spec) { return XML.getChild(spec, "zookeeper"); } /** Disallow renderers named "XmlRenderer" or "JsonRenderer" */ private static void validateRendererElement(Element element) { String idAttr = element.getAttribute("id"); if (idAttr.equals(xmlRendererId) || idAttr.equals(jsonRendererId)) { throw new IllegalArgumentException(String.format("Renderer id %s is reserved for internal use", idAttr)); } } public static boolean isContainerTag(Element element) { return CONTAINER_TAG.equals(element.getTagName()); } /** * Validates JVM options and logs a warning or fails deployment (depending on feature flag) * if anyone of them has invalid syntax or is an option that is unsupported for the running system. */ private static class JvmOptions { private static final Pattern validPattern = Pattern.compile("-[a-zA-z0-9=:./,+*-]+"); // debug port will not be available in hosted, don't allow private static final Pattern invalidInHostedatttern = Pattern.compile("-Xrunjdwp:transport=.*"); private final Element nodesElement; private final DeployLogger logger; private final boolean legacyOptions; private final boolean isHosted; public JvmOptions(Element nodesElement, DeployState deployState, boolean legacyOptions) { this.nodesElement = nodesElement; this.logger = deployState.getDeployLogger(); this.legacyOptions = legacyOptions; this.isHosted = deployState.isHosted(); } String build() { if (legacyOptions) return buildLegacyOptions(); Element jvmElement = XML.getChild(nodesElement, "jvm"); if (jvmElement == null) return ""; String jvmOptions = jvmElement.getAttribute(VespaDomBuilder.OPTIONS); if (jvmOptions.isEmpty()) return ""; validateJvmOptions(jvmOptions); return jvmOptions; } String buildLegacyOptions() { String jvmOptions = null; if (nodesElement.hasAttribute(VespaDomBuilder.JVM_OPTIONS)) { jvmOptions = nodesElement.getAttribute(VespaDomBuilder.JVM_OPTIONS); if (! jvmOptions.isEmpty()) logger.logApplicationPackage(WARNING, "'jvm-options' is deprecated and will be removed in Vespa 8." + " Please merge 'jvm-options' into 'options' or 'gc-options' in 'jvm' element." + " See https://docs.vespa.ai/en/reference/services-container.html#jvm"); } validateJvmOptions(jvmOptions); return jvmOptions; } private void validateJvmOptions(String jvmOptions) { if (jvmOptions == null || jvmOptions.isEmpty()) return; String[] optionList = jvmOptions.split(" "); List invalidOptions = Arrays.stream(optionList) .filter(option -> !option.isEmpty()) .filter(option -> !Pattern.matches(validPattern.pattern(), option)) .sorted() .collect(Collectors.toList()); if (isHosted) invalidOptions.addAll(Arrays.stream(optionList) .filter(option -> !option.isEmpty()) .filter(option -> Pattern.matches(invalidInHostedatttern.pattern(), option)) .sorted().toList()); if (invalidOptions.isEmpty()) return; String message = "Invalid or misplaced JVM options in services.xml: " + String.join(",", invalidOptions) + "." + " See https://docs.vespa.ai/en/reference/services-container.html#jvm"; if (isHosted) throw new IllegalArgumentException(message); else logger.logApplicationPackage(WARNING, message); } } /** * Validates JVM GC options and logs a warning or fails deployment (depending on feature flag) * if anyone of them has invalid syntax or is an option that is unsupported for the running system * (e.g. uses CMS options for hosted Vespa, which uses JDK 17). */ private static class JvmGcOptions { private static final Pattern validPattern = Pattern.compile("-XX:[+-]*[a-zA-z0-9=]+"); private static final Pattern invalidCMSPattern = Pattern.compile("-XX:[+-]\\w*CMS[a-zA-z0-9=]+"); private final DeployState deployState; private final String jvmGcOptions; private final DeployLogger logger; private final boolean isHosted; public JvmGcOptions(DeployState deployState, String jvmGcOptions) { this.deployState = deployState; this.jvmGcOptions = jvmGcOptions; this.logger = deployState.getDeployLogger(); this.isHosted = deployState.isHosted(); } private String build() { String options = deployState.getProperties().jvmGCOptions(); if (jvmGcOptions != null) { options = jvmGcOptions; String[] optionList = options.split(" "); List invalidOptions = Arrays.stream(optionList) .filter(option -> !option.isEmpty()) .filter(option -> !Pattern.matches(validPattern.pattern(), option)) .collect(Collectors.toList()); if (isHosted) { // CMS GC options cannot be used in hosted, CMS is unsupported in JDK 17 invalidOptions.addAll(Arrays.stream(optionList) .filter(option -> !option.isEmpty()) .filter(option -> Pattern.matches(invalidCMSPattern.pattern(), option) || option.equals("-XX:+UseConcMarkSweepGC")).toList()); } logOrFailInvalidOptions(invalidOptions); } if (options == null || options.isEmpty()) options = deployState.isHosted() ? ContainerCluster.PARALLEL_GC : ContainerCluster.G1GC; return options; } private void logOrFailInvalidOptions(List options) { if (options.isEmpty()) return; Collections.sort(options); String message = "Invalid or misplaced JVM GC options in services.xml: " + String.join(",", options) + "." + " See https://docs.vespa.ai/en/reference/services-container.html#jvm"; if (isHosted) throw new IllegalArgumentException(message); else logger.logApplicationPackage(WARNING, message); } } }