// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.model.api.HostProvisioner; import com.yahoo.config.model.producer.AnyConfigProducer; import com.yahoo.config.model.producer.TreeConfigProducer; import com.yahoo.config.provision.Capacity; import com.yahoo.config.provision.ClusterMembership; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.HostSpec; import com.yahoo.config.provision.ProvisionLogger; import java.net.UnknownHostException; import java.util.LinkedHashMap; import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.function.BiConsumer; import java.util.logging.Level; import java.util.logging.Logger; import java.util.stream.Collectors; import static java.util.logging.Level.FINE; /** * The parent node for all Host instances, and thus accessible * to enable services to get their Host. * * @author gjoranv */ public class HostSystem extends TreeConfigProducer { private static final Logger log = Logger.getLogger(HostSystem.class.getName()); private static final boolean doCheckIp; private final Map hostname2host = new LinkedHashMap<>(); private final HostProvisioner provisioner; private final DeployLogger deployLogger; private final boolean isHosted; static { String checkIpProperty = System.getProperty("config_model.ip_check", "true"); doCheckIp = ! checkIpProperty.equalsIgnoreCase("false"); } public HostSystem(TreeConfigProducer parent, String name, HostProvisioner provisioner, DeployLogger deployLogger, boolean isHosted) { super(parent, name); this.provisioner = provisioner; this.deployLogger = deployLogger; this.isHosted = isHosted; } String checkHostname(String hostname) { if (isHosted) return hostname; // Done in node-repo instead if (doCheckIp) { BiConsumer logFunction = deployLogger::logApplicationPackage; // Give a warning if the host does not exist try { var inetAddr = java.net.InetAddress.getByName(hostname); String canonical = inetAddr.getCanonicalHostName(); if (!hostname.equals(canonical)) { logFunction.accept(Level.WARNING, "Host named '" + hostname + "' may not receive any config " + "since it differs from its canonical hostname '" + canonical + "' (check DNS and /etc/hosts)."); } } catch (UnknownHostException e) { logFunction.accept(Level.WARNING, "Unable to lookup IP address of host: " + hostname); } } return hostname; } @Override public String toString() { return "hosts [" + hostname2host.values().stream() .map(HostResource::getHostname) .collect(Collectors.joining(", ")) + "]"; } public HostResource getHost(String hostAlias) { HostSpec hostSpec = provisioner.allocateHost(hostAlias); HostResource resource = hostname2host.get(hostSpec.hostname()); return resource != null ? resource : addNewHost(hostSpec); } private HostResource addNewHost(HostSpec hostSpec) { String hostname = checkHostname(hostSpec.hostname()); HostResource hostResource = new HostResource(Host.createHost(this, hostname), hostSpec); hostSpec.networkPorts().ifPresent(np -> hostResource.ports().addNetworkPorts(np)); hostname2host.put(hostname, hostResource); return hostResource; } /** Returns the hosts owned by the application having this system - i.e. all hosts except config servers */ public List getHosts() { return hostname2host.values().stream() .filter(host -> !host.getHost().runsConfigServer()) .toList(); } /** Returns the hosts in this system */ public List getAllHosts() { return hostname2host.values().stream().toList(); } public void dumpPortAllocations() { for (HostResource hr : getHosts()) { hr.ports().flushPortReservations(); } } public Map allocateHosts(ClusterSpec cluster, Capacity capacity, DeployLogger logger) { List allocatedHosts = provisioner.prepare(cluster, capacity, new ProvisionDeployLogger(logger)); // TODO: Even if HostResource owns a set of memberships, we need to return a map because the caller needs the current membership. Map retAllocatedHosts = new LinkedHashMap<>(); for (HostSpec spec : allocatedHosts) { // This is needed for single node host provisioner to work in unit tests for hosted vespa applications. HostResource host = getExistingHost(spec).orElseGet(() -> addNewHost(spec)); retAllocatedHosts.put(host, spec.membership().orElse(null)); } retAllocatedHosts.keySet().forEach(host -> log.log(FINE, () -> "Allocated host " + host.getHostname() + " with resources " + host.advertisedResources())); return retAllocatedHosts; } private Optional getExistingHost(HostSpec key) { List hosts = hostname2host.values().stream() .filter(resource -> resource.getHostname().equals(key.hostname())) .toList(); if (hosts.isEmpty()) { return Optional.empty(); } else { log.log(FINE, () -> "Found existing host resource for " + key.hostname() + " with resources" + hosts.get(0).advertisedResources()); return Optional.of(hosts.get(0)); } } public void addBoundHost(HostResource host) { hostname2host.put(host.getHostname(), host); } Set getHostSpecs() { return getHosts().stream().map(HostResource::spec).collect(Collectors.toCollection(LinkedHashSet::new)); } /** A provision logger which forwards to a deploy logger */ private static class ProvisionDeployLogger implements ProvisionLogger { private final DeployLogger deployLogger; public ProvisionDeployLogger(DeployLogger deployLogger) { this.deployLogger = deployLogger; } @Override public void log(Level level, String message) { deployLogger.log(level, message); } @Override public void logApplicationPackage(Level level, String message) { deployLogger.logApplicationPackage(level, message); } } }