diff options
65 files changed, 660 insertions, 276 deletions
diff --git a/config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java b/config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java index c21b532b03b..6047b6a9818 100644 --- a/config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java +++ b/config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java @@ -108,7 +108,8 @@ public class InMemoryProvisioner implements HostProvisioner { List<Host> defaultHosts = freeNodes.get(defaultResources); if (defaultHosts.isEmpty()) throw new IllegalArgumentException("No more hosts with default resources available"); Host newHost = freeNodes.removeValue(defaultResources, 0); - return new HostSpec(newHost.hostname(), newHost.aliases(), newHost.flavor(), Optional.empty(), newHost.version()); + // Note: Always returns HostSpec with empty dockerImageRepo, which is OK since this method is never used when docker image repo is set + return new HostSpec(newHost.hostname(), newHost.aliases(), newHost.flavor(), Optional.empty(), newHost.version(), Optional.empty()); } @Override @@ -155,7 +156,10 @@ public class InMemoryProvisioner implements HostProvisioner { host.aliases(), host.flavor(), Optional.of(host.membership().get().retire()), - host.version()); + host.version(), + Optional.empty(), + Optional.empty(), + host.dockerImageRepo()); } private List<HostSpec> allocateHostGroup(ClusterSpec clusterGroup, Optional<NodeResources> requestedResources, diff --git a/config-model/src/main/java/com/yahoo/vespa/model/HostSystem.java b/config-model/src/main/java/com/yahoo/vespa/model/HostSystem.java index eda562bea5a..3765e683b18 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/HostSystem.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/HostSystem.java @@ -11,7 +11,6 @@ import com.yahoo.config.provision.HostSpec; import com.yahoo.config.provision.ProvisionLogger; import java.net.UnknownHostException; -import java.util.Collections; import java.util.LinkedHashMap; import java.util.LinkedHashSet; import java.util.List; @@ -34,13 +33,11 @@ public class HostSystem extends AbstractConfigProducer<Host> { private static Logger log = Logger.getLogger(HostSystem.class.getName()); - private Map<String,String> hostnames = new LinkedHashMap<>(); - private final Map<String, HostResource> hostname2host = new LinkedHashMap<>(); private final HostProvisioner provisioner; private final DeployLogger deployLogger; - public HostSystem(AbstractConfigProducer parent, String name, HostProvisioner provisioner, DeployLogger deployLogger) { + public HostSystem(AbstractConfigProducer<?> parent, String name, HostProvisioner provisioner, DeployLogger deployLogger) { super(parent, name); this.provisioner = provisioner; this.deployLogger = deployLogger; @@ -49,7 +46,8 @@ public class HostSystem extends AbstractConfigProducer<Host> { void checkName(String hostname) { // Give a warning if the host does not exist try { - Object address = java.net.InetAddress.getByName(hostname); + @SuppressWarnings("unused") + Object ignore = java.net.InetAddress.getByName(hostname); } catch (UnknownHostException e) { deployLogger.log(Level.WARNING, "Unable to lookup IP address of host: " + hostname); } @@ -78,36 +76,10 @@ public class HostSystem extends AbstractConfigProducer<Host> { return hostname2host.get(name); } - /** - * Returns the canonical name of a given host. This will cache names for faster lookup. - * - * @param hostname the hostname to retrieve the canonical hostname for. - * @return The canonical hostname, or null if unable to resolve. - * @throws UnknownHostException if the hostname cannot be resolved - */ - public String getCanonicalHostname(String hostname) throws UnknownHostException { - if ( ! hostnames.containsKey(hostname)) { - hostnames.put(hostname, lookupCanonicalHostname(hostname)); - } - return hostnames.get(hostname); - } - - /** - * Static helper method that looks up the canonical name of a given host. - * - * @param hostname the hostname to retrieve the canonical hostname for. - * @return The canonical hostname, or null if unable to resolve. - * @throws UnknownHostException if the hostname cannot be resolved - */ - // public - This is used by amenders outside this repo - public static String lookupCanonicalHostname(String hostname) throws UnknownHostException { - return java.net.InetAddress.getByName(hostname).getCanonicalHostName(); - } - @Override public String toString() { return "hosts [" + hostname2host.values().stream() - .map(host -> host.getHostname()) + .map(HostResource::getHostname) .collect(Collectors.joining(", ")) + "]"; } @@ -169,7 +141,7 @@ public class HostSystem extends AbstractConfigProducer<Host> { } Set<HostSpec> getHostSpecs() { - return getHosts().stream().map(host -> host.spec()).collect(Collectors.toCollection(LinkedHashSet::new)); + return getHosts().stream().map(HostResource::spec).collect(Collectors.toCollection(LinkedHashSet::new)); } /** A provision logger which forwards to a deploy logger */ diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/NodesSpecification.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/NodesSpecification.java index f9e29fe3a28..a0b573859bc 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/NodesSpecification.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/NodesSpecification.java @@ -195,7 +195,8 @@ public class NodesSpecification { DeployLogger logger) { if (combinedId.isPresent()) clusterType = ClusterSpec.Type.combined; - ClusterSpec cluster = ClusterSpec.request(clusterType, clusterId, version, exclusive, combinedId.map(ClusterSpec.Id::from)); + ClusterSpec cluster = ClusterSpec.request(clusterType, clusterId, version, exclusive, + combinedId.map(ClusterSpec.Id::from), dockerImageRepo); return hostSystem.allocateHosts(cluster, Capacity.fromCount(count, resources, required, canFail), groups, logger); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java index c840a8b93cd..14e667aba00 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java @@ -535,7 +535,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { private void addNodesFromXml(ApplicationContainerCluster cluster, Element containerElement, ConfigModelContext context) { Element nodesElement = XML.getChild(containerElement, "nodes"); - if (nodesElement == null) { // default single node on localhost + if (nodesElement == null) { ApplicationContainer node = new ApplicationContainer(cluster, "container.0", 0, cluster.isHostedVespa()); HostResource host = allocateSingleNodeHost(cluster, log, containerElement, context); node.setHostResource(host); @@ -621,23 +621,18 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { private HostResource allocateSingleNodeHost(ApplicationContainerCluster cluster, DeployLogger logger, Element containerElement, ConfigModelContext context) { DeployState deployState = context.getDeployState(); HostSystem hostSystem = cluster.hostSystem(); - if (deployState.isHosted()) { - Optional<HostResource> singleContentHost = getHostResourceFromContentClusters(cluster, containerElement, context); - if (singleContentHost.isPresent()) { // there is a content cluster; put the container on its first node - return singleContentHost.get(); - } - else { // request 1 node - ClusterSpec clusterSpec = ClusterSpec.request(ClusterSpec.Type.container, - ClusterSpec.Id.from(cluster.getName()), - deployState.getWantedNodeVespaVersion(), - false, - Optional.empty()); - Capacity capacity = Capacity.fromCount(1, - Optional.empty(), - false, - ! deployState.getProperties().isBootstrap()); - return hostSystem.allocateHosts(clusterSpec, capacity, 1, logger).keySet().iterator().next(); - } + if (deployState.isHosted()) { // request 1 node + ClusterSpec clusterSpec = ClusterSpec.request(ClusterSpec.Type.container, + ClusterSpec.Id.from(cluster.getName()), + deployState.getWantedNodeVespaVersion(), + false, + Optional.empty(), + deployState.getWantedDockerImageRepo()); + Capacity capacity = Capacity.fromCount(1, + Optional.empty(), + false, + !deployState.getProperties().isBootstrap()); + return hostSystem.allocateHosts(clusterSpec, capacity, 1, logger).keySet().iterator().next(); } else { return hostSystem.getHost(Container.SINGLENODE_CONTAINER_SERVICESPEC); } @@ -658,7 +653,8 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { ClusterSpec.Id.from(cluster.getName()), context.getDeployState().getWantedNodeVespaVersion(), false, - Optional.empty()); + Optional.empty(), + context.getDeployState().getWantedDockerImageRepo()); Map<HostResource, ClusterMembership> hosts = cluster.getRoot().hostSystem().allocateHosts(clusterSpec, Capacity.fromRequiredNodeType(type), 1, log); @@ -683,43 +679,6 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { return createNodesFromHosts(context.getDeployLogger(), hosts, cluster); } - /** - * This is used in case we are on hosted Vespa and no nodes tag is supplied: - * If there are content clusters this will pick the first host in the first cluster as the container node. - * If there are no content clusters this will return empty (such that the node can be created by the container here). - */ - private Optional<HostResource> getHostResourceFromContentClusters(ApplicationContainerCluster cluster, Element containersElement, ConfigModelContext context) { - Optional<Element> services = servicesRootOf(containersElement); - if ( ! services.isPresent()) - return Optional.empty(); - List<Element> contentServices = XML.getChildren(services.get(), "content"); - if ( contentServices.isEmpty() ) return Optional.empty(); - Element contentNodesElementOrNull = XML.getChild(contentServices.get(0), "nodes"); - - NodesSpecification nodesSpec; - if (contentNodesElementOrNull == null) - nodesSpec = NodesSpecification.nonDedicated(1, context); - else - nodesSpec = NodesSpecification.from(new ModelElement(contentNodesElementOrNull), context); - - Map<HostResource, ClusterMembership> hosts = - StorageGroup.provisionHosts(nodesSpec, - contentServices.get(0).getAttribute("id"), - cluster.getRoot().hostSystem(), - context.getDeployLogger()); - return Optional.of(hosts.keySet().iterator().next()); - } - - /** Returns the services element above the given Element, or empty if there is no services element */ - private Optional<Element> servicesRootOf(Element element) { - Node parent = element.getParentNode(); - if (parent == null) return Optional.empty(); - if ( ! (parent instanceof Element)) return Optional.empty(); - Element parentElement = (Element)parent; - if (parentElement.getTagName().equals("services")) return Optional.of(parentElement); - return servicesRootOf(parentElement); - } - private List<ApplicationContainer> createNodesFromHosts(DeployLogger deployLogger, Map<HostResource, ClusterMembership> hosts, ApplicationContainerCluster cluster) { List<ApplicationContainer> nodes = new ArrayList<>(); for (Map.Entry<HostResource, ClusterMembership> entry : hosts.entrySet()) { diff --git a/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java b/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java index d215fdbb7a0..f303ea9a42d 100644 --- a/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java +++ b/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java @@ -1374,7 +1374,7 @@ public class ModelProvisioningTest { } @Test - public void testNoNodeTagMeans1Node() { + public void testNoNodeTagMeans1NodePerCluster() { String services = "<?xml version='1.0' encoding='utf-8' ?>\n" + "<services>" + @@ -1389,9 +1389,9 @@ public class ModelProvisioningTest { " </content>" + "</services>"; VespaModelTester tester = new VespaModelTester(); - tester.addHosts(1); + tester.addHosts(2); VespaModel model = tester.createModel(services, true); - assertEquals(1, model.getRoot().hostSystem().getHosts().size()); + assertEquals(2, model.getRoot().hostSystem().getHosts().size()); assertEquals(1, model.getAdmin().getSlobroks().size()); assertEquals(1, model.getContainerClusters().get("foo").getContainers().size()); assertEquals(1, model.getContentClusters().get("bar").getRootGroup().countNodes()); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/HostResourceTest.java b/config-model/src/test/java/com/yahoo/vespa/model/HostResourceTest.java index 4d8d8e52e0e..a4f0caef85b 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/HostResourceTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/HostResourceTest.java @@ -50,7 +50,7 @@ public class HostResourceTest { private static ClusterSpec clusterSpec(ClusterSpec.Type type, String id) { return ClusterSpec.from(type, ClusterSpec.Id.from(id), ClusterSpec.Group.from(0), - Version.fromString("6.42"), false, Optional.empty()); + Version.fromString("6.42"), false, Optional.empty(), Optional.empty()); } private static HostResource hostResourceWithMemberships(ClusterMembership membership) { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/VespaModelFactoryTest.java b/config-model/src/test/java/com/yahoo/vespa/model/VespaModelFactoryTest.java index 5e3f337b2d7..16a5e8cb6f2 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/VespaModelFactoryTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/VespaModelFactoryTest.java @@ -111,6 +111,7 @@ public class VespaModelFactoryTest { ClusterSpec.Group.from(0), Version.fromString("6.42"), false, + Optional.empty(), Optional.empty()), 0)); } @@ -124,6 +125,7 @@ public class VespaModelFactoryTest { ClusterSpec.Group.from(0), Version.fromString("6.42"), false, + Optional.empty(), Optional.empty()), 0))); } diff --git a/config-provisioning/abi-spec.json b/config-provisioning/abi-spec.json index 9d09ea10c2c..9f785699667 100644 --- a/config-provisioning/abi-spec.json +++ b/config-provisioning/abi-spec.json @@ -199,6 +199,7 @@ "public boolean equals(java.lang.Object)", "public java.lang.String toString()", "public static com.yahoo.config.provision.ClusterMembership from(java.lang.String, com.yahoo.component.Version)", + "public static com.yahoo.config.provision.ClusterMembership from(java.lang.String, com.yahoo.component.Version, java.util.Optional)", "public static com.yahoo.config.provision.ClusterMembership from(com.yahoo.config.provision.ClusterSpec, int)", "public static com.yahoo.config.provision.ClusterMembership retiredFrom(com.yahoo.config.provision.ClusterSpec, int)" ], @@ -269,6 +270,7 @@ "methods": [ "public com.yahoo.config.provision.ClusterSpec$Type type()", "public com.yahoo.config.provision.ClusterSpec$Id id()", + "public java.util.Optional dockerImageRepo()", "public com.yahoo.component.Version vespaVersion()", "public java.util.Optional group()", "public java.util.Optional combinedId()", @@ -276,7 +278,9 @@ "public com.yahoo.config.provision.ClusterSpec with(java.util.Optional)", "public com.yahoo.config.provision.ClusterSpec exclusive(boolean)", "public static com.yahoo.config.provision.ClusterSpec request(com.yahoo.config.provision.ClusterSpec$Type, com.yahoo.config.provision.ClusterSpec$Id, com.yahoo.component.Version, boolean, java.util.Optional)", + "public static com.yahoo.config.provision.ClusterSpec request(com.yahoo.config.provision.ClusterSpec$Type, com.yahoo.config.provision.ClusterSpec$Id, com.yahoo.component.Version, boolean, java.util.Optional, java.util.Optional)", "public static com.yahoo.config.provision.ClusterSpec from(com.yahoo.config.provision.ClusterSpec$Type, com.yahoo.config.provision.ClusterSpec$Id, com.yahoo.config.provision.ClusterSpec$Group, com.yahoo.component.Version, boolean, java.util.Optional)", + "public static com.yahoo.config.provision.ClusterSpec from(com.yahoo.config.provision.ClusterSpec$Type, com.yahoo.config.provision.ClusterSpec$Id, com.yahoo.config.provision.ClusterSpec$Group, com.yahoo.component.Version, boolean, java.util.Optional, java.util.Optional)", "public java.lang.String toString()", "public int hashCode()", "public boolean equals(java.lang.Object)", @@ -482,6 +486,7 @@ "public void <init>(java.lang.String, java.util.List, java.util.Optional, java.util.Optional, java.util.Optional)", "public void <init>(java.lang.String, java.util.List, java.util.Optional, java.util.Optional, java.util.Optional, java.util.Optional)", "public void <init>(java.lang.String, java.util.List, java.util.Optional, java.util.Optional, java.util.Optional, java.util.Optional, java.util.Optional)", + "public void <init>(java.lang.String, java.util.List, java.util.Optional, java.util.Optional, java.util.Optional, java.util.Optional, java.util.Optional, java.util.Optional)", "public java.lang.String hostname()", "public java.util.List aliases()", "public java.util.Optional flavor()", @@ -489,6 +494,7 @@ "public java.util.Optional membership()", "public java.util.Optional networkPorts()", "public java.util.Optional requestedResources()", + "public java.util.Optional dockerImageRepo()", "public com.yahoo.config.provision.HostSpec withPorts(java.util.Optional)", "public java.lang.String toString()", "public boolean equals(java.lang.Object)", diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterMembership.java b/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterMembership.java index 0fb78b59aaf..0cd30061139 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterMembership.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterMembership.java @@ -20,7 +20,7 @@ public class ClusterMembership { protected ClusterMembership() {} - private ClusterMembership(String stringValue, Version vespaVersion) { + private ClusterMembership(String stringValue, Version vespaVersion, Optional<String> dockerImageRepo) { String[] components = stringValue.split("/"); if (components.length < 4) throw new RuntimeException("Could not parse '" + stringValue + "' to a cluster membership. " + @@ -41,7 +41,7 @@ public class ClusterMembership { this.cluster = ClusterSpec.from(ClusterSpec.Type.valueOf(components[0]), ClusterSpec.Id.from(components[1]), ClusterSpec.Group.from(Integer.parseInt(components[2])), vespaVersion, - exclusive, combinedId.map(ClusterSpec.Id::from)); + exclusive, combinedId.map(ClusterSpec.Id::from), dockerImageRepo); this.index = Integer.parseInt(components[3]); this.stringValue = toStringValue(); } @@ -106,8 +106,14 @@ public class ClusterMembership { @Override public String toString() { return stringValue(); } + // TODO: Remove when when 7.195 is oldest model version in use + @Deprecated public static ClusterMembership from(String stringValue, Version vespaVersion) { - return new ClusterMembership(stringValue, vespaVersion); + return new ClusterMembership(stringValue, vespaVersion, Optional.empty()); + } + + public static ClusterMembership from(String stringValue, Version vespaVersion, Optional<String> dockerImageRepo) { + return new ClusterMembership(stringValue, vespaVersion, dockerImageRepo); } public static ClusterMembership from(ClusterSpec cluster, int index) { diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterSpec.java b/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterSpec.java index afdae0023bc..3f7502bd3f4 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterSpec.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterSpec.java @@ -22,8 +22,10 @@ public final class ClusterSpec { private final Version vespaVersion; private boolean exclusive; private final Optional<Id> combinedId; + private final Optional<String> dockerImageRepo; - private ClusterSpec(Type type, Id id, Optional<Group> groupId, Version vespaVersion, boolean exclusive, Optional<Id> combinedId) { + private ClusterSpec(Type type, Id id, Optional<Group> groupId, Version vespaVersion, boolean exclusive, + Optional<Id> combinedId, Optional<String> dockerImageRepo) { this.type = type; this.id = id; this.groupId = groupId; @@ -34,6 +36,7 @@ public final class ClusterSpec { throw new IllegalArgumentException("combinedId must be empty for cluster of type " + type); } this.combinedId = combinedId; + this.dockerImageRepo = dockerImageRepo; } /** Returns the cluster type */ @@ -42,6 +45,9 @@ public final class ClusterSpec { /** Returns the cluster id */ public Id id() { return id; } + /** Returns the docker image repository part of a docker image we want this cluster to run */ + public Optional<String> dockerImageRepo() { return dockerImageRepo; } + /** Returns the version of Vespa that we want this cluster to run */ public Version vespaVersion() { return vespaVersion; } @@ -61,24 +67,42 @@ public final class ClusterSpec { public boolean isExclusive() { return exclusive; } public ClusterSpec with(Optional<Group> newGroup) { - return new ClusterSpec(type, id, newGroup, vespaVersion, exclusive, combinedId); + return new ClusterSpec(type, id, newGroup, vespaVersion, exclusive, combinedId, dockerImageRepo); } public ClusterSpec exclusive(boolean exclusive) { - return new ClusterSpec(type, id, groupId, vespaVersion, exclusive, combinedId); + return new ClusterSpec(type, id, groupId, vespaVersion, exclusive, combinedId, dockerImageRepo); + } + + // TODO: Remove when when 7.195 is oldest model version in use + // TODO: Add @Deprecated when internal repo has been updated to not use this method + // @Deprecated + public static ClusterSpec request(Type type, Id id, Version vespaVersion, boolean exclusive, + Optional<Id> combinedId) { + return request(type, id, vespaVersion, exclusive, combinedId, Optional.empty()); + } + + public static ClusterSpec request(Type type, Id id, Version vespaVersion, boolean exclusive, + Optional<Id> combinedId, Optional<String> dockerImageRepo) { + return new ClusterSpec(type, id, Optional.empty(), vespaVersion, exclusive, combinedId, dockerImageRepo); } - public static ClusterSpec request(Type type, Id id, Version vespaVersion, boolean exclusive, Optional<Id> combinedId) { - return new ClusterSpec(type, id, Optional.empty(), vespaVersion, exclusive, combinedId); + // TODO: Remove when when 7.195 is oldest model version in use + // TODO: Add @Deprecated when internal repo has been updated to not use this method + // @Deprecated + public static ClusterSpec from(Type type, Id id, Group groupId, Version vespaVersion, boolean exclusive, + Optional<Id> combinedId) { + return from(type, id, groupId, vespaVersion, exclusive, combinedId, Optional.empty()); } - public static ClusterSpec from(Type type, Id id, Group groupId, Version vespaVersion, boolean exclusive, Optional<Id> combinedId) { - return new ClusterSpec(type, id, Optional.of(groupId), vespaVersion, exclusive, combinedId); + public static ClusterSpec from(Type type, Id id, Group groupId, Version vespaVersion, boolean exclusive, + Optional<Id> combinedId, Optional<String> dockerImageRepo) { + return new ClusterSpec(type, id, Optional.of(groupId), vespaVersion, exclusive, combinedId, dockerImageRepo); } @Override public String toString() { - return type + " " + id + " " + groupId.map(group -> group + " ").orElse("") + vespaVersion; + return type + " " + id + " " + groupId.map(group -> group + " ").orElse("") + vespaVersion + (dockerImageRepo.map(repo -> " " + repo).orElse("")); } @Override @@ -93,6 +117,7 @@ public final class ClusterSpec { if ( ! other.id.equals(this.id)) return false; if ( ! other.groupId.equals(this.groupId)) return false; if ( ! other.vespaVersion.equals(this.vespaVersion)) return false; + if ( ! other.dockerImageRepo.orElse("").equals(this.dockerImageRepo.orElse(""))) return false; return true; } diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/HostSpec.java b/config-provisioning/src/main/java/com/yahoo/config/provision/HostSpec.java index 63725d9a535..c49448a2e43 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/HostSpec.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/HostSpec.java @@ -1,6 +1,8 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.config.provision; +import com.yahoo.component.Version; + import java.util.ArrayList; import java.util.List; import java.util.Objects; @@ -25,7 +27,9 @@ public class HostSpec implements Comparable<HostSpec> { private final Optional<Flavor> flavor; - private final Optional<com.yahoo.component.Version> version; + private final Optional<Version> version; + + private final Optional<String> dockerImageRepo; private final Optional<NetworkPorts> networkPorts; @@ -35,7 +39,7 @@ public class HostSpec implements Comparable<HostSpec> { this(hostname, new ArrayList<>(), Optional.empty(), membership); } - public HostSpec(String hostname, ClusterMembership membership, Flavor flavor, Optional<com.yahoo.component.Version> version) { + public HostSpec(String hostname, ClusterMembership membership, Flavor flavor, Optional<Version> version) { this(hostname, new ArrayList<>(), Optional.of(flavor), Optional.of(membership), version); } @@ -56,19 +60,26 @@ public class HostSpec implements Comparable<HostSpec> { } public HostSpec(String hostname, List<String> aliases, Optional<Flavor> flavor, - Optional<ClusterMembership> membership, Optional<com.yahoo.component.Version> version) { + Optional<ClusterMembership> membership, Optional<Version> version) { this(hostname, aliases, flavor, membership, version, Optional.empty()); } public HostSpec(String hostname, List<String> aliases, Optional<Flavor> flavor, - Optional<ClusterMembership> membership, Optional<com.yahoo.component.Version> version, + Optional<ClusterMembership> membership, Optional<Version> version, Optional<NetworkPorts> networkPorts) { this(hostname, aliases, flavor, membership, version, networkPorts, Optional.empty()); } public HostSpec(String hostname, List<String> aliases, Optional<Flavor> flavor, - Optional<ClusterMembership> membership, Optional<com.yahoo.component.Version> version, + Optional<ClusterMembership> membership, Optional<Version> version, Optional<NetworkPorts> networkPorts, Optional<NodeResources> requestedResources) { + this(hostname, aliases, flavor, membership, version, networkPorts, requestedResources, Optional.empty()); + } + + public HostSpec(String hostname, List<String> aliases, Optional<Flavor> flavor, + Optional<ClusterMembership> membership, Optional<Version> version, + Optional<NetworkPorts> networkPorts, Optional<NodeResources> requestedResources, + Optional<String> dockerImageRepo) { if (hostname == null || hostname.isEmpty()) throw new IllegalArgumentException("Hostname must be specified"); this.hostname = hostname; this.aliases = List.copyOf(aliases); @@ -77,6 +88,7 @@ public class HostSpec implements Comparable<HostSpec> { this.version = Objects.requireNonNull(version, "Version cannot be null but can be empty");; this.networkPorts = Objects.requireNonNull(networkPorts, "Network ports cannot be null but can be empty");; this.requestedResources = Objects.requireNonNull(requestedResources, "RequestedResources cannot be null"); + this.dockerImageRepo = Objects.requireNonNull(dockerImageRepo, "Docker image repo cannot be null but can be empty"); } /** Returns the name identifying this host */ @@ -99,8 +111,10 @@ public class HostSpec implements Comparable<HostSpec> { /** Returns the requested resources leading to this host being provisioned, or empty if not known */ public Optional<NodeResources> requestedResources() { return requestedResources; } + public Optional<String> dockerImageRepo() { return dockerImageRepo; } + public HostSpec withPorts(Optional<NetworkPorts> ports) { - return new HostSpec(hostname, aliases, flavor, membership, version, ports, requestedResources); + return new HostSpec(hostname, aliases, flavor, membership, version, ports, requestedResources, dockerImageRepo); } @Override diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/serialization/AllocatedHostsSerializer.java b/config-provisioning/src/main/java/com/yahoo/config/provision/serialization/AllocatedHostsSerializer.java index f66bacbc383..779dd1d24f7 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/serialization/AllocatedHostsSerializer.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/serialization/AllocatedHostsSerializer.java @@ -59,6 +59,9 @@ public class AllocatedHostsSerializer { /** Wanted version */ private static final String hostSpecVespaVersionKey = "vespaVersion"; + /** Wanted docker image repo */ + private static final String hostSpecDockerImageRepoKey = "dockerImageRepo"; + /** Current version */ private static final String hostSpecCurrentVespaVersionKey = "currentVespaVersion"; private static final String hostSpecNetworkPortsKey = "ports"; @@ -79,6 +82,7 @@ public class AllocatedHostsSerializer { private static void toSlime(HostSpec host, Cursor object) { object.setString(hostSpecHostNameKey, host.hostname()); aliasesToSlime(host, object); + // TODO serialize dockerImageRepo host.membership().ifPresent(membership -> { object.setString(hostSpecMembershipKey, membership.stringValue()); object.setString(hostSpecVespaVersionKey, membership.cluster().vespaVersion().toFullString()); @@ -196,7 +200,10 @@ public class AllocatedHostsSerializer { private static ClusterMembership membershipFromSlime(Inspector object) { return ClusterMembership.from(object.field(hostSpecMembershipKey).asString(), - com.yahoo.component.Version.fromString(object.field(hostSpecVespaVersionKey).asString())); + com.yahoo.component.Version.fromString(object.field(hostSpecVespaVersionKey).asString()), + object.field(hostSpecDockerImageRepoKey).valid() + ? Optional.of(object.field(hostSpecDockerImageRepoKey).asString()) + : Optional.empty()); } private static Optional<String> optionalString(Inspector inspector) { diff --git a/config-provisioning/src/test/java/com/yahoo/config/provision/ClusterMembershipTest.java b/config-provisioning/src/test/java/com/yahoo/config/provision/ClusterMembershipTest.java index 4418b5c85ca..3ac04cd1cb5 100644 --- a/config-provisioning/src/test/java/com/yahoo/config/provision/ClusterMembershipTest.java +++ b/config-provisioning/src/test/java/com/yahoo/config/provision/ClusterMembershipTest.java @@ -18,72 +18,81 @@ public class ClusterMembershipTest { @Test public void testContainerServiceInstance() { - ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("id1"), Version.fromString("6.42"), false, Optional.empty()); + ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("id1"), + Version.fromString("6.42"), false, Optional.empty(), Optional.empty()); assertContainerService(ClusterMembership.from(cluster, 3)); } @Test public void testSerializationWithOptionalParts() { { - ClusterMembership instance = ClusterMembership.from("container/id1/4/37/exclusive/retired", Vtag.currentVersion); - ClusterMembership serialized = ClusterMembership.from(instance.stringValue(), Vtag.currentVersion); + ClusterMembership instance = ClusterMembership.from("container/id1/4/37/exclusive/retired", Vtag.currentVersion, Optional.empty()); + ClusterMembership serialized = ClusterMembership.from(instance.stringValue(), Vtag.currentVersion, Optional.empty()); assertEquals(instance, serialized); assertTrue(instance.retired()); assertTrue(instance.cluster().isExclusive()); assertFalse(instance.cluster().combinedId().isPresent()); + assertTrue(instance.cluster().dockerImageRepo().isEmpty()); } { - ClusterMembership instance = ClusterMembership.from("container/id1/4/37/exclusive", Vtag.currentVersion); - ClusterMembership serialized = ClusterMembership.from(instance.stringValue(), Vtag.currentVersion); + ClusterMembership instance = ClusterMembership.from("container/id1/4/37/exclusive", Vtag.currentVersion, Optional.empty()); + ClusterMembership serialized = ClusterMembership.from(instance.stringValue(), Vtag.currentVersion, Optional.empty()); assertEquals(instance, serialized); assertFalse(instance.retired()); assertTrue(instance.cluster().isExclusive()); assertFalse(instance.cluster().combinedId().isPresent()); + assertTrue(instance.cluster().dockerImageRepo().isEmpty()); } { - ClusterMembership instance = ClusterMembership.from("combined/id1/4/37/exclusive/containerId1", Vtag.currentVersion); - ClusterMembership serialized = ClusterMembership.from(instance.stringValue(), Vtag.currentVersion); + Optional<String> dockerImageRepo = Optional.of("docker.foo.com:4443/vespa/bar"); + ClusterMembership instance = ClusterMembership.from("combined/id1/4/37/exclusive/containerId1", Vtag.currentVersion, dockerImageRepo); + ClusterMembership serialized = ClusterMembership.from(instance.stringValue(), Vtag.currentVersion, dockerImageRepo); assertEquals(instance, serialized); assertFalse(instance.retired()); assertTrue(instance.cluster().isExclusive()); assertEquals(ClusterSpec.Id.from("containerId1"), instance.cluster().combinedId().get()); + assertEquals(dockerImageRepo.get(), instance.cluster().dockerImageRepo().get()); } } @Test public void testServiceInstance() { - ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("id1"), Version.fromString("6.42"), false, Optional.empty()); + ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("id1"), + Version.fromString("6.42"), false, Optional.empty(), Optional.empty()); assertContentService(ClusterMembership.from(cluster, 37)); } @Test public void testServiceInstanceWithGroup() { ClusterSpec cluster = ClusterSpec.from(ClusterSpec.Type.content, ClusterSpec.Id.from("id1"), - ClusterSpec.Group.from(4), Version.fromString("6.42"), false, Optional.empty()); + ClusterSpec.Group.from(4), Version.fromString("6.42"), + false, Optional.empty(), Optional.empty()); assertContentServiceWithGroup(ClusterMembership.from(cluster, 37)); } @Test public void testServiceInstanceWithGroupFromString() { - assertContentServiceWithGroup(ClusterMembership.from("content/id1/4/37", Vtag.currentVersion)); + assertContentServiceWithGroup(ClusterMembership.from("content/id1/4/37", Vtag.currentVersion, Optional.empty())); } @Test public void testServiceInstanceWithRetire() { - ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("id1"), Version.fromString("6.42"), false, Optional.empty()); + ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("id1"), + Version.fromString("6.42"), false, Optional.empty(), Optional.empty()); assertContentServiceWithRetire(ClusterMembership.retiredFrom(cluster, 37)); } @Test public void testServiceInstanceWithGroupAndRetire() { ClusterSpec cluster = ClusterSpec.from(ClusterSpec.Type.content, ClusterSpec.Id.from("id1"), - ClusterSpec.Group.from(4), Version.fromString("6.42"), false, Optional.empty()); + ClusterSpec.Group.from(4), Version.fromString("6.42"), + false, Optional.empty(), Optional.empty()); assertContentServiceWithGroupAndRetire(ClusterMembership.retiredFrom(cluster, 37)); } @Test public void testServiceInstanceWithGroupAndRetireFromString() { - assertContentServiceWithGroupAndRetire(ClusterMembership.from("content/id1/4/37/retired", Vtag.currentVersion)); + assertContentServiceWithGroupAndRetire(ClusterMembership.from("content/id1/4/37/retired", Vtag.currentVersion, Optional.empty())); } private void assertContainerService(ClusterMembership instance) { diff --git a/config-provisioning/src/test/java/com/yahoo/config/provision/ClusterSpecTest.java b/config-provisioning/src/test/java/com/yahoo/config/provision/ClusterSpecTest.java index 7f8e41116a6..eb2407e988a 100644 --- a/config-provisioning/src/test/java/com/yahoo/config/provision/ClusterSpecTest.java +++ b/config-provisioning/src/test/java/com/yahoo/config/provision/ClusterSpecTest.java @@ -54,7 +54,7 @@ public class ClusterSpecTest { private static ClusterSpec spec(ClusterSpec.Type type, String id) { return ClusterSpec.from(type, ClusterSpec.Id.from(id), ClusterSpec.Group.from(1), Version.emptyVersion, - false, Optional.empty()); + false, Optional.empty(), Optional.empty()); } } diff --git a/config-provisioning/src/test/java/com/yahoo/config/provision/HostFilterTest.java b/config-provisioning/src/test/java/com/yahoo/config/provision/HostFilterTest.java index e1d6f061446..66ca1170487 100644 --- a/config-provisioning/src/test/java/com/yahoo/config/provision/HostFilterTest.java +++ b/config-provisioning/src/test/java/com/yahoo/config/provision/HostFilterTest.java @@ -71,7 +71,7 @@ public class HostFilterTest { } private Optional<ClusterMembership> membership(String membershipString) { - return Optional.of(ClusterMembership.from(membershipString, Vtag.currentVersion)); + return Optional.of(ClusterMembership.from(membershipString, Vtag.currentVersion, Optional.empty())); } } diff --git a/config-provisioning/src/test/java/com/yahoo/config/provision/serialization/AllocatedHostsSerializerTest.java b/config-provisioning/src/test/java/com/yahoo/config/provision/serialization/AllocatedHostsSerializerTest.java index 193caf3edba..137e1478073 100644 --- a/config-provisioning/src/test/java/com/yahoo/config/provision/serialization/AllocatedHostsSerializerTest.java +++ b/config-provisioning/src/test/java/com/yahoo/config/provision/serialization/AllocatedHostsSerializerTest.java @@ -37,7 +37,8 @@ public class AllocatedHostsSerializerTest { hosts.add(new HostSpec("with-aliases", List.of("alias1", "alias2"))); hosts.add(new HostSpec("allocated", - Optional.of(ClusterMembership.from("container/test/0/0", Version.fromString("6.73.1"))))); + Optional.of(ClusterMembership.from("container/test/0/0", Version.fromString("6.73.1"), + Optional.of("docker.foo.com:4443/vespa/bar"))))); hosts.add(new HostSpec("flavor-from-resources-1", Collections.emptyList(), new Flavor(new NodeResources(0.5, 3.1, 4, 1)))); hosts.add(new HostSpec("flavor-from-resources-2", @@ -72,6 +73,7 @@ public class AllocatedHostsSerializerTest { assertEquals(expectedHost.networkPorts(), deserializedHost.networkPorts()); assertEquals(expectedHost.aliases(), deserializedHost.aliases()); assertEquals(expectedHost.requestedResources(), deserializedHost.requestedResources()); + assertEquals(expectedHost.dockerImageRepo(), deserializedHost.dockerImageRepo()); } } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackageTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackageTest.java index f2c6aac2bda..97d90d8b6b6 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackageTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackageTest.java @@ -39,7 +39,8 @@ public class ZKApplicationPackageTest { private static final Optional<Flavor> TEST_FLAVOR = new MockNodeFlavors().getFlavor(TEST_FLAVOR_NAME); private static final AllocatedHosts ALLOCATED_HOSTS = AllocatedHosts.withHosts( Collections.singleton(new HostSpec("foo.yahoo.com", Collections.emptyList(), TEST_FLAVOR, Optional.empty(), - Optional.of(com.yahoo.component.Version.fromString("6.0.1"))))); + Optional.of(Version.fromString("6.0.1")), Optional.empty(), + Optional.empty(), Optional.of("docker repo")))); private ConfigCurator configCurator; @@ -80,6 +81,8 @@ public class ZKApplicationPackageTest { assertThat(Utf8.toString(toJson(readInfo)), is(Utf8.toString(toJson(ALLOCATED_HOSTS)))); assertThat(readInfo.getHosts().iterator().next().flavor(), is(TEST_FLAVOR)); assertEquals("6.0.1", readInfo.getHosts().iterator().next().version().get().toString()); + // TODO: Enable when dockerImageRepo is written to zk + //assertEquals("docker repo", readInfo.getHosts().iterator().next().dockerImageRepo().get()); assertTrue(zkApp.getDeployment().isPresent()); assertEquals("mydisc", DeploymentSpec.fromXml(zkApp.getDeployment().get()).requireInstance("default").globalServiceId().get()); } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/DeploymentData.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/DeploymentData.java index 43d702d108f..ee2e292ade9 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/DeploymentData.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/DeploymentData.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.hosted.controller.api.application.v4.model; import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.DockerImage; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateMetadata; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ContainerEndpoint; @@ -24,16 +25,19 @@ public class DeploymentData { private final Version platform; private final Set<ContainerEndpoint> containerEndpoints; private final Optional<EndpointCertificateMetadata> endpointCertificateMetadata; + private final Optional<DockerImage> dockerImageRepo; public DeploymentData(ApplicationId instance, ZoneId zone, byte[] applicationPackage, Version platform, Set<ContainerEndpoint> containerEndpoints, - Optional<EndpointCertificateMetadata> endpointCertificateMetadata) { + Optional<EndpointCertificateMetadata> endpointCertificateMetadata, + Optional<DockerImage> dockerImageRepo) { this.instance = requireNonNull(instance); this.zone = requireNonNull(zone); this.applicationPackage = requireNonNull(applicationPackage); this.platform = requireNonNull(platform); this.containerEndpoints = requireNonNull(containerEndpoints); this.endpointCertificateMetadata = requireNonNull(endpointCertificateMetadata); + this.dockerImageRepo = requireNonNull(dockerImageRepo); } public ApplicationId instance() { @@ -60,4 +64,7 @@ public class DeploymentData { return endpointCertificateMetadata; } + public Optional<DockerImage> dockerImageRepo() { + return dockerImageRepo; + } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java index a382d357e49..cb6d02b4b77 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java @@ -6,6 +6,7 @@ import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.application.api.ValidationId; import com.yahoo.config.application.api.ValidationOverrides; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.DockerImage; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.TenantName; @@ -17,7 +18,10 @@ import com.yahoo.vespa.athenz.api.AthenzPrincipal; import com.yahoo.vespa.athenz.api.AthenzService; import com.yahoo.vespa.athenz.api.AthenzUser; import com.yahoo.vespa.curator.Lock; +import com.yahoo.vespa.flags.FetchVector; import com.yahoo.vespa.flags.FlagSource; +import com.yahoo.vespa.flags.Flags; +import com.yahoo.vespa.flags.StringFlag; import com.yahoo.vespa.hosted.controller.api.ActivateResult; import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeployOptions; import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeploymentData; @@ -108,6 +112,7 @@ public class ApplicationController { private final DeploymentTrigger deploymentTrigger; private final ApplicationPackageValidator applicationPackageValidator; private final EndpointCertificateManager endpointCertificateManager; + private final StringFlag dockerImageRepoFlag; ApplicationController(Controller controller, CuratorDb curator, AccessControl accessControl, Clock clock, SecretStore secretStore, FlagSource flagSource) { @@ -119,6 +124,7 @@ public class ApplicationController { this.clock = clock; this.artifactRepository = controller.serviceRegistry().artifactRepository(); this.applicationStore = controller.serviceRegistry().applicationStore(); + this.dockerImageRepoFlag = Flags.DOCKER_IMAGE_REPO.bindTo(flagSource); deploymentTrigger = new DeploymentTrigger(controller, clock); applicationPackageValidator = new ApplicationPackageValidator(controller); @@ -485,9 +491,16 @@ public class ApplicationController { ZoneId zone, Version platform, Set<ContainerEndpoint> endpoints, Optional<EndpointCertificateMetadata> endpointCertificateMetadata) { try { + Optional<DockerImage> dockerImageRepo = Optional.ofNullable( + dockerImageRepoFlag + .with(FetchVector.Dimension.ZONE_ID, zone.value()) + .with(FetchVector.Dimension.APPLICATION_ID, application.serializedForm()) + .value()) + .filter(s -> !s.isBlank()) + .map(DockerImage::fromString); ConfigServer.PreparedApplication preparedApplication = configServer.deploy(new DeploymentData(application, zone, applicationPackage.zippedContent(), platform, - endpoints, endpointCertificateMetadata)); + endpoints, endpointCertificateMetadata, dockerImageRepo)); return new ActivateResult(new RevisionId(applicationPackage.hash()), preparedApplication.prepareResponse(), applicationPackage.zippedContent().length); } finally { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java index 1730f033d61..df62c501cc6 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java @@ -65,6 +65,7 @@ public class RoutingController { private final RoutingPolicies routingPolicies; private final RotationRepository rotationRepository; private final BooleanFlag allowDirectRouting; + private final BooleanFlag disableRoutingGenerator; public RoutingController(Controller controller, RotationsConfig rotationsConfig) { this.controller = Objects.requireNonNull(controller, "controller must be non-null"); @@ -72,6 +73,7 @@ public class RoutingController { this.rotationRepository = new RotationRepository(rotationsConfig, controller.applications(), controller.curator()); this.allowDirectRouting = Flags.ALLOW_DIRECT_ROUTING.bindTo(controller.flagSource()); + this.disableRoutingGenerator = Flags.DISABLE_ROUTING_GENERATOR.bindTo(controller.flagSource()); } public RoutingPolicies policies() { @@ -87,12 +89,17 @@ public class RoutingController { var endpoints = new LinkedHashSet<Endpoint>(); // TODO(mpolden): Remove this once all applications have deployed once and config server passes correct cluster // id for combined cluster type - controller.serviceRegistry().routingGenerator().clusterEndpoints(deployment) - .forEach((cluster, url) -> endpoints.add(Endpoint.of(deployment.applicationId()) - .target(cluster, deployment.zoneId()) - .routingMethod(RoutingMethod.shared) - .on(Port.fromRoutingMethod(RoutingMethod.shared)) - .in(controller.system()))); + var disableRoutingGenerator = this.disableRoutingGenerator.with(FetchVector.Dimension.APPLICATION_ID, + deployment.applicationId().serializedForm()) + .value(); + if (!disableRoutingGenerator) { + controller.serviceRegistry().routingGenerator().clusterEndpoints(deployment) + .forEach((cluster, url) -> endpoints.add(Endpoint.of(deployment.applicationId()) + .target(cluster, deployment.zoneId()) + .routingMethod(RoutingMethod.shared) + .on(Port.fromRoutingMethod(RoutingMethod.shared)) + .in(controller.system()))); + } boolean hasSharedEndpoint = !endpoints.isEmpty(); // Avoid reading application more than once per call to this var application = Suppliers.memoize(() -> controller.applications().requireApplication(TenantAndApplicationId.from(deployment.applicationId()))); diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index a17b4aa76c3..14ac9ecd678 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -233,6 +233,12 @@ public class Flags { "Takes effect at redeployment", APPLICATION_ID); + public static final UnboundStringFlag DOCKER_IMAGE_REPO = defineStringFlag( + "docker-image-repo", "", + "Override default docker image repo. Docker image version will be Vespa version.", + "Takes effect on next deployment from controller", + ZONE_ID, APPLICATION_ID); + public static final UnboundStringFlag DOCKER_IMAGE_OVERRIDE = defineStringFlag( "docker-image-override", "", "Override the Docker image to use for deployments. This must containing the image name only, without tag", @@ -261,6 +267,12 @@ public class Flags { "Takes effect immediately", APPLICATION_ID); + public static final UnboundBooleanFlag DISABLE_ROUTING_GENERATOR = defineFeatureFlag( + "disable-routing-generator", false, + "Whether the controller should stop asking the routing layer for endpoints", + "Takes effect immediately", + APPLICATION_ID); + /** WARNING: public for testing: All flags should be defined in {@link Flags}. */ public static UnboundBooleanFlag defineFeatureFlag(String flagId, boolean defaultValue, String description, String modificationEffect, FetchVector.Dimension... dimensions) { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java index 0632d85906a..8ddb85e9505 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java @@ -67,7 +67,6 @@ public class Autoscaler { if (clusterNodes.stream().anyMatch(node -> node.status().wantToRetire() || node.allocation().get().membership().retired() || node.allocation().get().isRemovable())) { - log.fine("Autoscaling " + applicationId + " " + cluster + ": Cluster is in flux"); return Optional.empty(); // Don't autoscale clusters that are in flux } AllocatableClusterResources currentAllocation = new AllocatableClusterResources(clusterNodes, resourcesCalculator); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MetricsResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MetricsResponse.java index 653b786ebe5..87551a5bd5f 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MetricsResponse.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MetricsResponse.java @@ -44,17 +44,17 @@ public class MetricsResponse { } private void consumeNodeMetrics(String hostname, Inspector node) { - long timestamp = node.field("timestamp").asLong(); + long timestampSecond = node.field("timestamp").asLong(); Map<String, Double> values = consumeMetrics(node.field("metrics")); for (Resource resource : Resource.values()) - addMetricIfPresent(hostname, resource, timestamp, values); + addMetricIfPresent(hostname, resource, timestampSecond, values); } - private void addMetricIfPresent(String hostname, Resource resource, long timestamp, Map<String, Double> values) { + private void addMetricIfPresent(String hostname, Resource resource, long timestampSecond, Map<String, Double> values) { if (values.containsKey(resource.metricName())) metricValues.add(new NodeMetrics.MetricValue(hostname, resource.metricName(), - timestamp, + timestampSecond, values.get(resource.metricName()))); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetrics.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetrics.java index 8abc2327d88..b0d73833bc6 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetrics.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetrics.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.hosted.provision.autoscale; import com.yahoo.config.provision.ApplicationId; +import java.time.Instant; import java.util.Collection; /** @@ -23,24 +24,24 @@ public interface NodeMetrics { private final String hostname; private final String name; - private long timestamp; + private long timestampSecond; private final double value; - public MetricValue(String hostname, String name, long timestamp, double value) { + public MetricValue(String hostname, String name, long timestampSecond, double value) { this.hostname = hostname; this.name = name; - this.timestamp = timestamp; + this.timestampSecond = timestampSecond; this.value = value; } public String hostname() { return hostname; } public String name() { return name; } - public long timestamp() { return timestamp; } + public long timestampSecond() { return timestampSecond; } public double value() { return value; } @Override public String toString() { - return "metric value " + name + ": " + value + " at " + timestamp + " for " + hostname; + return "metric value " + name + ": " + value + " at " + Instant.ofEpochSecond(timestampSecond) + " for " + hostname; } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetricsDb.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetricsDb.java index aa20752d8a7..c86546a7790 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetricsDb.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetricsDb.java @@ -11,6 +11,7 @@ import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Set; import java.util.logging.Logger; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -34,21 +35,19 @@ public class NodeMetricsDb { /** Add a measurement to this */ public void add(Collection<NodeMetrics.MetricValue> metricValues) { - log.fine("Adding " + metricValues.size() + " metric values" + - (metricValues.size() > 0 ? ". First: " + metricValues.iterator().next() : "")); synchronized (lock) { for (var value : metricValues) { Resource resource = Resource.fromMetric(value.name()); List<Measurement> measurements = db.computeIfAbsent(new MeasurementKey(value.hostname(), resource), (__) -> new ArrayList<>()); - measurements.add(new Measurement(value.timestamp(), (float)resource.valueFromMetric(value.value()))); + measurements.add(new Measurement(value.timestampSecond() * 1000, + (float)resource.valueFromMetric(value.value()))); } } } /** Must be called intermittently (as long as add is called) to gc old measurements */ public void gc(Clock clock) { - int gcCount = 0; synchronized (lock) { // TODO: We may need to do something more complicated to avoid spending too much memory to // lower the measurement interval (see NodeRepositoryMaintenance) @@ -58,16 +57,13 @@ public class NodeMetricsDb { long oldestTimestamp = clock.instant().minus(dbWindow).toEpochMilli(); for (Iterator<List<Measurement>> i = db.values().iterator(); i.hasNext(); ) { List<Measurement> measurements = i.next(); - while (!measurements.isEmpty() && measurements.get(0).timestamp < oldestTimestamp) { + while (!measurements.isEmpty() && measurements.get(0).timestamp < oldestTimestamp) measurements.remove(0); - gcCount++; - } if (measurements.isEmpty()) i.remove(); } } - log.fine("Gc'ed " + gcCount + " metric values"); } /** Returns a window within which we can ask for specific information from this db */ @@ -87,9 +83,6 @@ public class NodeMetricsDb { public int measurementCount() { synchronized (lock) { - List<MeasurementKey> matches = keys.stream().filter(key -> db.get(key) != null).collect(Collectors.toList()); - List<MeasurementKey> nonMatches = keys.stream().filter(key -> db.get(key) == null).collect(Collectors.toList()); - log.fine("Counting measurements after " + startTime + ". Matches: " + matches + ". Non-matches: " + nonMatches); return (int) keys.stream() .flatMap(key -> db.getOrDefault(key, List.of()).stream()) .filter(measurement -> measurement.timestamp >= startTime) diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java index 5546f88ff47..be43c0139e0 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java @@ -103,6 +103,7 @@ public class NodeSerializer { private static final String removableKey = "removable"; // Saved as part of allocation instead of serviceId, since serviceId serialized form is not easily extendable. private static final String wantedVespaVersionKey = "wantedVespaVersion"; + private static final String wantedDockerImageRepoKey = "wantedDockerImageRepo"; // History event fields private static final String historyEventTypeKey = "type"; @@ -186,6 +187,8 @@ public class NodeSerializer { object.setLong(currentRestartGenerationKey, allocation.restartGeneration().current()); object.setBool(removableKey, allocation.isRemovable()); object.setString(wantedVespaVersionKey, allocation.membership().cluster().vespaVersion().toString()); + // TODO serialize dockerImageRepo + //object.setString(wantedDockerImageRepoKey, allocation.membership().cluster().dockerImageRepo().orElse("")); allocation.networkPorts().ifPresent(ports -> NetworkPortsSerializer.toSlime(ports, object.setArray(networkPortsKey))); } @@ -306,7 +309,8 @@ public class NodeSerializer { private ClusterMembership clusterMembershipFromSlime(Inspector object) { return ClusterMembership.from(object.field(serviceIdKey).asString(), - versionFromSlime(object.field(wantedVespaVersionKey)).get()); + versionFromSlime(object.field(wantedVespaVersionKey)).get(), + dockerImageRepoFromSlime(object.field(wantedDockerImageRepoKey))); } private Optional<Version> versionFromSlime(Inspector object) { @@ -314,6 +318,11 @@ public class NodeSerializer { return Optional.of(Version.fromString(object.asString())); } + private Optional<String> dockerImageRepoFromSlime(Inspector object) { + if ( ! object.valid() || object.asString().isEmpty()) return Optional.empty(); + return Optional.of(object.asString()); + } + private Optional<DockerImage> dockerImageFromSlime(Inspector object) { if ( ! object.valid()) return Optional.empty(); return Optional.of(DockerImage.fromString(object.asString())); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java index 3a6d5aa6cbe..c923940570a 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java @@ -141,28 +141,28 @@ public class MockNodeRepository extends NodeRepository { ClusterSpec zoneCluster = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("node-admin"), Version.fromString("6.42"), - false, Optional.empty()); + false, Optional.empty(), Optional.empty()); activate(provisioner.prepare(zoneApp, zoneCluster, Capacity.fromRequiredNodeType(NodeType.host), 1, null), zoneApp, provisioner); ApplicationId app1 = ApplicationId.from(TenantName.from("tenant1"), ApplicationName.from("application1"), InstanceName.from("instance1")); ClusterSpec cluster1 = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("id1"), Version.fromString("6.42"), - false, Optional.empty()); + false, Optional.empty(), Optional.empty()); provisioner.prepare(app1, cluster1, Capacity.fromCount(2, new NodeResources(2, 8, 50, 1)), 1, null); ApplicationId app2 = ApplicationId.from(TenantName.from("tenant2"), ApplicationName.from("application2"), InstanceName.from("instance2")); ClusterSpec cluster2 = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("id2"), Version.fromString("6.42"), - false, Optional.empty()); + false, Optional.empty(), Optional.empty()); activate(provisioner.prepare(app2, cluster2, Capacity.fromCount(2, new NodeResources(2, 8, 50, 1)), 1, null), app2, provisioner); ApplicationId app3 = ApplicationId.from(TenantName.from("tenant3"), ApplicationName.from("application3"), InstanceName.from("instance3")); ClusterSpec cluster3 = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("id3"), Version.fromString("6.42"), - false, Optional.empty()); + false, Optional.empty(), Optional.empty()); activate(provisioner.prepare(app3, cluster3, Capacity.fromCount(2, new NodeResources(1, 4, 100, 1), false, true), 1, null), app3, provisioner); List<Node> largeNodes = new ArrayList<>(); @@ -176,7 +176,7 @@ public class MockNodeRepository extends NodeRepository { ClusterSpec cluster4 = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("id4"), Version.fromString("6.42"), - false, Optional.empty()); + false, Optional.empty(), Optional.empty()); activate(provisioner.prepare(app4, cluster4, Capacity.fromCount(2, new NodeResources(10, 48, 500, 1), false, true), 1, null), app4, provisioner); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java index f9dff774a82..5a2ba674d99 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java @@ -1,6 +1,7 @@ // Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.autoscale; +import com.google.common.collect.Sets; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.CloudName; import com.yahoo.config.provision.ClusterSpec; @@ -10,11 +11,14 @@ import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.Zone; +import com.yahoo.io.IOUtils; import org.junit.Test; import java.util.ArrayList; +import java.util.HashSet; import java.util.List; import java.util.Optional; +import java.util.Set; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTester.java index 015bf60113a..b83addf7bdb 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTester.java @@ -79,6 +79,7 @@ class AutoscalingTester { ClusterSpec.Id.from(clusterId), Version.fromString("7"), false, + Optional.empty(), Optional.empty()); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetricsDbTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetricsDbTest.java index 519235857f1..af87c008260 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetricsDbTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetricsDbTest.java @@ -18,15 +18,15 @@ public class NodeMetricsDbTest { NodeMetricsDb db = new NodeMetricsDb(); List<NodeMetrics.MetricValue> values = new ArrayList<>(); for (int i = 0; i < 40; i++) { - values.add(new NodeMetrics.MetricValue("host0", "cpu.util", clock.instant().toEpochMilli(), 0.9f)); + values.add(new NodeMetrics.MetricValue("host0", "cpu.util", clock.instant().getEpochSecond(), 0.9f)); clock.advance(Duration.ofHours(1)); } db.add(values); - assertEquals(30, db.getWindow(clock.instant().minus(Duration.ofHours(30)), Resource.cpu, List.of("host0")).measurementCount()); + assertEquals(29, db.getWindow(clock.instant().minus(Duration.ofHours(30)), Resource.cpu, List.of("host0")).measurementCount()); assertEquals( 0, db.getWindow(clock.instant().minus(Duration.ofHours(30)), Resource.memory, List.of("host0")).measurementCount()); db.gc(clock); - assertEquals(24, db.getWindow(clock.instant().minus(Duration.ofHours(30)), Resource.cpu, List.of("host0")).measurementCount()); + assertEquals(23, db.getWindow(clock.instant().minus(Duration.ofHours(30)), Resource.cpu, List.of("host0")).measurementCount()); assertEquals( 0, db.getWindow(clock.instant().minus(Duration.ofHours(30)), Resource.memory, List.of("host0")).measurementCount()); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetricsFetcherTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetricsFetcherTest.java index 4f039582125..da4f74bf05b 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetricsFetcherTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetricsFetcherTest.java @@ -40,11 +40,11 @@ public class NodeMetricsFetcherTest { assertEquals("http://host-1.yahoo.com:4080/metrics/v2/values?consumer=vespa-consumer-metrics", httpClient.requestsReceived.get(0)); assertEquals(5, values.size()); - assertEquals("metric value cpu.util: 16.2 at 1234 for host-1.yahoo.com", values.get(0).toString()); - assertEquals("metric value mem.util: 23.1 at 1234 for host-1.yahoo.com", values.get(1).toString()); - assertEquals("metric value disk.util: 82.0 at 1234 for host-1.yahoo.com", values.get(2).toString()); - assertEquals("metric value cpu.util: 20.0 at 1200 for host-2.yahoo.com", values.get(3).toString()); - assertEquals("metric value disk.util: 40.0 at 1200 for host-2.yahoo.com", values.get(4).toString()); + assertEquals("metric value cpu.util: 16.2 at 1970-01-01T00:20:34Z for host-1.yahoo.com", values.get(0).toString()); + assertEquals("metric value mem.util: 23.1 at 1970-01-01T00:20:34Z for host-1.yahoo.com", values.get(1).toString()); + assertEquals("metric value disk.util: 82.0 at 1970-01-01T00:20:34Z for host-1.yahoo.com", values.get(2).toString()); + assertEquals("metric value cpu.util: 20.0 at 1970-01-01T00:20:00Z for host-2.yahoo.com", values.get(3).toString()); + assertEquals("metric value disk.util: 40.0 at 1970-01-01T00:20:00Z for host-2.yahoo.com", values.get(4).toString()); } { @@ -53,9 +53,9 @@ public class NodeMetricsFetcherTest { assertEquals("http://host-3.yahoo.com:4080/metrics/v2/values?consumer=vespa-consumer-metrics", httpClient.requestsReceived.get(1)); assertEquals(3, values.size()); - assertEquals("metric value cpu.util: 10.0 at 1300 for host-3.yahoo.com", values.get(0).toString()); - assertEquals("metric value mem.util: 15.0 at 1300 for host-3.yahoo.com", values.get(1).toString()); - assertEquals("metric value disk.util: 20.0 at 1300 for host-3.yahoo.com", values.get(2).toString()); + assertEquals("metric value cpu.util: 10.0 at 1970-01-01T00:21:40Z for host-3.yahoo.com", values.get(0).toString()); + assertEquals("metric value mem.util: 15.0 at 1970-01-01T00:21:40Z for host-3.yahoo.com", values.get(1).toString()); + assertEquals("metric value disk.util: 20.0 at 1970-01-01T00:21:40Z for host-3.yahoo.com", values.get(2).toString()); } } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityCheckerTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityCheckerTester.java index 5e8d22275af..3fef302f334 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityCheckerTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityCheckerTester.java @@ -254,7 +254,8 @@ public class CapacityCheckerTester { if (nodeModel.membership != null && nodeModel.owner != null) { membership = ClusterMembership.from( nodeModel.membership.toString(), - Version.fromString(nodeModel.wantedVespaVersion)); + Version.fromString(nodeModel.wantedVespaVersion), + Optional.empty()); owner = ApplicationId.from(nodeModel.owner.tenant, nodeModel.owner.application, nodeModel.owner.instance); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainerTest.java index 4aa73e0bd31..1c65b9124c0 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainerTest.java @@ -222,7 +222,7 @@ public class DynamicProvisioningMaintainerTest { Optional<Allocation> allocation = application .map(app -> new Allocation( app, - ClusterMembership.from("container/default/0/0", Version.fromString("7.3")), + ClusterMembership.from("container/default/0/0", Version.fromString("7.3"), Optional.empty()), flavor.resources(), Generation.initial(), false)); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirerTest.java index ec604ab1387..b0374d9213f 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirerTest.java @@ -57,7 +57,7 @@ public class FailedExpirerTest { ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("node-admin"), Version.fromString("6.42"), - false, Optional.empty()); + false, Optional.empty(), Optional.empty()); private static final Capacity tenantHostApplicationCapacity = Capacity.fromRequiredNodeType(NodeType.host); @@ -144,7 +144,7 @@ public class FailedExpirerTest { .withNode(NodeType.proxy, FailureScenario.defaultFlavor, "proxy3") .setReady("proxy1", "proxy2", "proxy3") .allocate( ApplicationId.from("vespa", "zone-app", "default"), - ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("routing"), Version.fromString("6.42"), false, Optional.empty()), + ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("routing"), Version.fromString("6.42"), false, Optional.empty(), Optional.empty()), Capacity.fromRequiredNodeType(NodeType.proxy)) .failNode(1, "proxy1"); @@ -329,6 +329,7 @@ public class FailedExpirerTest { ClusterSpec.Id.from("test"), Version.fromString("6.42"), false, + Optional.empty(), Optional.empty()); Capacity capacity = Capacity.fromCount(hostname.length, Optional.of(flavor), false, true); return allocate(applicationId, clusterSpec, capacity); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/InactiveAndFailedExpirerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/InactiveAndFailedExpirerTest.java index b617a2dcf85..d63e401cd97 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/InactiveAndFailedExpirerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/InactiveAndFailedExpirerTest.java @@ -55,7 +55,7 @@ public class InactiveAndFailedExpirerTest { List<Node> nodes = tester.makeReadyNodes(2, nodeResources); // Allocate then deallocate 2 nodes - ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("test"), Version.fromString("6.42"), false, Optional.empty()); + ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("test"), Version.fromString("6.42"), false, Optional.empty(), Optional.empty()); List<HostSpec> preparedNodes = tester.prepare(applicationId, cluster, Capacity.fromCount(2, nodeResources), 1); tester.activate(applicationId, new HashSet<>(preparedNodes)); assertEquals(2, tester.getNodes(applicationId, Node.State.active).size()); @@ -96,7 +96,7 @@ public class InactiveAndFailedExpirerTest { ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("test"), Version.fromString("6.42"), - false, Optional.empty()); + false, Optional.empty(), Optional.empty()); List<HostSpec> preparedNodes = tester.prepare(applicationId, cluster, Capacity.fromCount(2, nodeResources), 1); tester.activate(applicationId, new HashSet<>(preparedNodes)); assertEquals(2, tester.getNodes(applicationId, Node.State.active).size()); @@ -123,7 +123,7 @@ public class InactiveAndFailedExpirerTest { public void node_that_wants_to_retire_is_moved_to_parked() throws OrchestrationException { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).build(); ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("test"), - Version.fromString("6.42"), false, Optional.empty()); + Version.fromString("6.42"), false, Optional.empty(), Optional.empty()); tester.makeReadyNodes(5, nodeResources); // Allocate two nodes @@ -179,7 +179,7 @@ public class InactiveAndFailedExpirerTest { // Allocate then deallocate a node ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).build(); tester.makeReadyNodes(1, nodeResources); - ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("test"), Version.fromString("6.42"), false, Optional.empty()); + ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("test"), Version.fromString("6.42"), false, Optional.empty(), Optional.empty()); List<HostSpec> preparedNodes = tester.prepare(testerId, cluster, Capacity.fromCount(2, nodeResources), 1); tester.activate(testerId, new HashSet<>(preparedNodes)); assertEquals(1, tester.getNodes(testerId, Node.State.active).size()); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirerTest.java index a3c95e1a825..e2b068994ce 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirerTest.java @@ -160,7 +160,7 @@ public class LoadBalancerExpirerTest { List<HostSpec> hosts = new ArrayList<>(); for (var cluster : clusters) { hosts.addAll(tester.prepare(application, ClusterSpec.request(ClusterSpec.Type.container, cluster, - Vtag.currentVersion, false, Optional.empty()), + Vtag.currentVersion, false, Optional.empty(), Optional.empty()), 2, 1, new NodeResources(1, 4, 10, 0.3))); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java index dc7ae74ec6e..d6d5816dc1d 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java @@ -211,7 +211,7 @@ public class MetricsReporterTest { private Optional<Allocation> allocation(Optional<String> tenant, Node owner) { if (tenant.isPresent()) { Allocation allocation = new Allocation(app(tenant.get()), - ClusterMembership.from("container/id1/0/3", new Version()), + ClusterMembership.from("container/id1/0/3", new Version(), Optional.empty()), owner.flavor().resources(), Generation.initial(), false); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailTester.java index 35a93ebe8d7..d1bb3fe1949 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailTester.java @@ -89,8 +89,8 @@ public class NodeFailTester { tester.createHostNodes(3); // Create applications - ClusterSpec clusterApp1 = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("test"), Version.fromString("6.42"), false, Optional.empty()); - ClusterSpec clusterApp2 = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("test"), Version.fromString("6.42"), false, Optional.empty()); + ClusterSpec clusterApp1 = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("test"), Version.fromString("6.42"), false, Optional.empty(), Optional.empty()); + ClusterSpec clusterApp2 = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("test"), Version.fromString("6.42"), false, Optional.empty(), Optional.empty()); Capacity capacity1 = Capacity.fromCount(5, nodeResources, false, true); Capacity capacity2 = Capacity.fromCount(7, nodeResources, false, true); @@ -120,9 +120,9 @@ public class NodeFailTester { } // Create applications - ClusterSpec clusterNodeAdminApp = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("node-admin"), Version.fromString("6.42"), false, Optional.empty()); - ClusterSpec clusterApp1 = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("test"), Version.fromString("6.75.0"), false, Optional.empty()); - ClusterSpec clusterApp2 = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("test"), Version.fromString("6.75.0"), false, Optional.empty()); + ClusterSpec clusterNodeAdminApp = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("node-admin"), Version.fromString("6.42"), false, Optional.empty(), Optional.empty()); + ClusterSpec clusterApp1 = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("test"), Version.fromString("6.75.0"), false, Optional.empty(), Optional.empty()); + ClusterSpec clusterApp2 = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("test"), Version.fromString("6.75.0"), false, Optional.empty(), Optional.empty()); Capacity allHosts = Capacity.fromRequiredNodeType(NodeType.host); Capacity capacity1 = Capacity.fromCount(3, new NodeResources(1, 4, 10, 0.3), false, true); Capacity capacity2 = Capacity.fromCount(5, new NodeResources(1, 4, 10, 0.3), false, true); @@ -154,7 +154,7 @@ public class NodeFailTester { ClusterSpec clusterApp1 = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("test"), Version.fromString("6.42"), - false, Optional.empty()); + false, Optional.empty(), Optional.empty()); tester.activate(app1, clusterApp1, allNodes); assertEquals(count, tester.nodeRepository.getNodes(nodeType, Node.State.active).size()); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/OperatorChangeApplicationMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/OperatorChangeApplicationMaintainerTest.java index d833eab9eb0..8cd34402b9d 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/OperatorChangeApplicationMaintainerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/OperatorChangeApplicationMaintainerTest.java @@ -134,9 +134,9 @@ public class OperatorChangeApplicationMaintainerTest { final ApplicationId app1 = ApplicationId.from(TenantName.from("foo1"), ApplicationName.from("bar"), InstanceName.from("fuz")); final ApplicationId app2 = ApplicationId.from(TenantName.from("foo2"), ApplicationName.from("bar"), InstanceName.from("fuz")); final ApplicationId app3 = ApplicationId.from(TenantName.from("vespa-hosted"), ApplicationName.from("routing"), InstanceName.from("default")); - final ClusterSpec clusterApp1 = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("test"), Version.fromString("6.42"), false, Optional.empty()); - final ClusterSpec clusterApp2 = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("test"), Version.fromString("6.42"), false, Optional.empty()); - final ClusterSpec clusterApp3 = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("routing"), Version.fromString("6.42"), false, Optional.empty()); + final ClusterSpec clusterApp1 = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("test"), Version.fromString("6.42"), false, Optional.empty(), Optional.empty()); + final ClusterSpec clusterApp2 = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("test"), Version.fromString("6.42"), false, Optional.empty(), Optional.empty()); + final ClusterSpec clusterApp3 = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("routing"), Version.fromString("6.42"), false, Optional.empty(), Optional.empty()); final int wantedNodesApp1 = 5; final int wantedNodesApp2 = 7; final int wantedNodesApp3 = 2; diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/PeriodicApplicationMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/PeriodicApplicationMaintainerTest.java index 71a2b6b7e4c..5a6abaac2d5 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/PeriodicApplicationMaintainerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/PeriodicApplicationMaintainerTest.java @@ -244,8 +244,8 @@ public class PeriodicApplicationMaintainerTest { final NodeResources nodeResources = new NodeResources(2, 8, 50, 1); final ApplicationId app1 = ApplicationId.from(TenantName.from("foo1"), ApplicationName.from("bar"), InstanceName.from("fuz")); final ApplicationId app2 = ApplicationId.from(TenantName.from("foo2"), ApplicationName.from("bar"), InstanceName.from("fuz")); - final ClusterSpec clusterApp1 = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("test"), Version.fromString("6.42"), false, Optional.empty()); - final ClusterSpec clusterApp2 = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("test"), Version.fromString("6.42"), false, Optional.empty()); + final ClusterSpec clusterApp1 = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("test"), Version.fromString("6.42"), false, Optional.empty(), Optional.empty()); + final ClusterSpec clusterApp2 = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("test"), Version.fromString("6.42"), false, Optional.empty(), Optional.empty()); final int wantedNodesApp1 = 5; final int wantedNodesApp2 = 7; diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RebalancerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RebalancerTest.java index d5b7a5b09c6..ce4e62b5f7b 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RebalancerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RebalancerTest.java @@ -129,7 +129,7 @@ public class RebalancerTest { } private ClusterSpec clusterSpec(String clusterId) { - return ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from(clusterId), Version.fromString("6.42"), false, Optional.empty()); + return ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from(clusterId), Version.fromString("6.42"), false, Optional.empty(), Optional.empty()); } private ApplicationId makeApplicationId(String tenant, String appName) { diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ReservationExpirerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ReservationExpirerTest.java index 64be8b45f67..0f26fcaae6b 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ReservationExpirerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ReservationExpirerTest.java @@ -61,7 +61,7 @@ public class ReservationExpirerTest { assertEquals(2, nodeRepository.getNodes(NodeType.tenant, Node.State.dirty).size()); nodeRepository.setReady(nodes, Agent.system, getClass().getSimpleName()); ApplicationId applicationId = new ApplicationId.Builder().tenant("foo").applicationName("bar").instanceName("fuz").build(); - ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("test"), Version.fromString("6.42"), false, Optional.empty()); + ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("test"), Version.fromString("6.42"), false, Optional.empty(), Optional.empty()); provisioner.prepare(applicationId, cluster, Capacity.fromCount(2, new NodeResources(2, 8, 50, 1)), 1, null); assertEquals(2, nodeRepository.getNodes(NodeType.tenant, Node.State.reserved).size()); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirerTest.java index c6b74d5eae2..41dcc238a47 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirerTest.java @@ -86,7 +86,7 @@ public class RetiredExpirerTest { // Allocate content cluster of sizes 7 -> 2 -> 3: // Should end up with 3 nodes in the cluster (one previously retired), and 4 retired - ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("test"), Version.fromString("6.42"), false, Optional.empty()); + ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("test"), Version.fromString("6.42"), false, Optional.empty(), Optional.empty()); int wantedNodes; activate(applicationId, cluster, wantedNodes=7, 1, provisioner); activate(applicationId, cluster, wantedNodes=2, 1, provisioner); @@ -117,7 +117,7 @@ public class RetiredExpirerTest { ApplicationId applicationId = ApplicationId.from(TenantName.from("foo"), ApplicationName.from("bar"), InstanceName.from("fuz")); - ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("test"), Version.fromString("6.42"), false, Optional.empty()); + ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("test"), Version.fromString("6.42"), false, Optional.empty(), Optional.empty()); activate(applicationId, cluster, 8, 8, provisioner); activate(applicationId, cluster, 2, 2, provisioner); assertEquals(8, nodeRepository.getNodes(applicationId, Node.State.active).size()); @@ -148,7 +148,7 @@ public class RetiredExpirerTest { // Allocate content cluster of sizes 7 -> 2 -> 3: // Should end up with 3 nodes in the cluster (one previously retired), and 4 retired - ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("test"), Version.fromString("6.42"), false, Optional.empty()); + ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("test"), Version.fromString("6.42"), false, Optional.empty(), Optional.empty()); int wantedNodes; activate(applicationId, cluster, wantedNodes=7, 1, provisioner); activate(applicationId, cluster, wantedNodes=2, 1, provisioner); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/SerializationTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/SerializationTest.java index 8e048001b98..21f25dcede4 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/SerializationTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/SerializationTest.java @@ -78,7 +78,7 @@ public class SerializationTest { node = node.allocate(ApplicationId.from(TenantName.from("myTenant"), ApplicationName.from("myApplication"), InstanceName.from("myInstance")), - ClusterMembership.from("content/myId/0/0", Vtag.currentVersion), + ClusterMembership.from("content/myId/0/0", Vtag.currentVersion, Optional.empty()), requestedResources, clock.instant()); assertEquals(1, node.history().events().size()); @@ -166,7 +166,7 @@ public class SerializationTest { node = node.allocate(ApplicationId.from(TenantName.from("myTenant"), ApplicationName.from("myApplication"), InstanceName.from("myInstance")), - ClusterMembership.from("content/myId/0/0", Vtag.currentVersion), + ClusterMembership.from("content/myId/0/0", Vtag.currentVersion, Optional.empty()), node.flavor().resources(), clock.instant()); assertEquals(1, node.history().events().size()); @@ -216,7 +216,7 @@ public class SerializationTest { node = node.allocate(ApplicationId.from(TenantName.from("myTenant"), ApplicationName.from("myApplication"), InstanceName.from("myInstance")), - ClusterMembership.from("content/myId/0/0", Vtag.currentVersion), + ClusterMembership.from("content/myId/0/0", Vtag.currentVersion, Optional.empty()), node.flavor().resources(), clock.instant()); @@ -401,7 +401,7 @@ public class SerializationTest { node = node.allocate(ApplicationId.from(TenantName.from("myTenant"), ApplicationName.from("myApplication"), InstanceName.from("myInstance")), - ClusterMembership.from("content/myId/0/0", Vtag.currentVersion), + ClusterMembership.from("content/myId/0/0", Vtag.currentVersion, Optional.empty()), node.flavor().resources(), clock.instant()); assertTrue(node.allocation().isPresent()); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AllocationSimulator.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AllocationSimulator.java index 68c68a849b1..fb656532fda 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AllocationSimulator.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AllocationSimulator.java @@ -93,7 +93,7 @@ public class AllocationSimulator { private Optional<Allocation> allocation(Optional<String> tenant, Flavor flavor) { if (tenant.isPresent()) { Allocation allocation = new Allocation(app(tenant.get()), - ClusterMembership.from("container/id1/3", new Version()), + ClusterMembership.from("container/id1/3", new Version(), Optional.empty()), flavor.resources(), Generation.initial(), false); @@ -111,7 +111,7 @@ public class AllocationSimulator { private ClusterSpec cluster() { return ClusterSpec.from(ClusterSpec.Type.container, ClusterSpec.Id.from("test"), ClusterSpec.Group.from(1), - Version.fromString("6.41"), false, Optional.empty()); + Version.fromString("6.41"), false, Optional.empty(), Optional.empty()); } /* ------------ Methods to add events to the system ----------------*/ diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerProvisioningTest.java index cb2e3f65c40..43288999e16 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerProvisioningTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerProvisioningTest.java @@ -52,7 +52,7 @@ public class DockerProvisioningTest { Version wantedVespaVersion = Version.fromString("6.39"); int nodeCount = 7; List<HostSpec> hosts = tester.prepare(application1, - ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("myContent"), wantedVespaVersion, false, Optional.empty()), + ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("myContent"), wantedVespaVersion, false, Optional.empty(), Optional.empty()), nodeCount, 1, dockerFlavor); tester.activate(application1, new HashSet<>(hosts)); @@ -63,7 +63,7 @@ public class DockerProvisioningTest { // Upgrade Vespa version on nodes Version upgradedWantedVespaVersion = Version.fromString("6.40"); List<HostSpec> upgradedHosts = tester.prepare(application1, - ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("myContent"), upgradedWantedVespaVersion, false, Optional.empty()), + ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("myContent"), upgradedWantedVespaVersion, false, Optional.empty(), Optional.empty()), nodeCount, 1, dockerFlavor); tester.activate(application1, new HashSet<>(upgradedHosts)); NodeList upgradedNodes = tester.getNodes(application1, Node.State.active); @@ -85,7 +85,7 @@ public class DockerProvisioningTest { Version wantedVespaVersion = Version.fromString("6.39"); int nodeCount = 7; List<HostSpec> nodes = tester.prepare(application1, - ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("myContent"), wantedVespaVersion, false, Optional.empty()), + ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("myContent"), wantedVespaVersion, false, Optional.empty(), Optional.empty()), nodeCount, 1, dockerFlavor); try { tester.activate(application1, new HashSet<>(nodes)); @@ -94,13 +94,13 @@ public class DockerProvisioningTest { // Activate the zone-app, thereby allocating the parents List<HostSpec> hosts = tester.prepare(zoneApplication, - ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("zone-app"), wantedVespaVersion, false, Optional.empty()), + ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("zone-app"), wantedVespaVersion, false, Optional.empty(), Optional.empty()), Capacity.fromRequiredNodeType(NodeType.host), 1); tester.activate(zoneApplication, hosts); // Try allocating tenants again nodes = tester.prepare(application1, - ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("myContent"), wantedVespaVersion, false, Optional.empty()), + ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("myContent"), wantedVespaVersion, false, Optional.empty(), Optional.empty()), nodeCount, 1, dockerFlavor); tester.activate(application1, new HashSet<>(nodes)); @@ -124,14 +124,14 @@ public class DockerProvisioningTest { Version wantedVespaVersion = Version.fromString("6.39"); List<HostSpec> nodes = tester.prepare(application2_1, - ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("myContent"), wantedVespaVersion, false, Optional.empty()), + ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("myContent"), wantedVespaVersion, false, Optional.empty(), Optional.empty()), 6, 1, resources); assertHostSpecParentReservation(nodes, Optional.empty(), tester); // We do not get nodes on hosts reserved to tenant1 tester.activate(application2_1, nodes); try { tester.prepare(application2_2, - ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("myContent"), wantedVespaVersion, false, Optional.empty()), + ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("myContent"), wantedVespaVersion, false, Optional.empty(), Optional.empty()), 5, 1, resources); fail("Expected exception"); } @@ -140,7 +140,7 @@ public class DockerProvisioningTest { } nodes = tester.prepare(application1_1, - ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("myContent"), wantedVespaVersion, false, Optional.empty()), + ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("myContent"), wantedVespaVersion, false, Optional.empty(), Optional.empty()), 10, 1, resources); assertHostSpecParentReservation(nodes, Optional.of(tenant1), tester); tester.activate(application1_1, nodes); @@ -262,7 +262,10 @@ public class DockerProvisioningTest { ApplicationId application1 = tester.makeApplicationId(); tester.makeReadyVirtualDockerNodes(1, dockerFlavor, "dockerHost"); - List<HostSpec> hosts = tester.prepare(application1, ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("myContent"), Version.fromString("6.42"), false, Optional.empty()), 1, 1, dockerFlavor); + List<HostSpec> hosts = tester.prepare(application1, + ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("myContent"), + Version.fromString("6.42"), false, Optional.empty(), Optional.empty()), + 1, 1, dockerFlavor); tester.activate(application1, new HashSet<>(hosts)); NodeList nodes = tester.getNodes(application1, Node.State.active); @@ -279,8 +282,9 @@ public class DockerProvisioningTest { tester.makeReadyVirtualDockerNodes(1, dockerFlavor, "dockerHost1"); tester.makeReadyVirtualDockerNodes(1, dockerFlavor, "dockerHost2"); - List<HostSpec> hosts = tester.prepare(application1, ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("myContent"), - Version.fromString("6.42"), false, Optional.empty()), 2, 1, + List<HostSpec> hosts = tester.prepare(application1, + ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("myContent"), + Version.fromString("6.42"), false, Optional.empty(), Optional.empty()), 2, 1, dockerFlavor.with(NodeResources.StorageType.remote)); } catch (OutOfCapacityException e) { @@ -294,7 +298,7 @@ public class DockerProvisioningTest { private void prepareAndActivate(ApplicationId application, int nodeCount, boolean exclusive, ProvisioningTester tester) { Set<HostSpec> hosts = new HashSet<>(tester.prepare(application, - ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("myContainer"), Version.fromString("6.39"), exclusive, Optional.empty()), + ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("myContainer"), Version.fromString("6.39"), exclusive, Optional.empty(), Optional.empty()), Capacity.fromCount(nodeCount, Optional.of(dockerFlavor), false, true), 1)); tester.activate(application, hosts); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerAllocationTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerAllocationTest.java index 25f61b1088f..b42960b55b8 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerAllocationTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerAllocationTest.java @@ -343,7 +343,7 @@ public class DynamicDockerAllocationTest { tester.deployZoneApp(); ApplicationId application = tester.makeApplicationId(); - ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("test"), Version.fromString("1"), false, Optional.empty()); + ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("test"), Version.fromString("1"), false, Optional.empty(), Optional.empty()); NodeResources resources = new NodeResources(1, 4, 10, 1, NodeResources.DiskSpeed.any); List<HostSpec> hosts = tester.prepare(application, cluster, 2, 1, resources); @@ -360,7 +360,7 @@ public class DynamicDockerAllocationTest { tester.deployZoneApp(); ApplicationId application = tester.makeApplicationId(); - ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("test"), Version.fromString("1"), false, Optional.empty()); + ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("test"), Version.fromString("1"), false, Optional.empty(), Optional.empty()); NodeResources resources = new NodeResources(1, 4, 10, 1, requestDiskSpeed); try { @@ -382,7 +382,7 @@ public class DynamicDockerAllocationTest { tester.deployZoneApp(); ApplicationId application = tester.makeApplicationId(); - ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("test"), Version.fromString("1"), false, Optional.empty()); + ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("test"), Version.fromString("1"), false, Optional.empty(), Optional.empty()); NodeResources resources = new NodeResources(1, 4, 10, 1, NodeResources.DiskSpeed.fast); List<HostSpec> hosts = tester.prepare(application, cluster, 4, 1, resources); @@ -400,7 +400,7 @@ public class DynamicDockerAllocationTest { tester.deployZoneApp(); ApplicationId application = tester.makeApplicationId(); - ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("test"), Version.fromString("1"), false, Optional.empty()); + ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("test"), Version.fromString("1"), false, Optional.empty(), Optional.empty()); List<HostSpec> hosts1 = tester.prepare(application, cluster, Capacity.fromCount(2, Optional.of(NodeResources.fromLegacyName("d-2-8-50")), false, true), 1); tester.activate(application, hosts1); @@ -465,7 +465,7 @@ public class DynamicDockerAllocationTest { } private ClusterSpec clusterSpec(String clusterId) { - return ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from(clusterId), Version.fromString("6.42"), false, Optional.empty()); + return ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from(clusterId), Version.fromString("6.42"), false, Optional.empty(), Optional.empty()); } } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerProvisionTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerProvisionTest.java index 2868edeaa58..d83921c1174 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerProvisionTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerProvisionTest.java @@ -145,7 +145,8 @@ public class DynamicDockerProvisionTest { } private static ClusterSpec clusterSpec(String clusterId) { - return ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from(clusterId), Version.fromString("6.42"), false, Optional.empty()); + return ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from(clusterId), Version.fromString("6.42"), + false, Optional.empty(), Optional.empty()); } @SuppressWarnings("unchecked") diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/InPlaceResizeProvisionTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/InPlaceResizeProvisionTest.java index d83ed0c12dc..abceacc4dea 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/InPlaceResizeProvisionTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/InPlaceResizeProvisionTest.java @@ -52,9 +52,9 @@ public class InPlaceResizeProvisionTest { private static final NodeResources mediumResources = new NodeResources(4, 8, 16, 1, NodeResources.DiskSpeed.any, NodeResources.StorageType.any); private static final NodeResources largeResources = new NodeResources(8, 16, 32, 1, NodeResources.DiskSpeed.any, NodeResources.StorageType.any); - private static final ClusterSpec container1 = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("container1"), Version.fromString("7.157.9"), false, Optional.empty()); - private static final ClusterSpec container2 = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("container2"), Version.fromString("7.157.9"), false, Optional.empty()); - private static final ClusterSpec content1 = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("content1"), Version.fromString("7.157.9"), false, Optional.empty()); + private static final ClusterSpec container1 = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("container1"), Version.fromString("7.157.9"), false, Optional.empty(), Optional.empty()); + private static final ClusterSpec container2 = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("container2"), Version.fromString("7.157.9"), false, Optional.empty(), Optional.empty()); + private static final ClusterSpec content1 = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("content1"), Version.fromString("7.157.9"), false, Optional.empty(), Optional.empty()); private final InMemoryFlagSource flagSource = new InMemoryFlagSource(); private final ProvisioningTester tester = new ProvisioningTester.Builder() diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java index d4037c94a45..8ecee1c808b 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java @@ -272,7 +272,7 @@ public class LoadBalancerProvisionerTest { } private static ClusterSpec clusterRequest(ClusterSpec.Type type, ClusterSpec.Id id, Optional<ClusterSpec.Id> combinedId) { - return ClusterSpec.request(type, id, Version.fromString("6.42"), false, combinedId); + return ClusterSpec.request(type, id, Version.fromString("6.42"), false, combinedId, Optional.empty()); } private static <T> T get(Set<T> set, int position) { diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/MultigroupProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/MultigroupProvisioningTest.java index 549441eb9e6..a54f45af0e1 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/MultigroupProvisioningTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/MultigroupProvisioningTest.java @@ -204,7 +204,8 @@ public class MultigroupProvisioningTest { assertEquals("No additional groups are retained containing retired nodes", wantedGroups, allGroups.size()); } - private ClusterSpec cluster() { return ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("test"), Version.fromString("6.42"), false, Optional.empty()); } + private ClusterSpec cluster() { return ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("test"), + Version.fromString("6.42"), false, Optional.empty(), Optional.empty()); } private Set<HostSpec> prepare(ApplicationId application, Capacity capacity, int groupCount, ProvisioningTester tester) { return new HashSet<>(tester.prepare(application, cluster(), capacity, groupCount)); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/NodeTypeProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/NodeTypeProvisioningTest.java index 5cd31f00895..5c06b50bb10 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/NodeTypeProvisioningTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/NodeTypeProvisioningTest.java @@ -39,7 +39,7 @@ public class NodeTypeProvisioningTest { private final ClusterSpec clusterSpec = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("test"), Version.fromString("6.42"), - false, Optional.empty()); + false, Optional.empty(), Optional.empty()); @Before public void setup() { diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java index d831ab038db..a5527a4a6ee 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java @@ -473,7 +473,7 @@ public class ProvisioningTest { ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("music"), new com.yahoo.component.Version(4, 5, 6), - false, Optional.empty()); + false, Optional.empty(), Optional.empty()); tester.prepare(application, cluster, Capacity.fromCount(5, Optional.empty(), false, false), 1); // No exception; Success } @@ -510,7 +510,7 @@ public class ProvisioningTest { ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("music"), new com.yahoo.component.Version(4, 5, 6), - false, Optional.empty()); + false, Optional.empty(), Optional.empty()); tester.activate(application, tester.prepare(application, cluster, capacity, 1)); assertEquals(5, NodeList.copyOf(tester.nodeRepository().getNodes(application, Node.State.active)).not().retired().size()); assertEquals(0, NodeList.copyOf(tester.nodeRepository().getNodes(application, Node.State.active)).retired().size()); @@ -683,6 +683,7 @@ public class ProvisioningTest { ClusterSpec.Id.from("music"), Version.fromString("1.2.3"), false, + Optional.empty(), Optional.empty()); var initialNodes = tester.activate(application, tester.prepare(application, cluster, Capacity.fromCount(2, defaultResources, false, false), @@ -693,6 +694,7 @@ public class ProvisioningTest { ClusterSpec.Id.from("music"), Version.fromString("1.2.3"), false, + Optional.empty(), Optional.empty()); var newNodes = tester.activate(application, tester.prepare(application, cluster, Capacity.fromCount(2, defaultResources, false, false), @@ -708,6 +710,7 @@ public class ProvisioningTest { ClusterSpec.Id.from("music"), Version.fromString("1.2.3"), false, + Optional.empty(), Optional.empty()); newNodes = tester.activate(application, tester.prepare(application, cluster, Capacity.fromCount(2, defaultResources, false, false), @@ -734,10 +737,10 @@ public class ProvisioningTest { int content1Size, boolean required, NodeResources nodeResources, Version wantedVersion, ProvisioningTester tester) { // "deploy prepare" with a two container clusters and a storage cluster having of two groups - ClusterSpec containerCluster0 = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("container0"), wantedVersion, false, Optional.empty()); - ClusterSpec containerCluster1 = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("container1"), wantedVersion, false, Optional.empty()); - ClusterSpec contentCluster0 = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("content0"), wantedVersion, false, Optional.empty()); - ClusterSpec contentCluster1 = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("content1"), wantedVersion, false, Optional.empty()); + ClusterSpec containerCluster0 = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("container0"), wantedVersion, false, Optional.empty(), Optional.empty()); + ClusterSpec containerCluster1 = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("container1"), wantedVersion, false, Optional.empty(), Optional.empty()); + ClusterSpec contentCluster0 = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("content0"), wantedVersion, false, Optional.empty(), Optional.empty()); + ClusterSpec contentCluster1 = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("content1"), wantedVersion, false, Optional.empty(), Optional.empty()); Set<HostSpec> container0 = prepare(application, containerCluster0, container0Size, 1, required, nodeResources, tester); Set<HostSpec> container1 = prepare(application, containerCluster1, container1Size, 1, required, nodeResources, tester); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java index c4f6ef3b94e..78b4ccbd868 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java @@ -158,7 +158,8 @@ public class ProvisioningTester { } public void prepareAndActivateInfraApplication(ApplicationId application, NodeType nodeType, Version version) { - ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from(nodeType.toString()), version, false, Optional.empty()); + ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from(nodeType.toString()), + version, false, Optional.empty(), Optional.empty()); Capacity capacity = Capacity.fromRequiredNodeType(nodeType); List<HostSpec> hostSpecs = prepare(application, cluster, capacity, 1, true); activate(application, hostSpecs); @@ -412,7 +413,7 @@ public class ProvisioningTester { ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("node-admin"), Version.fromString("6.42"), - false, Optional.empty()), + false, Optional.empty(), Optional.empty()), Capacity.fromRequiredNodeType(NodeType.host), 1); activate(applicationId, Set.copyOf(list)); @@ -420,7 +421,7 @@ public class ProvisioningTester { public ClusterSpec clusterSpec() { return ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("test"), - Version.fromString("6.42"), false, Optional.empty()); + Version.fromString("6.42"), false, Optional.empty(), Optional.empty()); } public List<Node> deploy(ApplicationId application, Capacity capacity) { diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/VirtualNodeProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/VirtualNodeProvisioningTest.java index 60616481665..64c91e66c26 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/VirtualNodeProvisioningTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/VirtualNodeProvisioningTest.java @@ -36,8 +36,10 @@ import static org.junit.Assert.assertNotNull; public class VirtualNodeProvisioningTest { private static final NodeResources flavor = new NodeResources(4, 8, 100, 1); - private static final ClusterSpec contentClusterSpec = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("myContent"), Version.fromString("6.42"), false, Optional.empty()); - private static final ClusterSpec containerClusterSpec = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("myContainer"), Version.fromString("6.42"), false, Optional.empty()); + private static final ClusterSpec contentClusterSpec = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("myContent"), + Version.fromString("6.42"), false, Optional.empty(), Optional.empty()); + private static final ClusterSpec containerClusterSpec = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("myContainer"), + Version.fromString("6.42"), false, Optional.empty(), Optional.empty()); private ProvisioningTester tester = new ProvisioningTester.Builder().build(); private ApplicationId applicationId = tester.makeApplicationId(); diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/HostResource.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/HostResource.java index c93bb9e1d4a..054d3320215 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/HostResource.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/HostResource.java @@ -78,10 +78,10 @@ public class HostResource implements HostApi { applicationUri.toString(), hostServices); } catch (UncheckedTimeoutException e) { - log.log(LogLevel.INFO, "Failed to get host " + hostName + ": " + e.getMessage()); + log.log(LogLevel.DEBUG, "Failed to get host " + hostName + ": " + e.getMessage()); throw webExceptionFromTimeout("getHost", hostName, e); } catch (HostNameNotFoundException e) { - log.log(LogLevel.INFO, "Host not found: " + hostName); + log.log(LogLevel.DEBUG, "Host not found: " + hostName); throw new NotFoundException(e); } } @@ -101,14 +101,14 @@ public class HostResource implements HostApi { try { orchestrator.setNodeStatus(hostName, state); } catch (HostNameNotFoundException e) { - log.log(LogLevel.INFO, "Host not found: " + hostName); + log.log(LogLevel.DEBUG, "Host not found: " + hostName); throw new NotFoundException(e); } catch (UncheckedTimeoutException e) { - log.log(LogLevel.INFO, "Failed to patch " + hostName + ": " + e.getMessage()); + log.log(LogLevel.DEBUG, "Failed to patch " + hostName + ": " + e.getMessage()); throw webExceptionFromTimeout("patch", hostName, e); } catch (OrchestrationException e) { String message = "Failed to set " + hostName + " to " + state + ": " + e.getMessage(); - log.log(LogLevel.INFO, message, e); + log.log(LogLevel.DEBUG, message, e); throw new InternalServerErrorException(message); } } @@ -124,13 +124,13 @@ public class HostResource implements HostApi { try { orchestrator.suspend(hostName); } catch (HostNameNotFoundException e) { - log.log(LogLevel.INFO, "Host not found: " + hostName); + log.log(LogLevel.DEBUG, "Host not found: " + hostName); throw new NotFoundException(e); } catch (UncheckedTimeoutException e) { - log.log(LogLevel.INFO, "Failed to suspend " + hostName + ": " + e.getMessage()); + log.log(LogLevel.DEBUG, "Failed to suspend " + hostName + ": " + e.getMessage()); throw webExceptionFromTimeout("suspend", hostName, e); } catch (HostStateChangeDeniedException e) { - log.log(LogLevel.INFO, "Failed to suspend " + hostName + ": " + e.getMessage()); + log.log(LogLevel.DEBUG, "Failed to suspend " + hostName + ": " + e.getMessage()); throw webExceptionWithDenialReason("suspend", hostName, e); } return new UpdateHostResponse(hostName.s(), null); @@ -142,13 +142,13 @@ public class HostResource implements HostApi { try { orchestrator.resume(hostName); } catch (HostNameNotFoundException e) { - log.log(LogLevel.INFO, "Host not found: " + hostName); + log.log(LogLevel.DEBUG, "Host not found: " + hostName); throw new NotFoundException(e); } catch (UncheckedTimeoutException e) { - log.log(LogLevel.INFO, "Failed to resume " + hostName + ": " + e.getMessage()); + log.log(LogLevel.DEBUG, "Failed to resume " + hostName + ": " + e.getMessage()); throw webExceptionFromTimeout("resume", hostName, e); } catch (HostStateChangeDeniedException e) { - log.log(LogLevel.INFO, "Failed to resume " + hostName + ": " + e.getMessage()); + log.log(LogLevel.DEBUG, "Failed to resume " + hostName + ": " + e.getMessage()); throw webExceptionWithDenialReason("resume", hostName, e); } return new UpdateHostResponse(hostName.s(), null); diff --git a/searchlib/CMakeLists.txt b/searchlib/CMakeLists.txt index d754fd78394..30f7fc4f54c 100644 --- a/searchlib/CMakeLists.txt +++ b/searchlib/CMakeLists.txt @@ -134,6 +134,7 @@ vespa_define_module( src/tests/features/item_raw_score src/tests/features/max_reduce_prod_join_replacer src/tests/features/native_dot_product + src/tests/features/nns_distance src/tests/features/ranking_expression src/tests/features/raw_score src/tests/features/subqueries diff --git a/searchlib/src/tests/features/nns_distance/CMakeLists.txt b/searchlib/src/tests/features/nns_distance/CMakeLists.txt new file mode 100644 index 00000000000..5b4c8f86b44 --- /dev/null +++ b/searchlib/src/tests/features/nns_distance/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +vespa_add_executable(searchlib_nns_distance_test_app TEST + SOURCES + nns_distance_test.cpp + DEPENDS + searchlib +) +vespa_add_test(NAME searchlib_nns_distance_test_app COMMAND searchlib_nns_distance_test_app) diff --git a/searchlib/src/tests/features/nns_distance/nns_distance_test.cpp b/searchlib/src/tests/features/nns_distance/nns_distance_test.cpp new file mode 100644 index 00000000000..ea864f9b7a0 --- /dev/null +++ b/searchlib/src/tests/features/nns_distance/nns_distance_test.cpp @@ -0,0 +1,177 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/searchlib/features/setup.h> +#include <vespa/searchlib/fef/test/indexenvironment.h> +#include <vespa/searchlib/fef/test/indexenvironmentbuilder.h> +#include <vespa/searchlib/fef/test/queryenvironment.h> +#include <vespa/searchlib/features/distancefeature.h> +#include <vespa/searchlib/fef/fef.h> +#include <vespa/searchlib/fef/test/dummy_dependency_handler.h> +#include <vespa/vespalib/stllike/asciistream.h> +#include <vespa/vespalib/util/stringfmt.h> + +using search::feature_t; +using namespace search::fef; +using namespace search::fef::test; +using namespace search::features; +using CollectionType = FieldInfo::CollectionType; +using DataType = FieldInfo::DataType; + +const vespalib::string labelFeatureName("distance(label)"); +const vespalib::string fieldFeatureName("distance(bar)"); + +struct BlueprintFactoryFixture { + BlueprintFactory factory; + BlueprintFactoryFixture() : factory() + { + setup_search_features(factory); + } +}; + +struct IndexFixture { + IndexEnvironment indexEnv; + IndexFixture() : indexEnv() + { + IndexEnvironmentBuilder builder(indexEnv); + builder.addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, DataType::INT64, "foo"); + builder.addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, DataType::TENSOR, "bar"); + } +}; + +struct FeatureDumpFixture : public IDumpFeatureVisitor { + virtual void visitDumpFeature(const vespalib::string &) override { + TEST_ERROR("no features should be dumped"); + } + FeatureDumpFixture() : IDumpFeatureVisitor() {} +}; + +struct Labels { + virtual void inject(Properties &p) const = 0; + virtual ~Labels() {} +}; +struct NoLabel : public Labels { + virtual void inject(Properties &) const override {} +}; +struct SingleLabel : public Labels { + vespalib::string label; + uint32_t uid; + SingleLabel(const vespalib::string &l, uint32_t x) : label(l), uid(x) {} + virtual void inject(Properties &p) const override { + vespalib::asciistream key; + key << "vespa.label." << label << ".id"; + vespalib::asciistream value; + value << uid; + p.add(key.str(), value.str()); + } +}; + +struct RankFixture : BlueprintFactoryFixture, IndexFixture { + QueryEnvironment queryEnv; + RankSetup rankSetup; + MatchDataLayout mdl; + MatchData::UP match_data; + RankProgram::UP rankProgram; + std::vector<TermFieldHandle> fooHandles; + std::vector<TermFieldHandle> barHandles; + RankFixture(size_t fooCnt, size_t barCnt, const Labels &labels, const vespalib::string &featureName) + : queryEnv(&indexEnv), rankSetup(factory, indexEnv), + mdl(), match_data(), rankProgram(), fooHandles(), barHandles() + { + for (size_t i = 0; i < fooCnt; ++i) { + uint32_t fieldId = indexEnv.getFieldByName("foo")->id(); + fooHandles.push_back(mdl.allocTermField(fieldId)); + SimpleTermData term; + term.setUniqueId(i + 1); + term.addField(fieldId).setHandle(fooHandles.back()); + queryEnv.getTerms().push_back(term); + } + for (size_t i = 0; i < barCnt; ++i) { + uint32_t fieldId = indexEnv.getFieldByName("bar")->id(); + barHandles.push_back(mdl.allocTermField(fieldId)); + SimpleTermData term; + term.setUniqueId(fooCnt + i + 1); + term.addField(fieldId).setHandle(barHandles.back()); + queryEnv.getTerms().push_back(term); + } + labels.inject(queryEnv.getProperties()); + rankSetup.setFirstPhaseRank(featureName); + rankSetup.setIgnoreDefaultRankFeatures(true); + ASSERT_TRUE(rankSetup.compile()); + match_data = mdl.createMatchData(); + rankProgram = rankSetup.create_first_phase_program(); + rankProgram->setup(*match_data, queryEnv); + } + feature_t getScore(uint32_t docId) { + return Utils::getScoreFeature(*rankProgram, docId); + } + void setScore(TermFieldHandle handle, uint32_t docId, feature_t score) { + match_data->resolveTermField(handle)->setRawScore(docId, score); + } + void setFooScore(uint32_t i, uint32_t docId, feature_t score) { + ASSERT_LESS(i, fooHandles.size()); + setScore(fooHandles[i], docId, score); + } + void setBarScore(uint32_t i, uint32_t docId, feature_t score) { + ASSERT_LESS(i, barHandles.size()); + setScore(barHandles[i], docId, score); + } +}; + +TEST_F("require that blueprint can be created from factory", BlueprintFactoryFixture) { + Blueprint::SP bp = f.factory.createBlueprint("distance"); + EXPECT_TRUE(bp.get() != 0); + EXPECT_TRUE(dynamic_cast<DistanceBlueprint*>(bp.get()) != 0); +} + +TEST_FFF("require that no features are dumped", DistanceBlueprint, IndexFixture, FeatureDumpFixture) { + f1.visitDumpFeatures(f2.indexEnv, f3); +} + +TEST_FF("require that setup can be done on random label", DistanceBlueprint, IndexFixture) { + DummyDependencyHandler deps(f1); + f1.setName(vespalib::make_string("%s(random_label)", f1.getBaseName().c_str())); + EXPECT_TRUE(static_cast<Blueprint&>(f1).setup(f2.indexEnv, std::vector<vespalib::string>(1, "random_label"))); +} + +TEST_FF("require that no label gives max-double distance", NoLabel(), RankFixture(2, 2, f1, labelFeatureName)) { + EXPECT_EQUAL(std::numeric_limits<feature_t>::max(), f2.getScore(10)); +} + +TEST_FF("require that unrelated label gives max-double distance", SingleLabel("unrelated", 1), RankFixture(2, 2, f1, labelFeatureName)) { + EXPECT_EQUAL(std::numeric_limits<feature_t>::max(), f2.getScore(10)); +} + +TEST_FF("require that labeled item raw score can be obtained", SingleLabel("label", 1), RankFixture(2, 2, f1, labelFeatureName)) { + f2.setFooScore(0, 10, 5.0); + EXPECT_EQUAL(5.0, f2.getScore(10)); +} + +TEST_FF("require that field raw score can be obtained", NoLabel(), RankFixture(2, 2, f1, fieldFeatureName)) { + f2.setBarScore(0, 10, 5.0); + EXPECT_EQUAL(5.0, f2.getScore(10)); +} + +TEST_FF("require that other raw scores are ignored", SingleLabel("label", 2), RankFixture(2, 2, f1, labelFeatureName)) { + f2.setFooScore(0, 10, 1.0); + f2.setFooScore(1, 10, 2.0); + f2.setBarScore(0, 10, 5.0); + f2.setBarScore(1, 10, 6.0); + EXPECT_EQUAL(2.0, f2.getScore(10)); +} + +TEST_FF("require that the correct raw score is used", NoLabel(), RankFixture(2, 2, f1, fieldFeatureName)) { + f2.setFooScore(0, 10, 3.0); + f2.setFooScore(1, 10, 4.0); + f2.setBarScore(0, 10, 8.0); + f2.setBarScore(1, 10, 7.0); + EXPECT_EQUAL(7.0, f2.getScore(10)); +} + +TEST_FF("require that stale data is ignored", SingleLabel("label", 2), RankFixture(2, 2, f1, labelFeatureName)) { + f2.setFooScore(0, 10, 1.0); + f2.setFooScore(1, 5, 2.0); + EXPECT_EQUAL(std::numeric_limits<feature_t>::max(), f2.getScore(10)); +} + +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchlib/src/tests/features/prod_features.cpp b/searchlib/src/tests/features/prod_features.cpp index 5f70085b9ec..253749dcdb7 100644 --- a/searchlib/src/tests/features/prod_features.cpp +++ b/searchlib/src/tests/features/prod_features.cpp @@ -474,11 +474,11 @@ Test::testCloseness() } { // Test executor. - assertCloseness(1, "pos", 0); + TEST_DO(assertCloseness(1, "pos", 0)); assertCloseness(0.8, "pos", 1802661); assertCloseness(0, "pos", 9013306); // use non-existing attribute -> default distance - assertCloseness(0, "no", 0); + TEST_DO(assertCloseness(0, "no", 0)); // use non-default maxDistance assertCloseness(1, "pos", 0, 100); @@ -852,15 +852,23 @@ Test::testDistance() { // test default distance { // non-existing attribute FtFeatureTest ft(_factory, "distance(pos)"); + ft.getIndexEnv().getBuilder().addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, DataType::INT64, "pos"); ft.getQueryEnv().getLocation().setValid(true); ASSERT_TRUE(ft.setup()); ASSERT_TRUE(ft.execute(RankResult().addScore("distance(pos)", 6400000000.0))); } + { // non-existing field + FtFeatureTest ft(_factory, "distance(pos)"); + ft.getQueryEnv().getLocation().setValid(true); + ASSERT_TRUE(ft.setup()); + ASSERT_TRUE(ft.execute(RankResult().addScore("distance(pos)", std::numeric_limits<feature_t>::max()))); + } { // wrong attribute type (float) FtFeatureTest ft(_factory, "distance(pos)"); AttributePtr pos = AttributeFactory::createAttribute("pos", AVC(AVBT::FLOAT, AVCT::SINGLE)); pos->commit(); ft.getIndexEnv().getAttributeMap().add(pos); + ft.getIndexEnv().getBuilder().addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, DataType::INT64, "pos"); ft.getQueryEnv().getLocation().setValid(true); ASSERT_TRUE(ft.setup()); ASSERT_TRUE(ft.execute(RankResult().addScore("distance(pos)", 6400000000.0))); @@ -870,6 +878,7 @@ Test::testDistance() AttributePtr pos = AttributeFactory::createAttribute("pos", AVC(AVBT::STRING, AVCT::SINGLE)); pos->commit(); ft.getIndexEnv().getAttributeMap().add(pos); + ft.getIndexEnv().getBuilder().addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, DataType::INT64, "pos"); ft.getQueryEnv().getLocation().setValid(true); ASSERT_TRUE(ft.setup()); ASSERT_TRUE(ft.execute(RankResult().addScore("distance(pos)", 6400000000.0))); @@ -879,6 +888,7 @@ Test::testDistance() AttributePtr pos = AttributeFactory::createAttribute("pos", AVC(AVBT::INT64, AVCT::WSET)); pos->commit(); ft.getIndexEnv().getAttributeMap().add(pos); + ft.getIndexEnv().getBuilder().addField(FieldType::ATTRIBUTE, CollectionType::WEIGHTEDSET, DataType::INT64, "pos"); ft.getQueryEnv().getLocation().setValid(true); ASSERT_TRUE(ft.setup()); ASSERT_TRUE(ft.execute(RankResult().addScore("distance(pos)", 6400000000.0))); @@ -896,6 +906,7 @@ Test::setupForDistanceTest(FtFeatureTest &ft, const vespalib::string & attrName, pos->addReservedDoc(); pos->addDocs(1); ft.getIndexEnv().getAttributeMap().add(pos); + ft.getIndexEnv().getBuilder().addField(FieldType::ATTRIBUTE, CollectionType::ARRAY, DataType::INT64, attrName); auto ia = dynamic_cast<IntegerAttribute *>(pos.get()); for (const auto & p : positions) { diff --git a/searchlib/src/vespa/searchlib/features/distancefeature.cpp b/searchlib/src/vespa/searchlib/features/distancefeature.cpp index 54863d310e7..588fdcf17b7 100644 --- a/searchlib/src/vespa/searchlib/features/distancefeature.cpp +++ b/searchlib/src/vespa/searchlib/features/distancefeature.cpp @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "distancefeature.h" +#include <vespa/searchcommon/common/schema.h> #include <vespa/searchlib/fef/location.h> #include <vespa/searchlib/fef/matchdata.h> #include <vespa/document/datatype/positiondatatype.h> @@ -8,14 +9,76 @@ #include <vespa/vespalib/util/stash.h> #include <cmath> #include <limits> +#include "utils.h" #include <vespa/log/log.h> LOG_SETUP(".features.distancefeature"); using namespace search::fef; +using namespace search::index::schema; namespace search::features { +/** Implements the executor for converting NNS rawscore to a distance feature. */ +class ConvertRawscoreExecutor : public fef::FeatureExecutor { +private: + std::vector<fef::TermFieldHandle> _handles; + const fef::MatchData *_md; + void handle_bind_match_data(const fef::MatchData &md) override { + _md = &md; + } +public: + ConvertRawscoreExecutor(const fef::IQueryEnvironment &env, uint32_t fieldId); + ConvertRawscoreExecutor(const fef::IQueryEnvironment &env, const vespalib::string &label); + void execute(uint32_t docId) override; +}; + +ConvertRawscoreExecutor::ConvertRawscoreExecutor(const fef::IQueryEnvironment &env, uint32_t fieldId) + : _handles(), + _md(nullptr) +{ + _handles.reserve(env.getNumTerms()); + for (uint32_t i = 0; i < env.getNumTerms(); ++i) { + search::fef::TermFieldHandle handle = util::getTermFieldHandle(env, i, fieldId); + if (handle != search::fef::IllegalHandle) { + _handles.push_back(handle); + } + } +} + +ConvertRawscoreExecutor::ConvertRawscoreExecutor(const fef::IQueryEnvironment &env, const vespalib::string &label) + : _handles(), + _md(nullptr) +{ + const ITermData *term = util::getTermByLabel(env, label); + if (term != nullptr) { + // expect numFields() == 1 + for (uint32_t i = 0; i < term->numFields(); ++i) { + TermFieldHandle handle = term->field(i).getHandle(); + if (handle != IllegalHandle) { + _handles.push_back(handle); + } + } + } +} + +void +ConvertRawscoreExecutor::execute(uint32_t docId) +{ + feature_t output = std::numeric_limits<feature_t>::max(); + assert(_md); + for (auto handle : _handles) { + const TermFieldMatchData *tfmd = _md->resolveTermField(handle); + if (tfmd->getDocId() == docId) { + // add conversion from "closeness" RawScore later: + feature_t converted = tfmd->getRawScore(); + output = std::min(output, converted); + } + } + outputs().set_number(0, output); +} + + feature_t DistanceExecutor::calculateDistance(uint32_t docId) { @@ -82,7 +145,11 @@ const feature_t DistanceExecutor::DEFAULT_DISTANCE(6400000000.0); DistanceBlueprint::DistanceBlueprint() : Blueprint("distance"), - _posAttr() + _arg_string(), + _attr_id(search::index::Schema::UNKNOWN_FIELD_ID), + _use_geo_pos(false), + _use_nns_tensor(false), + _use_item_label(false) { } @@ -101,30 +168,70 @@ DistanceBlueprint::createInstance() const } bool +DistanceBlueprint::setup_geopos(const IIndexEnvironment & env, + const vespalib::string &attr) +{ + _arg_string = attr; + _use_geo_pos = true; + describeOutput("out", "The euclidean distance from the query position."); + env.hintAttributeAccess(_arg_string); + return true; +} + +bool +DistanceBlueprint::setup_nns(const IIndexEnvironment & env, + const vespalib::string &attr) +{ + _arg_string = attr; + _use_nns_tensor = true; + describeOutput("out", "The euclidean distance from the query position."); + env.hintAttributeAccess(_arg_string); + return true; +} + +bool DistanceBlueprint::setup(const IIndexEnvironment & env, const ParameterList & params) { - _posAttr = params[0].getValue(); - describeOutput("out", "The euclidean distance from the query position."); - env.hintAttributeAccess(_posAttr); - env.hintAttributeAccess(document::PositionDataType::getZCurveFieldName(_posAttr)); + vespalib::string arg = params[0].getValue(); + const FieldInfo *fi = env.getFieldByName(arg); + if (fi != nullptr && fi->hasAttribute()) { + auto dt = fi->get_data_type(); + auto ct = fi->collection(); + if (dt == DataType::TENSOR && ct == CollectionType::SINGLE) { + _attr_id = fi->id(); + return setup_nns(env, arg); + } + // could check if dt is DataType::INT64 + // could check if ct is CollectionType::SINGLE or CollectionType::ARRAY) + return setup_geopos(env, arg); + } + vespalib::string z = document::PositionDataType::getZCurveFieldName(arg); + fi = env.getFieldByName(z); + if (fi != nullptr && fi->hasAttribute()) { + return setup_geopos(env, z); + } + _arg_string = arg; + _use_item_label = true; + describeOutput("out", "The euclidean distance from the labeled query item."); return true; } FeatureExecutor & DistanceBlueprint::createExecutor(const IQueryEnvironment &env, vespalib::Stash &stash) const { + if (_use_nns_tensor) { + return stash.create<ConvertRawscoreExecutor>(env, _attr_id); + } + if (_use_item_label) { + return stash.create<ConvertRawscoreExecutor>(env, _arg_string); + } const search::attribute::IAttributeVector * pos = nullptr; const Location & location = env.getLocation(); - LOG(debug, "DistanceBlueprint::createExecutor location.valid='%s', '%s', alternatively '%s'", - location.isValid() ? "true" : "false", _posAttr.c_str(), document::PositionDataType::getZCurveFieldName(_posAttr).c_str()); - if (location.isValid()) { - pos = env.getAttributeContext().getAttribute(_posAttr); - if (pos == nullptr) { - LOG(debug, "Failed to find attribute '%s', resorting too '%s'", - _posAttr.c_str(), document::PositionDataType::getZCurveFieldName(_posAttr).c_str()); - pos = env.getAttributeContext().getAttribute(document::PositionDataType::getZCurveFieldName(_posAttr)); - } + LOG(debug, "DistanceBlueprint::createExecutor location.valid='%s', attribute='%s'", + location.isValid() ? "true" : "false", _arg_string.c_str()); + if (_use_geo_pos && location.isValid()) { + pos = env.getAttributeContext().getAttribute(_arg_string); if (pos != nullptr) { if (!pos->isIntegerType()) { LOG(warning, "The position attribute '%s' is not an integer attribute. Will use default distance.", @@ -136,7 +243,7 @@ DistanceBlueprint::createExecutor(const IQueryEnvironment &env, vespalib::Stash pos = nullptr; } } else { - LOG(warning, "The position attribute '%s' was not found. Will use default distance.", _posAttr.c_str()); + LOG(warning, "The position attribute '%s' was not found. Will use default distance.", _arg_string.c_str()); } } diff --git a/searchlib/src/vespa/searchlib/features/distancefeature.h b/searchlib/src/vespa/searchlib/features/distancefeature.h index 3c75d53ad77..e0eaa98ac4c 100644 --- a/searchlib/src/vespa/searchlib/features/distancefeature.h +++ b/searchlib/src/vespa/searchlib/features/distancefeature.h @@ -12,7 +12,7 @@ namespace search::features { */ class DistanceExecutor : public fef::FeatureExecutor { private: - const fef::Location & _location; + const fef::Location & _location; const attribute::IAttributeVector * _pos; attribute::IntegerContent _intBuf; @@ -37,7 +37,14 @@ public: */ class DistanceBlueprint : public fef::Blueprint { private: - vespalib::string _posAttr; + vespalib::string _arg_string; + uint32_t _attr_id; + bool _use_geo_pos; + bool _use_nns_tensor; + bool _use_item_label; + + bool setup_geopos(const fef::IIndexEnvironment & env, const vespalib::string &attr); + bool setup_nns(const fef::IIndexEnvironment & env, const vespalib::string &attr); public: DistanceBlueprint(); diff --git a/searchlib/src/vespa/searchlib/features/raw_score_feature.h b/searchlib/src/vespa/searchlib/features/raw_score_feature.h index 2a4eb946a68..42813cf9d22 100644 --- a/searchlib/src/vespa/searchlib/features/raw_score_feature.h +++ b/searchlib/src/vespa/searchlib/features/raw_score_feature.h @@ -10,7 +10,7 @@ class RawScoreExecutor : public fef::FeatureExecutor { private: std::vector<fef::TermFieldHandle> _handles; - const fef::MatchData *_md; + const fef::MatchData *_md; void handle_bind_match_data(const fef::MatchData &md) override; public: diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/InfraApplication.java b/service-monitor/src/main/java/com/yahoo/vespa/service/duper/InfraApplication.java index 7b79adf89d0..64cf4f6f26f 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/InfraApplication.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/duper/InfraApplication.java @@ -75,7 +75,7 @@ public abstract class InfraApplication implements InfraApplicationApi { @Override public ClusterSpec getClusterSpecWithVersion(Version version) { - return ClusterSpec.request(clusterSpecType, clusterSpecId, version, true, Optional.empty()); + return ClusterSpec.request(clusterSpecType, clusterSpecId, version, true, Optional.empty(), Optional.empty()); } public ClusterSpec.Type getClusterSpecType() { diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/InfraApplicationApi.java b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/InfraApplicationApi.java index dbb326b3c50..b4b61682a33 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/InfraApplicationApi.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/InfraApplicationApi.java @@ -6,6 +6,8 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Capacity; import com.yahoo.config.provision.ClusterSpec; +import java.util.Optional; + /** * API of infrastructure application that is accessible via DuperModelInfraApi. * |