diff options
89 files changed, 825 insertions, 465 deletions
diff --git a/config-model/src/main/java/com/yahoo/config/model/provision/SingleNodeProvisioner.java b/config-model/src/main/java/com/yahoo/config/model/provision/SingleNodeProvisioner.java index b7035594c71..180a16f3c8f 100644 --- a/config-model/src/main/java/com/yahoo/config/model/provision/SingleNodeProvisioner.java +++ b/config-model/src/main/java/com/yahoo/config/model/provision/SingleNodeProvisioner.java @@ -32,7 +32,7 @@ public class SingleNodeProvisioner implements HostProvisioner { } public SingleNodeProvisioner(Flavor flavor) { host = new Host(HostName.getLocalhost()); - this.hostSpec = new HostSpec(host.hostname(), host.aliases(),flavor); + this.hostSpec = new HostSpec(host.hostname(), host.aliases(), flavor); } @Override diff --git a/config-model/src/main/java/com/yahoo/vespa/model/HostPorts.java b/config-model/src/main/java/com/yahoo/vespa/model/HostPorts.java index 51b41e19a27..a80982fe75b 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/HostPorts.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/HostPorts.java @@ -20,10 +20,6 @@ import java.util.logging.Level; */ public class HostPorts { - public HostPorts(String hostname) { - this.hostname = hostname; - } - final String hostname; public final static int BASE_PORT = 19100; final static int MAX_PORTS = 799; @@ -42,6 +38,10 @@ public class HostPorts { private Optional<NetworkPorts> networkPortsList = Optional.empty(); + public HostPorts(String hostname) { + this.hostname = hostname; + } + /** * Get the allocated network ports. * Should be called after allocation is complete and flushPortReservations has been called diff --git a/config-model/src/main/java/com/yahoo/vespa/model/HostResource.java b/config-model/src/main/java/com/yahoo/vespa/model/HostResource.java index 0d4c0aa28af..9dba6fde9d4 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/HostResource.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/HostResource.java @@ -38,8 +38,6 @@ public class HostResource implements Comparable<HostResource> { /** Map from "sentinel name" to service */ private final Map<String, Service> services = new LinkedHashMap<>(); - private Set<ClusterMembership> clusterMemberships = new LinkedHashSet<>(); - /** * Create a new {@link HostResource} bound to a specific {@link com.yahoo.vespa.model.Host}. * @@ -64,7 +62,7 @@ public class HostResource implements Comparable<HostResource> { public HostPorts ports() { return hostPorts; } - public HostSpec spec() { return spec; } + public HostSpec spec() { return spec.withPorts(hostPorts.networkPorts()); } /** * Adds service and allocates resources for it. @@ -109,31 +107,6 @@ public class HostResource implements Comparable<HostResource> { /** Returns the flavor of this resource. Empty for self-hosted Vespa. */ public Optional<Flavor> getFlavor() { return spec.flavor(); } - public void addClusterMembership(ClusterMembership clusterMembership) { - if (clusterMembership != null) - clusterMemberships.add(clusterMembership); - } - - public Set<ClusterMembership> clusterMemberships() { - return Collections.unmodifiableSet(clusterMemberships); - } - - /** - * Returns the "primary" cluster membership. - * Content clusters are preferred, then container clusters, and finally admin clusters. - * If there is more than one cluster of the preferred type, the cluster that was added first will be chosen. - */ - public Optional<ClusterMembership> primaryClusterMembership() { - return clusterMemberships().stream() - .sorted(HostResource::compareClusters) - .findFirst(); - } - - private static int compareClusters(ClusterMembership cluster1, ClusterMembership cluster2) { - // This depends on the declared order of enum constants. - return cluster2.cluster().type().compareTo(cluster1.cluster().type()); - } - @Override public String toString() { return "host '" + host.getHostname() + "'"; @@ -163,10 +136,8 @@ public class HostResource implements Comparable<HostResource> { * Compare by hostname otherwise. */ public int comparePrimarilyByIndexTo(HostResource other) { - Optional<ClusterMembership> thisMembership = this.primaryClusterMembership(); - Optional<ClusterMembership> otherMembership = other.primaryClusterMembership(); - if (thisMembership.isPresent() && otherMembership.isPresent()) - return Integer.compare(thisMembership.get().index(), otherMembership.get().index()); + if (this.spec.membership().isPresent() && other.spec.membership().isPresent()) + return Integer.compare(this.spec.membership().get().index(), other.spec.membership().get().index()); else return this.getHostname().compareTo(other.getHostname()); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/HostSystem.java b/config-model/src/main/java/com/yahoo/vespa/model/HostSystem.java index 7d986eed877..3ac5f794426 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/HostSystem.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/HostSystem.java @@ -116,19 +116,13 @@ public class HostSystem extends AbstractConfigProducer<Host> { public HostResource getHost(String hostAlias) { HostSpec hostSpec = provisioner.allocateHost(hostAlias); - for (HostResource resource : hostname2host.values()) { - if (resource.getHostname().equals(hostSpec.hostname())) { - hostSpec.membership().ifPresent(resource::addClusterMembership); - return resource; - } - } - return addNewHost(hostSpec); + HostResource resource = hostname2host.get(hostSpec.hostname()); + return resource != null ? resource : addNewHost(hostSpec); } private HostResource addNewHost(HostSpec hostSpec) { Host host = Host.createHost(this, hostSpec.hostname()); HostResource hostResource = new HostResource(host, hostSpec); - hostSpec.membership().ifPresent(hostResource::addClusterMembership); hostSpec.networkPorts().ifPresent(np -> hostResource.ports().addNetworkPorts(np)); hostname2host.put(host.getHostname(), hostResource); return hostResource; @@ -177,15 +171,7 @@ public class HostSystem extends AbstractConfigProducer<Host> { } Set<HostSpec> getHostSpecs() { - return getHosts().stream() - .map(host -> new HostSpec(host.getHostname(), - Collections.emptyList(), - host.getFlavor(), - host.primaryClusterMembership(), - host.spec().version(), - host.ports().networkPorts(), - host.spec().requestedResources())) - .collect(Collectors.toCollection(LinkedHashSet::new)); + return getHosts().stream().map(host -> host.spec()).collect(Collectors.toCollection(LinkedHashSet::new)); } /** A provision logger which forwards to a deploy logger */ diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainer.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainer.java index c0dc029e009..719d3256889 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainer.java @@ -120,7 +120,7 @@ public class MetricsProxyContainer extends Container implements public void getConfig(NodeDimensionsConfig.Builder builder) { Map<String, String> dimensions = new LinkedHashMap<>(); if (isHostedVespa) { - getHostResource().primaryClusterMembership().map(ClusterMembership::cluster).ifPresent(cluster -> { + getHostResource().spec().membership().map(ClusterMembership::cluster).ifPresent(cluster -> { dimensions.put(CLUSTER_TYPE, cluster.type().name()); dimensions.put(CLUSTER_ID, cluster.id().value()); }); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/NodesSpecification.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/NodesSpecification.java index 86c72221cab..54850dedbba 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/NodesSpecification.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/NodesSpecification.java @@ -171,7 +171,8 @@ public class NodesSpecification { Optional.ofNullable(resources.stringAttribute("bandwidth")) .map(b -> parseGbAmount(b, "BPS")) .orElse(0.3), - parseOptionalDiskSpeed(resources.stringAttribute("disk-speed")))); + parseOptionalDiskSpeed(resources.stringAttribute("disk-speed")), + parseOptionalStorageType(resources.stringAttribute("storage-type")))); } else if (nodesElement.stringAttribute("flavor") != null) { // legacy fallback return Optional.of(NodeResources.fromLegacyName(nodesElement.stringAttribute("flavor"))); @@ -217,16 +218,27 @@ public class NodesSpecification { } private static NodeResources.DiskSpeed parseOptionalDiskSpeed(String diskSpeedString) { - if (diskSpeedString == null) return NodeResources.DiskSpeed.fast; + if (diskSpeedString == null) return NodeResources.DiskSpeed.getDefault(); switch (diskSpeedString) { case "fast" : return NodeResources.DiskSpeed.fast; case "slow" : return NodeResources.DiskSpeed.slow; - case "any" : return NodeResources.DiskSpeed.any; + case "any" : return NodeResources.DiskSpeed.any; default: throw new IllegalArgumentException("Illegal disk-speed value '" + diskSpeedString + "': Legal values are 'fast', 'slow' and 'any')"); } } + private static NodeResources.StorageType parseOptionalStorageType(String storageTypeString) { + if (storageTypeString == null) return NodeResources.StorageType.getDefault(); + switch (storageTypeString) { + case "remote" : return NodeResources.StorageType.remote; + case "local" : return NodeResources.StorageType.local; + case "any" : return NodeResources.StorageType.any; + default: throw new IllegalArgumentException("Illegal storage-type value '" + storageTypeString + + "': Legal values are 'remote', 'local' and 'any')"); + } + } + @Override public String toString() { return "specification of " + count + (dedicated ? " dedicated " : " ") + "nodes" + diff --git a/config-model/src/main/resources/schema/common.rnc b/config-model/src/main/resources/schema/common.rnc index c5690f9c915..e3ad942e7b3 100644 --- a/config-model/src/main/resources/schema/common.rnc +++ b/config-model/src/main/resources/schema/common.rnc @@ -27,7 +27,8 @@ Resources = element resources { attribute vcpu { xsd:double { minExclusive = "0.0" } } & attribute memory { xsd:string } & attribute disk { xsd:string } & - attribute disk-speed { xsd:string }? + attribute disk-speed { xsd:string }? & + attribute storage-type { xsd:string }? } OptionalDedicatedNodes = element nodes { diff --git a/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java b/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java index b8b50ba0eaa..93c3c9ea2ea 100644 --- a/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java +++ b/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java @@ -219,10 +219,9 @@ public class ModelProvisioningTest { assertEquals(1, model.getHostSystem().getHosts().size()); HostResource host = model.getHostSystem().getHosts().iterator().next(); - assertEquals(1, host.clusterMemberships().size()); - ClusterMembership membership = host.clusterMemberships().iterator().next(); - assertEquals("container", membership.cluster().type().name()); - assertEquals("container1", membership.cluster().id().value()); + assertTrue(host.spec().membership().isPresent()); + assertEquals("container", host.spec().membership().get().cluster().type().name()); + assertEquals("container1", host.spec().membership().get().cluster().id().value()); } @Test @@ -813,45 +812,51 @@ public class ModelProvisioningTest { "<?xml version='1.0' encoding='utf-8' ?>\n" + "<services>" + " <container version='1.0' id='container'>" + - " <nodes count='3' flavor='container-node'/>" + + " <nodes count='3'>" + + " <resources vcpu='1' memory='1Gb' disk='1Gb'/>" + + " </nodes>" + " </container>" + " <content version='1.0' id='content1'>" + " <redundancy>2</redundancy>" + " <documents>" + " <document type='type1' mode='index'/>" + " </documents>" + - " <nodes count='2' flavor='content1-node'/>" + + " <nodes count='2'>" + + " <resources vcpu='2' memory='2Gb' disk='2Gb'/>" + + " </nodes>" + " </content>" + " <content version='1.0' id='content2'>" + " <redundancy>2</redundancy>" + " <documents>" + " <document type='type1' mode='index'/>" + " </documents>" + - " <nodes count='2' flavor='content2-node'/>" + + " <nodes count='2'>" + + " <resources vcpu='4' memory='4Gb' disk='4Gb'/>" + + " </nodes>" + " </content>" + "</services>"; VespaModelTester tester = new VespaModelTester(); // use different flavors to make the test clearer - tester.addHosts("container-node", 3); - tester.addHosts("content1-node", 2); - tester.addHosts("content2-node", 2); + tester.addHosts(new NodeResources(1, 1, 1, 0.3), 3); + tester.addHosts(new NodeResources(2, 2, 2, 0.3), 2); + tester.addHosts(new NodeResources(4, 4, 4, 0.3), 2); VespaModel model = tester.createModel(services, true); ContentCluster cluster1 = model.getContentClusters().get("content1"); ClusterControllerContainerCluster clusterControllers1 = cluster1.getClusterControllers(); assertEquals(1, clusterControllers1.getContainers().size()); - assertEquals("content1-node0", clusterControllers1.getContainers().get(0).getHostName()); - assertEquals("content1-node1", clusterControllers1.getContainers().get(1).getHostName()); - assertEquals("container-node0", clusterControllers1.getContainers().get(2).getHostName()); + assertEquals("node-2-2-2-02", clusterControllers1.getContainers().get(0).getHostName()); + assertEquals("node-2-2-2-01", clusterControllers1.getContainers().get(1).getHostName()); + assertEquals("node-1-1-1-02", clusterControllers1.getContainers().get(2).getHostName()); ContentCluster cluster2 = model.getContentClusters().get("content2"); ClusterControllerContainerCluster clusterControllers2 = cluster2.getClusterControllers(); assertEquals(3, clusterControllers2.getContainers().size()); - assertEquals("content2-node0", clusterControllers2.getContainers().get(0).getHostName()); - assertEquals("content2-node1", clusterControllers2.getContainers().get(1).getHostName()); + assertEquals("node-4-4-4-02", clusterControllers2.getContainers().get(0).getHostName()); + assertEquals("node-4-4-4-01", clusterControllers2.getContainers().get(1).getHostName()); assertEquals("We do not pick the container used to supplement another cluster", - "container-node1", clusterControllers2.getContainers().get(2).getHostName()); + "node-1-1-1-01", clusterControllers2.getContainers().get(2).getHostName()); } @Test diff --git a/config-model/src/test/java/com/yahoo/vespa/model/HostResourceTest.java b/config-model/src/test/java/com/yahoo/vespa/model/HostResourceTest.java index 0624c2cd23a..234841f2b6c 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/HostResourceTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/HostResourceTest.java @@ -6,9 +6,11 @@ import com.yahoo.config.model.producer.AbstractConfigProducer; import com.yahoo.config.model.test.MockRoot; import com.yahoo.config.provision.ClusterMembership; import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.HostSpec; import org.junit.Test; import java.util.Arrays; +import java.util.Optional; import static com.yahoo.config.provision.ClusterSpec.Type.admin; import static com.yahoo.config.provision.ClusterSpec.Type.container; @@ -38,50 +40,13 @@ public class HostResourceTest { } @Test - public void no_clusters_yields_no_primary_cluster_membership() { - HostResource host = hostResourceWithMemberships(); - assertTrue(host.clusterMemberships().isEmpty()); - - assertFalse(host.primaryClusterMembership().isPresent()); - } - - @Test - public void one_cluster_yields_that_primary_cluster_membership() { + public void host_witrh_membership() { HostResource host = hostResourceWithMemberships(ClusterMembership.from(clusterSpec(container, "jdisc"), 0)); assertClusterMembership(host, container, "jdisc"); } - @Test - public void content_cluster_membership_is_preferred_over_other_types() { - HostResource host = hostResourceWithMemberships( - ClusterMembership.from(clusterSpec(container, "jdisc"), 0), - ClusterMembership.from(clusterSpec(content, "search"), 0), - ClusterMembership.from(clusterSpec(admin, "admin"), 0)); - - assertClusterMembership(host, content, "search"); - } - - @Test - public void container_cluster_membership_is_preferred_over_admin() { - HostResource host = hostResourceWithMemberships( - ClusterMembership.from(clusterSpec(admin, "admin"), 0), - ClusterMembership.from(clusterSpec(container, "jdisc"), 0)); - - assertClusterMembership(host, container, "jdisc"); - } - - @Test - public void cluster_membership_that_was_added_first_is_preferred() { - HostResource host = hostResourceWithMemberships( - ClusterMembership.from(clusterSpec(content, "content1"), 0), - ClusterMembership.from(clusterSpec(content, "content0"), 0), - ClusterMembership.from(clusterSpec(content, "content2"), 0)); - - assertClusterMembership(host, content, "content1"); - } - private void assertClusterMembership(HostResource host, ClusterSpec.Type type, String id) { - ClusterSpec membership = host.primaryClusterMembership().map(ClusterMembership::cluster) + ClusterSpec membership = host.spec().membership().map(ClusterMembership::cluster) .orElseThrow(() -> new RuntimeException("No cluster membership!")); assertEquals(type, membership.type()); @@ -92,14 +57,9 @@ public class HostResourceTest { return ClusterSpec.from(type, ClusterSpec.Id.from(id), ClusterSpec.Group.from(0), Version.fromString("6.42"), false); } - private HostResource mockHostResource(MockRoot root) { - return new HostResource(new Host(root)); - } - - private static HostResource hostResourceWithMemberships(ClusterMembership... memberships) { - HostResource host = new HostResource(Host.createHost(null, "hostname")); - Arrays.asList(memberships).forEach(host::addClusterMembership); - return host; + private static HostResource hostResourceWithMemberships(ClusterMembership membership) { + return new HostResource(Host.createHost(null, "hostname"), + new HostSpec("hostname", Optional.of(membership))); } private static int counter = 0; diff --git a/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTester.java b/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTester.java index 63269c45e5f..b6180ab78b9 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTester.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTester.java @@ -60,15 +60,6 @@ public class VespaModelTester { /** Adds some nodes with resources 1, 3, 9 */ public Hosts addHosts(int count) { return addHosts(new NodeResources(1, 3, 9, 1), count); } - /** Adds some hosts to this system */ - public Hosts addHosts(String flavor, int count) { - return addHosts(Optional.empty(), NodeResources.fromLegacyName(flavor), count); - } - - public Hosts addHosts(Flavor flavor, int count) { - return addHosts(Optional.of(flavor), NodeResources.fromLegacyName(flavor.name()), count); - } - public Hosts addHosts(NodeResources resources, int count) { return addHosts(Optional.of(new Flavor(resources)), resources, count); } diff --git a/config-provisioning/abi-spec.json b/config-provisioning/abi-spec.json index c3e3a2b74f9..61cbc7da52d 100644 --- a/config-provisioning/abi-spec.json +++ b/config-provisioning/abi-spec.json @@ -485,6 +485,7 @@ "public java.util.Optional membership()", "public java.util.Optional networkPorts()", "public java.util.Optional requestedResources()", + "public com.yahoo.config.provision.HostSpec withPorts(java.util.Optional)", "public java.lang.String toString()", "public boolean equals(java.lang.Object)", "public int hashCode()", @@ -590,7 +591,9 @@ "methods": [ "public static com.yahoo.config.provision.NodeResources$DiskSpeed[] values()", "public static com.yahoo.config.provision.NodeResources$DiskSpeed valueOf(java.lang.String)", - "public static int compare(com.yahoo.config.provision.NodeResources$DiskSpeed, com.yahoo.config.provision.NodeResources$DiskSpeed)" + "public static int compare(com.yahoo.config.provision.NodeResources$DiskSpeed, com.yahoo.config.provision.NodeResources$DiskSpeed)", + "public boolean isDefault()", + "public static com.yahoo.config.provision.NodeResources$DiskSpeed getDefault()" ], "fields": [ "public static final enum com.yahoo.config.provision.NodeResources$DiskSpeed fast", @@ -598,6 +601,27 @@ "public static final enum com.yahoo.config.provision.NodeResources$DiskSpeed any" ] }, + "com.yahoo.config.provision.NodeResources$StorageType": { + "superClass": "java.lang.Enum", + "interfaces": [], + "attributes": [ + "public", + "final", + "enum" + ], + "methods": [ + "public static com.yahoo.config.provision.NodeResources$StorageType[] values()", + "public static com.yahoo.config.provision.NodeResources$StorageType valueOf(java.lang.String)", + "public static int compare(com.yahoo.config.provision.NodeResources$StorageType, com.yahoo.config.provision.NodeResources$StorageType)", + "public boolean isDefault()", + "public static com.yahoo.config.provision.NodeResources$StorageType getDefault()" + ], + "fields": [ + "public static final enum com.yahoo.config.provision.NodeResources$StorageType remote", + "public static final enum com.yahoo.config.provision.NodeResources$StorageType local", + "public static final enum com.yahoo.config.provision.NodeResources$StorageType any" + ] + }, "com.yahoo.config.provision.NodeResources": { "superClass": "java.lang.Object", "interfaces": [], @@ -605,21 +629,23 @@ "public" ], "methods": [ - "public void <init>(double, double, double)", - "public void <init>(double, double, double, com.yahoo.config.provision.NodeResources$DiskSpeed)", "public void <init>(double, double, double, double)", "public void <init>(double, double, double, double, com.yahoo.config.provision.NodeResources$DiskSpeed)", + "public void <init>(double, double, double, double, com.yahoo.config.provision.NodeResources$DiskSpeed, com.yahoo.config.provision.NodeResources$StorageType)", "public double vcpu()", "public double memoryGb()", "public double diskGb()", "public double bandwidthGbps()", "public com.yahoo.config.provision.NodeResources$DiskSpeed diskSpeed()", + "public com.yahoo.config.provision.NodeResources$StorageType storageType()", "public com.yahoo.config.provision.NodeResources withVcpu(double)", "public com.yahoo.config.provision.NodeResources withMemoryGb(double)", "public com.yahoo.config.provision.NodeResources withDiskGb(double)", "public com.yahoo.config.provision.NodeResources withBandwidthGbps(double)", "public com.yahoo.config.provision.NodeResources withDiskSpeed(com.yahoo.config.provision.NodeResources$DiskSpeed)", - "public com.yahoo.config.provision.NodeResources anySpeed()", + "public com.yahoo.config.provision.NodeResources with(com.yahoo.config.provision.NodeResources$DiskSpeed)", + "public com.yahoo.config.provision.NodeResources with(com.yahoo.config.provision.NodeResources$StorageType)", + "public com.yahoo.config.provision.NodeResources justNumbers()", "public com.yahoo.config.provision.NodeResources subtract(com.yahoo.config.provision.NodeResources)", "public com.yahoo.config.provision.NodeResources add(com.yahoo.config.provision.NodeResources)", "public boolean equals(java.lang.Object)", diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/Flavor.java b/config-provisioning/src/main/java/com/yahoo/config/provision/Flavor.java index bb4aca0e34b..7a4df8febcf 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/Flavor.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/Flavor.java @@ -38,7 +38,8 @@ public class Flavor { flavorConfig.minMainMemoryAvailableGb(), flavorConfig.minDiskAvailableGb(), flavorConfig.bandwidth() / 1000, - flavorConfig.fastDisk() ? NodeResources.DiskSpeed.fast : NodeResources.DiskSpeed.slow), + flavorConfig.fastDisk() ? NodeResources.DiskSpeed.fast : NodeResources.DiskSpeed.slow, + flavorConfig.remoteStorage() ? NodeResources.StorageType.remote : NodeResources.StorageType.local), Optional.empty(), Type.valueOf(flavorConfig.environment()), true, @@ -71,11 +72,7 @@ public class Flavor { if (!configured) throw new IllegalArgumentException("Cannot override non-configured flavor"); - NodeResources newResources = new NodeResources(resources.vcpu(), - resources.memoryGb(), - flavorOverrides.diskGb().orElseGet(resources::diskGb), - resources.bandwidthGbps(), - resources.diskSpeed()); + NodeResources newResources = resources.withDiskGb(flavorOverrides.diskGb().orElseGet(resources::diskGb)); return new Flavor(name, newResources, Optional.of(flavorOverrides), type, true, cost, minCpuCores); } diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/HostSpec.java b/config-provisioning/src/main/java/com/yahoo/config/provision/HostSpec.java index 691107f4649..63725d9a535 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/HostSpec.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/HostSpec.java @@ -29,7 +29,7 @@ public class HostSpec implements Comparable<HostSpec> { private final Optional<NetworkPorts> networkPorts; - private Optional<NodeResources> requestedResources; + private final Optional<NodeResources> requestedResources; public HostSpec(String hostname, Optional<ClusterMembership> membership) { this(hostname, new ArrayList<>(), Optional.empty(), membership); @@ -99,6 +99,10 @@ public class HostSpec implements Comparable<HostSpec> { /** Returns the requested resources leading to this host being provisioned, or empty if not known */ public Optional<NodeResources> requestedResources() { return requestedResources; } + public HostSpec withPorts(Optional<NetworkPorts> ports) { + return new HostSpec(hostname, aliases, flavor, membership, version, ports, requestedResources); + } + @Override public String toString() { return hostname + diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/NodeResources.java b/config-provisioning/src/main/java/com/yahoo/config/provision/NodeResources.java index 66baecf6c82..7769060dc60 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/NodeResources.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/NodeResources.java @@ -12,9 +12,9 @@ public class NodeResources { public enum DiskSpeed { - fast, // SSD disk or similar speed is needed - slow, // This is tuned to work with the speed of spinning disks - any; // The performance of the cluster using this does not depend on disk speed + fast, // Has/requires SSD disk or similar speed + slow, // Has spinning disk/Is tuned to work with the speed of spinning disks + any; // In requests only: The performance of the cluster using this does not depend on disk speed /** * Compares disk speeds by cost: Slower is cheaper, and therefore before. @@ -29,6 +29,47 @@ public class NodeResources { return 0; } + private DiskSpeed combineWith(DiskSpeed other) { + if (this == any) return other; + if (other == any) return this; + if (this == other) return this; + throw new IllegalArgumentException(this + " cannot be combined with " + other); + } + + public boolean isDefault() { return this == getDefault(); } + public static DiskSpeed getDefault() { return fast; } + + } + + public enum StorageType { + + remote, // Has remote (network) storage/Is tuned to work with network storage + local, // Storage is/must be attached to the local host + any; // In requests only: Can use both local and remote storage + + /** + * Compares storage type by cost: Remote is cheaper, and therefore before. + * Any can be remote and therefore costs the same as remote. + */ + public static int compare(StorageType a, StorageType b) { + if (a == any) a = remote; + if (b == any) b = remote; + + if (a == remote && b == local) return -1; + if (a == local && b == remote) return 1; + return 0; + } + + private StorageType combineWith(StorageType other) { + if (this == any) return other; + if (other == any) return this; + if (this == other) return this; + throw new IllegalArgumentException(this + " cannot be combined with " + other); + } + + public boolean isDefault() { return this == getDefault(); } + public static StorageType getDefault() { return any; } + } private final double vcpu; @@ -36,30 +77,23 @@ public class NodeResources { private final double diskGb; private final double bandwidthGbps; private final DiskSpeed diskSpeed; + private final StorageType storageType; - /** Create node resources requiring fast disk and no bandwidth */ - @Deprecated // Remove Oct. 2019 - public NodeResources(double vcpu, double memoryGb, double diskGb) { - this(vcpu, memoryGb, diskGb, DiskSpeed.fast); - } - - /** Create node resources requiring no bandwidth */ - @Deprecated // Remove Oct. 2019 - public NodeResources(double vcpu, double memoryGb, double diskGb, DiskSpeed diskSpeed) { - this(vcpu, memoryGb, diskGb, 0.3, diskSpeed); - } - - /** Create node resources requiring fast disk */ public NodeResources(double vcpu, double memoryGb, double diskGb, double bandwidthGbps) { - this(vcpu, memoryGb, diskGb, bandwidthGbps, DiskSpeed.fast); + this(vcpu, memoryGb, diskGb, bandwidthGbps, DiskSpeed.getDefault()); } public NodeResources(double vcpu, double memoryGb, double diskGb, double bandwidthGbps, DiskSpeed diskSpeed) { + this(vcpu, memoryGb, diskGb, bandwidthGbps, diskSpeed, StorageType.getDefault()); + } + + public NodeResources(double vcpu, double memoryGb, double diskGb, double bandwidthGbps, DiskSpeed diskSpeed, StorageType storageType) { this.vcpu = vcpu; this.memoryGb = memoryGb; this.diskGb = diskGb; this.bandwidthGbps = bandwidthGbps; this.diskSpeed = diskSpeed; + this.storageType = storageType; } public double vcpu() { return vcpu; } @@ -67,30 +101,40 @@ public class NodeResources { public double diskGb() { return diskGb; } public double bandwidthGbps() { return bandwidthGbps; } public DiskSpeed diskSpeed() { return diskSpeed; } + public StorageType storageType() { return storageType; } public NodeResources withVcpu(double vcpu) { - return new NodeResources(vcpu, memoryGb, diskGb, bandwidthGbps, diskSpeed); + return new NodeResources(vcpu, memoryGb, diskGb, bandwidthGbps, diskSpeed, storageType); } public NodeResources withMemoryGb(double memoryGb) { - return new NodeResources(vcpu, memoryGb, diskGb, bandwidthGbps, diskSpeed); + return new NodeResources(vcpu, memoryGb, diskGb, bandwidthGbps, diskSpeed, storageType); } public NodeResources withDiskGb(double diskGb) { - return new NodeResources(vcpu, memoryGb, diskGb, bandwidthGbps, diskSpeed); + return new NodeResources(vcpu, memoryGb, diskGb, bandwidthGbps, diskSpeed, storageType); } public NodeResources withBandwidthGbps(double bandwidthGbps) { - return new NodeResources(vcpu, memoryGb, diskGb, bandwidthGbps, diskSpeed); + return new NodeResources(vcpu, memoryGb, diskGb, bandwidthGbps, diskSpeed, storageType); } + // TODO: Remove after November 2019 public NodeResources withDiskSpeed(DiskSpeed speed) { - return new NodeResources(vcpu, memoryGb, diskGb, bandwidthGbps, speed); + return new NodeResources(vcpu, memoryGb, diskGb, bandwidthGbps, speed, storageType); } - /** A shorthand for withDiskSpeed(NodeResources.DiskSpeed.any) */ - public NodeResources anySpeed() { - return withDiskSpeed(NodeResources.DiskSpeed.any); + public NodeResources with(DiskSpeed speed) { + return new NodeResources(vcpu, memoryGb, diskGb, bandwidthGbps, speed, storageType); + } + + public NodeResources with(StorageType storageType) { + return new NodeResources(vcpu, memoryGb, diskGb, bandwidthGbps, diskSpeed, storageType); + } + + /** Returns this with disk speed and storage type set to any */ + public NodeResources justNumbers() { + return with(NodeResources.DiskSpeed.any).with(StorageType.any); } public NodeResources subtract(NodeResources other) { @@ -100,7 +144,8 @@ public class NodeResources { memoryGb - other.memoryGb, diskGb - other.diskGb, bandwidthGbps - other.bandwidthGbps, - combine(this.diskSpeed, other.diskSpeed)); + this.diskSpeed.combineWith(other.diskSpeed), + this.storageType.combineWith(other.storageType)); } public NodeResources add(NodeResources other) { @@ -110,22 +155,18 @@ public class NodeResources { memoryGb + other.memoryGb, diskGb + other.diskGb, bandwidthGbps + other.bandwidthGbps, - combine(this.diskSpeed, other.diskSpeed)); + this.diskSpeed.combineWith(other.diskSpeed), + this.storageType.combineWith(other.storageType)); } private boolean isInterchangeableWith(NodeResources other) { if (this.diskSpeed != DiskSpeed.any && other.diskSpeed != DiskSpeed.any && this.diskSpeed != other.diskSpeed) return false; + if (this.storageType != StorageType.any && other.storageType != StorageType.any && this.storageType != other.storageType) + return false; return true; } - private DiskSpeed combine(DiskSpeed a, DiskSpeed b) { - if (a == DiskSpeed.any) return b; - if (b == DiskSpeed.any) return a; - if (a == b) return a; - throw new IllegalArgumentException(a + " cannot be combined with " + b); - } - @Override public boolean equals(Object o) { if (o == this) return true; @@ -136,19 +177,21 @@ public class NodeResources { if (this.diskGb != other.diskGb) return false; if (this.bandwidthGbps != other.bandwidthGbps) return false; if (this.diskSpeed != other.diskSpeed) return false; + if (this.storageType != other.storageType) return false; return true; } @Override public int hashCode() { - return Objects.hash(vcpu, memoryGb, diskGb, bandwidthGbps, diskSpeed); + return Objects.hash(vcpu, memoryGb, diskGb, bandwidthGbps, diskSpeed, storageType); } @Override public String toString() { return "[vcpu: " + vcpu + ", memory: " + memoryGb + " Gb, disk " + diskGb + " Gb" + (bandwidthGbps > 0 ? ", bandwidth: " + bandwidthGbps + " Gbps" : "") + - (diskSpeed != DiskSpeed.fast ? ", disk speed: " + diskSpeed : "") + "]"; + ( ! diskSpeed.isDefault() ? ", disk speed: " + diskSpeed : "") + + ( ! storageType.isDefault() ? ", storage type: " + storageType : "") + "]"; } /** Returns true if all the resources of this are the same or larger than the given resources */ @@ -163,6 +206,9 @@ public class NodeResources { // draw conclusions about performance on the basis of better resources than you think you have if (other.diskSpeed != DiskSpeed.any && other.diskSpeed != this.diskSpeed) return false; + // Same reasoning as the above + if (other.storageType != StorageType.any && other.storageType != this.storageType) return false; + return true; } @@ -173,6 +219,7 @@ public class NodeResources { if (this.diskGb != other.diskGb) return false; if (this.bandwidthGbps != other.bandwidthGbps) return false; if (other.diskSpeed != DiskSpeed.any && other.diskSpeed != this.diskSpeed) return false; + if (other.storageType != StorageType.any && other.storageType != this.storageType) return false; return true; } @@ -195,7 +242,7 @@ public class NodeResources { if (cpu == 0) cpu = 0.5; if (cpu == 2 && mem == 8 ) cpu = 1.5; if (cpu == 2 && mem == 12 ) cpu = 2.3; - return new NodeResources(cpu, mem, dsk, 0.3, DiskSpeed.fast); + return new NodeResources(cpu, mem, dsk, 0.3, DiskSpeed.getDefault(), StorageType.getDefault()); } } diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/serialization/AllocatedHostsSerializer.java b/config-provisioning/src/main/java/com/yahoo/config/provision/serialization/AllocatedHostsSerializer.java index 4a060fb5143..413e277655a 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/serialization/AllocatedHostsSerializer.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/serialization/AllocatedHostsSerializer.java @@ -55,6 +55,7 @@ public class AllocatedHostsSerializer { private static final String diskKey = "disk"; private static final String bandwidthKey = "bandwidth"; private static final String diskSpeedKey = "diskSpeed"; + private static final String storageTypeKey = "storageType"; /** Wanted version */ private static final String hostSpecVespaVersionKey = "vespaVersion"; @@ -109,6 +110,7 @@ public class AllocatedHostsSerializer { resourcesObject.setDouble(diskKey, resources.diskGb()); resourcesObject.setDouble(bandwidthKey, resources.bandwidthGbps()); resourcesObject.setString(diskSpeedKey, diskSpeedToString(resources.diskSpeed())); + resourcesObject.setString(storageTypeKey, storageTypeToString(resources.storageType())); } public static AllocatedHosts fromJson(byte[] json, Optional<NodeFlavors> nodeFlavors) { @@ -152,7 +154,8 @@ public class AllocatedHostsSerializer { resources.field(memoryKey).asDouble(), resources.field(diskKey).asDouble(), resources.field(bandwidthKey).asDouble(), - diskSpeedFromSlime(resources.field(diskSpeedKey)))); + diskSpeedFromSlime(resources.field(diskSpeedKey)), + storageTypeFromSlime(resources.field(storageTypeKey)))); } private static NodeResources.DiskSpeed diskSpeedFromSlime(Inspector diskSpeed) { @@ -171,7 +174,25 @@ public class AllocatedHostsSerializer { case any : return "any"; default: throw new IllegalStateException("Illegal disk-speed value '" + diskSpeed + "'"); } + } + + private static NodeResources.StorageType storageTypeFromSlime(Inspector storageType) { + if ( ! storageType.valid()) return NodeResources.StorageType.getDefault(); // TODO: Remove this line after December 2019 + switch (storageType.asString()) { + case "remote" : return NodeResources.StorageType.remote; + case "local" : return NodeResources.StorageType.local; + case "any" : return NodeResources.StorageType.any; + default: throw new IllegalStateException("Illegal storage-type value '" + storageType.asString() + "'"); + } + } + private static String storageTypeToString(NodeResources.StorageType storageType) { + switch (storageType) { + case remote : return "remote"; + case local : return "local"; + case any : return "any"; + default: throw new IllegalStateException("Illegal storage-type value '" + storageType + "'"); + } } private static ClusterMembership membershipFromSlime(Inspector object) { diff --git a/config-provisioning/src/main/resources/configdefinitions/flavors.def b/config-provisioning/src/main/resources/configdefinitions/flavors.def index dbdca724158..c5040c58b74 100644 --- a/config-provisioning/src/main/resources/configdefinitions/flavors.def +++ b/config-provisioning/src/main/resources/configdefinitions/flavors.def @@ -30,5 +30,8 @@ flavor[].minDiskAvailableGb double default=0.0 # Whether the disk is fast (typically SSD) or slow (typically spinning HDD). flavor[].fastDisk bool default=true +# Whether the storage is remote (network) or local. +flavor[].remoteStorage bool default=true + # Expected network interface bandwidth available for this flavor, in Mbit/s. flavor[].bandwidth double default=0.0 diff --git a/configgen/src/main/java/com/yahoo/config/codegen/DefLine.java b/configgen/src/main/java/com/yahoo/config/codegen/DefLine.java index 5d2dc2e13e8..2cc93f95df0 100644 --- a/configgen/src/main/java/com/yahoo/config/codegen/DefLine.java +++ b/configgen/src/main/java/com/yahoo/config/codegen/DefLine.java @@ -257,16 +257,18 @@ public class DefLine { } } - void validateReservedWords() { - if (ReservedWords.isReservedWord(name)) { - throw new IllegalArgumentException(name + " is a reserved word in " + - ReservedWords.getLanguageForReservedWord(name)); + private void validateReservedWords() { + String cleanName = (name.endsWith("[]") || name.endsWith("{}")) ? name.substring(0, name.length()-2) : name; + + if (ReservedWords.isReservedWord(cleanName)) { + throw new IllegalArgumentException(cleanName + " is a reserved word in " + + ReservedWords.getLanguageForReservedWord(cleanName)); } - if (ReservedWords.capitalizedPattern.matcher(name).matches()) { - throw new IllegalArgumentException("'" + name + "' cannot start with an uppercase letter"); + if (ReservedWords.capitalizedPattern.matcher(cleanName).matches()) { + throw new IllegalArgumentException("'" + cleanName + "' cannot start with an uppercase letter"); } - if (ReservedWords.internalPrefixPattern.matcher(name).matches()) { - throw new IllegalArgumentException("'" + name + "' cannot start with '" + ReservedWords.INTERNAL_PREFIX + "'"); + if (ReservedWords.internalPrefixPattern.matcher(cleanName).matches()) { + throw new IllegalArgumentException("'" + cleanName + "' cannot start with '" + ReservedWords.INTERNAL_PREFIX + "'"); } } diff --git a/configgen/src/test/java/com/yahoo/config/codegen/DefParserTest.java b/configgen/src/test/java/com/yahoo/config/codegen/DefParserTest.java index 98c30aa09cf..1c381e7c398 100644 --- a/configgen/src/test/java/com/yahoo/config/codegen/DefParserTest.java +++ b/configgen/src/test/java/com/yahoo/config/codegen/DefParserTest.java @@ -257,17 +257,35 @@ public class DefParserTest { } @Test + public void testReservedWordInCForArray() { + assertLineFails("auto[] int", + "auto is a reserved word in C"); + } + + @Test public void testReservedWordInJava() { assertLineFails("abstract int", "abstract is a reserved word in Java"); } @Test + public void testReservedWordInJavaForMap() { + assertLineFails("abstract{} int", + "abstract is a reserved word in Java"); + } + + @Test public void testReservedWordInCAndJava() { assertLineFails("continue int", "continue is a reserved word in C and Java"); } + @Test + public void testReservedWordInCAndJavaForArray() { + assertLineFails("continue[] int", + "continue is a reserved word in C and Java"); + } + static StringBuilder createDefTemplate() { StringBuilder sb = new StringBuilder(); sb.append("version=8\n"); diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastSearcher.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastSearcher.java index 1a373b6ea71..c04553ae2f5 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastSearcher.java @@ -147,7 +147,7 @@ public class FastSearcher extends VespaBackEndSearcher { * on the same host. */ private SearchInvoker getSearchInvoker(Query query) { - return dispatcher.getSearchInvoker(query, this).get(); + return dispatcher.getSearchInvoker(query, this); } /** @@ -156,11 +156,9 @@ public class FastSearcher extends VespaBackEndSearcher { * content nodes. */ private FillInvoker getFillInvoker(Result result) { - return dispatcher.getFillInvoker(result, this).get(); + return dispatcher.getFillInvoker(result, this); } - - private static Optional<String> quotedSummaryClass(String summaryClass) { return Optional.of(summaryClass == null ? "[null]" : quote(summaryClass)); } diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java b/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java index 1f58a2df5c2..ab010001d90 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java @@ -49,7 +49,6 @@ public class Dispatcher extends AbstractComponent { private static final String INTERNAL = "internal"; private static final String PROTOBUF = "protobuf"; - private static final String FDISPATCH_METRIC = "dispatch_fdispatch"; private static final String INTERNAL_METRIC = "dispatch_internal"; private static final int MAX_GROUP_SELECTION_ATTEMPTS = 3; @@ -61,7 +60,6 @@ public class Dispatcher extends AbstractComponent { private final SearchCluster searchCluster; private final LoadBalancer loadBalancer; - private final boolean multilevelDispatch; private final InvokerFactory invokerFactory; @@ -115,11 +113,13 @@ public class Dispatcher extends AbstractComponent { InvokerFactory invokerFactory, PingFactory pingFactory, Metric metric) { + if (dispatchConfig.useMultilevelDispatch()) + throw new IllegalArgumentException(searchCluster + " is configured with multilevel dispatch, but this is not supported"); + this.searchCluster = searchCluster; this.loadBalancer = new LoadBalancer(searchCluster, dispatchConfig.distributionPolicy() == DispatchConfig.DistributionPolicy.ROUNDROBIN); this.invokerFactory = invokerFactory; - this.multilevelDispatch = dispatchConfig.useMultilevelDispatch(); this.metric = metric; this.metricContext = metric.createContext(null); @@ -137,27 +137,18 @@ public class Dispatcher extends AbstractComponent { searchCluster.shutDown(); } - public Optional<FillInvoker> getFillInvoker(Result result, VespaBackEndSearcher searcher) { + public FillInvoker getFillInvoker(Result result, VespaBackEndSearcher searcher) { return invokerFactory.createFillInvoker(searcher, result); } - public Optional<SearchInvoker> getSearchInvoker(Query query, VespaBackEndSearcher searcher) { - if (multilevelDispatch) { - emitDispatchMetric(Optional.empty()); - return Optional.empty(); - } - - Optional<SearchInvoker> invoker = getSearchPathInvoker(query, searcher); + public SearchInvoker getSearchInvoker(Query query, VespaBackEndSearcher searcher) { + SearchInvoker invoker = getSearchPathInvoker(query, searcher).orElseGet(() -> getInternalInvoker(query, searcher)); - if (invoker.isEmpty()) { - invoker = getInternalInvoker(query, searcher); - } - if (invoker.isPresent() && query.properties().getBoolean(com.yahoo.search.query.Model.ESTIMATE)) { + if (query.properties().getBoolean(com.yahoo.search.query.Model.ESTIMATE)) { query.setHits(0); query.setOffset(0); } - emitDispatchMetric(invoker); - + metric.add(INTERNAL_METRIC, 1, metricContext); return invoker; } @@ -177,12 +168,13 @@ public class Dispatcher extends AbstractComponent { } } - private Optional<SearchInvoker> getInternalInvoker(Query query, VespaBackEndSearcher searcher) { + private SearchInvoker getInternalInvoker(Query query, VespaBackEndSearcher searcher) { Optional<Node> directNode = searchCluster.localCorpusDispatchTarget(); if (directNode.isPresent()) { Node node = directNode.get(); - query.trace(false, 2, "Dispatching directly to ", node); - return invokerFactory.createSearchInvoker(searcher, query, OptionalInt.empty(), Arrays.asList(node), true); + query.trace(false, 2, "Dispatching to ", node); + return invokerFactory.createSearchInvoker(searcher, query, OptionalInt.empty(), Arrays.asList(node), true) + .orElseThrow(() -> new IllegalStateException("Could not dispatch directly to " + node)); } int covered = searchCluster.groupsWithSufficientCoverage(); @@ -201,10 +193,10 @@ public class Dispatcher extends AbstractComponent { group.nodes(), acceptIncompleteCoverage); if (invoker.isPresent()) { - query.trace(false, 2, "Dispatching internally to search group ", group.id()); + query.trace(false, 2, "Dispatching to group ", group.id()); query.getModel().setSearchPath("/" + group.id()); invoker.get().teardown((success, time) -> loadBalancer.releaseGroup(group, success, time)); - return invoker; + return invoker.get(); } else { loadBalancer.releaseGroup(group, false, 0); if (rejected == null) { @@ -213,16 +205,7 @@ public class Dispatcher extends AbstractComponent { rejected.add(group.id()); } } - - return Optional.empty(); - } - - private void emitDispatchMetric(Optional<SearchInvoker> invoker) { - if (invoker.isEmpty()) { - metric.add(FDISPATCH_METRIC, 1, metricContext); - } else { - metric.add(INTERNAL_METRIC, 1, metricContext); - } + throw new IllegalStateException("No suitable groups to dispatch query. Rejected: " + rejected); } } diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/InvokerFactory.java b/container-search/src/main/java/com/yahoo/search/dispatch/InvokerFactory.java index 78c50254a84..6030e989595 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/InvokerFactory.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/InvokerFactory.java @@ -20,6 +20,7 @@ import java.util.Set; * @author ollivir */ public abstract class InvokerFactory { + protected final SearchCluster searchCluster; public InvokerFactory(SearchCluster searchCluster) { @@ -28,24 +29,18 @@ public abstract class InvokerFactory { protected abstract Optional<SearchInvoker> createNodeSearchInvoker(VespaBackEndSearcher searcher, Query query, Node node); - public abstract Optional<FillInvoker> createFillInvoker(VespaBackEndSearcher searcher, Result result); + public abstract FillInvoker createFillInvoker(VespaBackEndSearcher searcher, Result result); /** * Create a {@link SearchInvoker} for a list of content nodes. * - * @param searcher - * the searcher processing the query - * @param query - * the search query being processed - * @param groupId - * the id of the node group to which the nodes belong - * @param nodes - * pre-selected list of content nodes - * @param acceptIncompleteCoverage - * if some of the nodes are unavailable and this parameter is - * <b>false</b>, verify that the remaining set of nodes has enough - * coverage - * @return Optional containing the SearchInvoker or <i>empty</i> if some node in the + * @param searcher the searcher processing the query + * @param query the search query being processed + * @param groupId the id of the node group to which the nodes belong + * @param nodes pre-selected list of content nodes + * @param acceptIncompleteCoverage if some of the nodes are unavailable and this parameter is + * false, verify that the remaining set of nodes has sufficient coverage + * @return Optional containing the SearchInvoker or empty if some node in the * list is invalid and the remaining coverage is not sufficient */ public Optional<SearchInvoker> createSearchInvoker(VespaBackEndSearcher searcher, @@ -76,11 +71,11 @@ public abstract class InvokerFactory { if (failed != null) { List<Node> success = new ArrayList<>(nodes.size() - failed.size()); for (Node node : nodes) { - if (!failed.contains(node.key())) { + if ( ! failed.contains(node.key())) { success.add(node); } } - if (!searchCluster.isPartialGroupCoverageSufficient(groupId, success) && !acceptIncompleteCoverage) { + if ( ! searchCluster.isPartialGroupCoverageSufficient(groupId, success) && !acceptIncompleteCoverage) { return Optional.empty(); } if(invokers.size() == 0) { @@ -113,4 +108,5 @@ public abstract class InvokerFactory { } public void release() {} + } diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcInvokerFactory.java b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcInvokerFactory.java index 6160d3dfa08..870f7aef9c5 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcInvokerFactory.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcInvokerFactory.java @@ -40,7 +40,7 @@ public class RpcInvokerFactory extends InvokerFactory implements PingFactory { } @Override - public Optional<FillInvoker> createFillInvoker(VespaBackEndSearcher searcher, Result result) { + public FillInvoker createFillInvoker(VespaBackEndSearcher searcher, Result result) { Query query = result.getQuery(); boolean summaryNeedsQuery = searcher.summaryNeedsQuery(query); @@ -48,8 +48,8 @@ public class RpcInvokerFactory extends InvokerFactory implements PingFactory { boolean useDispatchDotSummaries = query.properties().getBoolean(dispatchSummaries, false); return ((useDispatchDotSummaries || !useProtoBuf) && ! summaryNeedsQuery) - ? Optional.of(new RpcFillInvoker(rpcResourcePool, searcher.getDocumentDatabase(query))) - : Optional.of(new RpcProtobufFillInvoker(rpcResourcePool, searcher.getDocumentDatabase(query), searcher.getServerId(), summaryNeedsQuery)); + ? new RpcFillInvoker(rpcResourcePool, searcher.getDocumentDatabase(query)) + : new RpcProtobufFillInvoker(rpcResourcePool, searcher.getDocumentDatabase(query), searcher.getServerId(), summaryNeedsQuery); } // for testing diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/DispatcherTest.java b/container-search/src/test/java/com/yahoo/search/dispatch/DispatcherTest.java index 30f6c5a495d..291b0f4890a 100644 --- a/container-search/src/test/java/com/yahoo/search/dispatch/DispatcherTest.java +++ b/container-search/src/test/java/com/yahoo/search/dispatch/DispatcherTest.java @@ -11,7 +11,6 @@ import com.yahoo.search.cluster.ClusterMonitor; import com.yahoo.search.dispatch.searchcluster.Node; import com.yahoo.search.dispatch.searchcluster.PingFactory; import com.yahoo.search.dispatch.searchcluster.SearchCluster; -import com.yahoo.vespa.config.search.DispatchConfig; import org.junit.Test; import java.util.List; @@ -20,46 +19,28 @@ import java.util.OptionalInt; import java.util.concurrent.Callable; import static com.yahoo.search.dispatch.MockSearchCluster.createDispatchConfig; -import static org.hamcrest.Matchers.is; -import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; /** * @author ollivir */ public class DispatcherTest { - private static final CompoundName internalDispatch = new CompoundName("dispatch.internal"); - - private static Query query() { - Query q = new Query(); - q.properties().set(internalDispatch, "true"); - return q; - } - - @Test - public void requireDispatcherToIgnoreMultilevelConfigurations() { - SearchCluster searchCluster = new MockSearchCluster("1", 2, 2); - DispatchConfig dispatchConfig = new DispatchConfig.Builder().useMultilevelDispatch(true).build(); - - var invokerFactory = new MockInvokerFactory(searchCluster); - - Dispatcher disp = new Dispatcher(searchCluster, dispatchConfig, invokerFactory, invokerFactory, new MockMetric()); - assertThat(disp.getSearchInvoker(query(), null).isPresent(), is(false)); - } @Test public void requireThatDispatcherSupportsSearchPath() { SearchCluster cl = new MockSearchCluster("1", 2, 2); - Query q = query(); + Query q = new Query(); q.getModel().setSearchPath("1/0"); // second node in first group MockInvokerFactory invokerFactory = new MockInvokerFactory(cl, (nodes, a) -> { - assertThat(nodes.size(), is(1)); - assertThat(nodes.get(0).key(), is(2)); + assertEquals(1, nodes.size()); + assertEquals(2, nodes.get(0).key()); return true; }); Dispatcher disp = new Dispatcher(cl, createDispatchConfig(), invokerFactory, invokerFactory, new MockMetric()); - Optional<SearchInvoker> invoker = disp.getSearchInvoker(q, null); - assertThat(invoker.isPresent(), is(true)); + SearchInvoker invoker = disp.getSearchInvoker(q, null); invokerFactory.verifyAllEventsProcessed(); } @@ -73,8 +54,7 @@ public class DispatcherTest { }; MockInvokerFactory invokerFactory = new MockInvokerFactory(cl, (n, a) -> true); Dispatcher disp = new Dispatcher(cl, createDispatchConfig(), invokerFactory, invokerFactory, new MockMetric()); - Optional<SearchInvoker> invoker = disp.getSearchInvoker(query(), null); - assertThat(invoker.isPresent(), is(true)); + SearchInvoker invoker = disp.getSearchInvoker(new Query(), null); invokerFactory.verifyAllEventsProcessed(); } @@ -83,31 +63,34 @@ public class DispatcherTest { SearchCluster cl = new MockSearchCluster("1", 2, 1); MockInvokerFactory invokerFactory = new MockInvokerFactory(cl, (n, acceptIncompleteCoverage) -> { - assertThat(acceptIncompleteCoverage, is(false)); + assertFalse(acceptIncompleteCoverage); return false; }, (n, acceptIncompleteCoverage) -> { - assertThat(acceptIncompleteCoverage, is(true)); + assertTrue(acceptIncompleteCoverage); return true; }); Dispatcher disp = new Dispatcher(cl, createDispatchConfig(), invokerFactory, invokerFactory, new MockMetric()); - Optional<SearchInvoker> invoker = disp.getSearchInvoker(query(), null); - assertThat(invoker.isPresent(), is(true)); + SearchInvoker invoker = disp.getSearchInvoker(new Query(), null); invokerFactory.verifyAllEventsProcessed(); } @Test public void requireThatInvokerConstructionDoesNotRepeatGroups() { - SearchCluster cl = new MockSearchCluster("1", 2, 1); + try { + SearchCluster cl = new MockSearchCluster("1", 2, 1); - MockInvokerFactory invokerFactory = new MockInvokerFactory(cl, (n, a) -> false, (n, a) -> false); - Dispatcher disp = new Dispatcher(cl, createDispatchConfig(), invokerFactory, invokerFactory, new MockMetric()); - Optional<SearchInvoker> invoker = disp.getSearchInvoker(query(), null); - assertThat(invoker.isPresent(), is(false)); - invokerFactory.verifyAllEventsProcessed(); + MockInvokerFactory invokerFactory = new MockInvokerFactory(cl, (n, a) -> false, (n, a) -> false); + Dispatcher disp = new Dispatcher(cl, createDispatchConfig(), invokerFactory, invokerFactory, new MockMetric()); + disp.getSearchInvoker(new Query(), null); + fail("Expected exception"); + } + catch (IllegalStateException e) { + assertEquals("No suitable groups to dispatch query. Rejected: [0, 1]", e.getMessage()); + } } interface FactoryStep { - public boolean returnInvoker(List<Node> nodes, boolean acceptIncompleteCoverage); + boolean returnInvoker(List<Node> nodes, boolean acceptIncompleteCoverage); } private static class MockInvokerFactory extends InvokerFactory implements PingFactory { @@ -121,8 +104,11 @@ public class DispatcherTest { } @Override - public Optional<SearchInvoker> createSearchInvoker(VespaBackEndSearcher searcher, Query query, OptionalInt groupId, - List<Node> nodes, boolean acceptIncompleteCoverage) { + public Optional<SearchInvoker> createSearchInvoker(VespaBackEndSearcher searcher, + Query query, + OptionalInt groupId, + List<Node> nodes, + boolean acceptIncompleteCoverage) { if (step >= events.length) { throw new RuntimeException("Was not expecting more calls to getSearchInvoker"); } @@ -136,7 +122,7 @@ public class DispatcherTest { } void verifyAllEventsProcessed() { - assertThat(step, is(events.length)); + assertEquals(events.length, step); } @Override @@ -146,7 +132,7 @@ public class DispatcherTest { } @Override - public Optional<FillInvoker> createFillInvoker(VespaBackEndSearcher searcher, Result result) { + public FillInvoker createFillInvoker(VespaBackEndSearcher searcher, Result result) { fail("Unexpected call to createFillInvoker"); return null; } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java index 8cb5b08bcdb..b9c6fd8c555 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java @@ -701,6 +701,12 @@ public class InternalStepRunner implements StepRunner { String resourceString = String.format(Locale.ENGLISH, "<resources vcpu=\"%.2f\" memory=\"%.2fGb\" disk=\"%.2fGb\" disk-speed=\"%s\"/>", resources.vcpu(), resources.memoryGb(), resources.diskGb(), resources.diskSpeed().name()); + /* TODO after 18 November 2019, include storageType: + String resourceString = String.format(Locale.ENGLISH, + "<resources vcpu=\"%.2f\" memory=\"%.2fGb\" disk=\"%.2fGb\" disk-speed=\"%s\" storage-type=\"%s\"/>", + resources.vcpu(), resources.memoryGb(), resources.diskGb(), resources.diskSpeed().name(), resources.storageType().name()); + + */ AthenzDomain idDomain = ("vespa.vespa.cd".equals(domain.value()) ? AthenzDomain.from("vespa.vespa") : domain); String servicesXml = diff --git a/document/src/vespa/document/select/resultlist.cpp b/document/src/vespa/document/select/resultlist.cpp index 60361223a64..2058e443c95 100644 --- a/document/src/vespa/document/select/resultlist.cpp +++ b/document/src/vespa/document/select/resultlist.cpp @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "resultlist.h" +#include <bitset> #include <ostream> namespace document::select { @@ -96,39 +97,58 @@ ResultList::combineVariables( ResultList ResultList::operator&&(const ResultList& other) const { - ResultList result; + ResultList results; - // TODO: optimize + std::bitset<3> resultForNoVariables; for (const auto & it : _results) { for (const auto & it2 : other._results) { fieldvalue::VariableMap vars = it.first; if (combineVariables(vars, it2.first)) { - result.add(vars, *it.second && *it2.second); + const Result & result = *it.second && *it2.second; + if (vars.empty()) { + resultForNoVariables.set(result.toEnum()); + } else { + results.add(vars, result); + } } } } + for (uint32_t i(0); i < resultForNoVariables.size(); i++) { + if (resultForNoVariables[i]) { + results.add(fieldvalue::VariableMap(), Result::fromEnum(i)); + } + } - return result; + return results; } ResultList ResultList::operator||(const ResultList& other) const { - ResultList result; + ResultList results; - // TODO: optimize + std::bitset<3> resultForNoVariables; for (const auto & it : _results) { for (const auto & it2 : other._results) { fieldvalue::VariableMap vars = it.first; - if (combineVariables(vars, it2.first)) { - result.add(vars, *it.second || *it2.second); + const Result & result = *it.second || *it2.second; + if (vars.empty()) { + resultForNoVariables.set(result.toEnum()); + } else { + results.add(vars, result); + } } } } + for (uint32_t i(0); i < resultForNoVariables.size(); i++) { + if (resultForNoVariables[i]) { + results.add(fieldvalue::VariableMap(), Result::fromEnum(i)); + } + } - return result; + return results; } ResultList diff --git a/document/src/vespa/document/select/value.cpp b/document/src/vespa/document/select/value.cpp index a33ee27afdb..8ada946e702 100644 --- a/document/src/vespa/document/select/value.cpp +++ b/document/src/vespa/document/select/value.cpp @@ -3,6 +3,7 @@ #include "value.h" #include "operator.h" #include <vespa/document/fieldvalue/fieldvalue.h> +#include <bitset> #include <ostream> namespace document::select { @@ -408,10 +409,20 @@ ArrayValue::doCompare(const Value& value, const Predicate& cmp) const } else { ResultList results; + std::bitset<3> resultForNoVariables; // If comparing with other value, must match one. - for (uint32_t i=0; i<_values.size(); ++i) { - results.add(_values[i].first, - cmp(*_values[i].second, value).combineResults()); + for (const auto & item : _values) { + const Result & result = cmp(*item.second, value).combineResults(); + if (item.first.empty()) { + resultForNoVariables.set(result.toEnum()); + } else { + results.add(item.first, result); + } + } + for (uint32_t i(0); i < resultForNoVariables.size(); i++) { + if (resultForNoVariables[i]) { + results.add(fieldvalue::VariableMap(), Result::fromEnum(i)); + } } return results; } diff --git a/document/src/vespa/document/select/valuenode.h b/document/src/vespa/document/select/valuenode.h index 3f493649e87..04ed8178b40 100644 --- a/document/src/vespa/document/select/valuenode.h +++ b/document/src/vespa/document/select/valuenode.h @@ -6,8 +6,6 @@ * @brief Node representing a value in the tree * * @author H�kon Humberset - * @date 2007-04-20 - * @version $Id$ */ #pragma once @@ -51,4 +49,3 @@ protected: }; } - diff --git a/document/src/vespa/document/select/valuenodes.cpp b/document/src/vespa/document/select/valuenodes.cpp index 1eb25b9b6c6..9cfdae14de1 100644 --- a/document/src/vespa/document/select/valuenodes.cpp +++ b/document/src/vespa/document/select/valuenodes.cpp @@ -192,7 +192,8 @@ FieldValueNode::FieldValueNode(const vespalib::string& doctype, { } -FieldValueNode::~FieldValueNode() {} +FieldValueNode::~FieldValueNode() = default; + vespalib::string FieldValueNode::extractFieldName(const std::string & fieldExpression) { std::smatch match; @@ -1075,4 +1076,3 @@ const vespalib::string& FieldExprNode::resolve_doctype() const { } } - diff --git a/document/src/vespa/document/select/valuenodes.h b/document/src/vespa/document/select/valuenodes.h index 02079c0b5df..fa592055477 100644 --- a/document/src/vespa/document/select/valuenodes.h +++ b/document/src/vespa/document/select/valuenodes.h @@ -6,14 +6,14 @@ #include <vespa/document/base/fieldpath.h> namespace document { + class BucketDistribution; + class BucketIdFactory; + class DocumentId; + class BucketId; + class DocumentType; +} -class BucketDistribution; -class BucketIdFactory; -class DocumentId; -class BucketId; -class DocumentType; - -namespace select { +namespace document::select { class InvalidValueNode : public ValueNode { @@ -358,5 +358,4 @@ private: std::ostream&) const; }; -} // select -} // document +} diff --git a/eval/src/tests/eval/function/function_test.cpp b/eval/src/tests/eval/function/function_test.cpp index 49793a62958..0e3100ae425 100644 --- a/eval/src/tests/eval/function/function_test.cpp +++ b/eval/src/tests/eval/function/function_test.cpp @@ -842,8 +842,8 @@ TEST("require that verbose tensor create handles spaces and reordering of variou } TEST("require that verbose tensor create detects invalid tensor type") { - TEST_DO(verify_error("tensor(x[,y}):ignored", - "[tensor(x[,y}):]...[invalid tensor type]...[ignored]")); + TEST_DO(verify_error("tensor(x[,y}):{{ignored}}", + "[tensor(x[,y})]...[invalid tensor type]...[:{{ignored}}]")); } TEST("require that verbose tensor create detects incomplete addresses") { @@ -868,6 +868,71 @@ TEST("require that verbose tensor create detects non-numeric indexes for indexed //----------------------------------------------------------------------------- +TEST("require that convenient tensor create can be parsed") { + auto dense = Function::parse("tensor(x[3]):[1,2,3]"); + auto sparse = Function::parse("tensor(x{}):{a:1,b:2,c:3}"); + auto mixed = Function::parse("tensor(x{},y[2]):{a:[1,2]}"); + EXPECT_EQUAL("tensor(x[3]):{{x:0}:1,{x:1}:2,{x:2}:3}", dense.dump()); + EXPECT_EQUAL("tensor(x{}):{{x:a}:1,{x:b}:2,{x:c}:3}", sparse.dump()); + EXPECT_EQUAL("tensor(x{},y[2]):{{x:a,y:0}:1,{x:a,y:1}:2}", mixed.dump()); +} + +TEST("require that convenient tensor create can contain expressions") { + auto fun = Function::parse("tensor(x[2]):[1,2+a]"); + EXPECT_EQUAL("tensor(x[2]):{{x:0}:1,{x:1}:(2+a)}", fun.dump()); + ASSERT_EQUAL(fun.num_params(), 1u); + EXPECT_EQUAL(fun.param_name(0), "a"); +} + +TEST("require that convenient tensor create handles dimension order") { + auto mixed = Function::parse("tensor(y{},x[2]):{a:[1,2]}"); + EXPECT_EQUAL("tensor(x[2],y{}):{{x:0,y:a}:1,{x:1,y:a}:2}", mixed.dump()); +} + +TEST("require that convenient tensor create can be highly nested") { + vespalib::string expect("tensor(a{},b{},c[1],d[1]):{{a:x,b:y,c:0,d:0}:5}"); + auto nested1 = Function::parse("tensor(a{},b{},c[1],d[1]):{x:{y:[[5]]}}"); + auto nested2 = Function::parse("tensor(c[1],d[1],a{},b{}):[[{x:{y:5}}]]"); + auto nested3 = Function::parse("tensor(a{},c[1],b{},d[1]): { x : [ { y : [ 5 ] } ] } "); + EXPECT_EQUAL(expect, nested1.dump()); + EXPECT_EQUAL(expect, nested2.dump()); + EXPECT_EQUAL(expect, nested3.dump()); +} + +TEST("require that convenient tensor create can have multiple values on multiple levels") { + vespalib::string expect("tensor(x{},y[2]):{{x:a,y:0}:1,{x:a,y:1}:2,{x:b,y:0}:3,{x:b,y:1}:4}"); + auto fun1 = Function::parse("tensor(x{},y[2]):{a:[1,2],b:[3,4]}"); + auto fun2 = Function::parse("tensor(y[2],x{}):[{a:1,b:3},{a:2,b:4}]"); + auto fun3 = Function::parse("tensor(x{},y[2]): { a : [ 1 , 2 ] , b : [ 3 , 4 ] } "); + auto fun4 = Function::parse("tensor(y[2],x{}): [ { a : 1 , b : 3 } , { a : 2 , b : 4 } ] "); + EXPECT_EQUAL(expect, fun1.dump()); + EXPECT_EQUAL(expect, fun2.dump()); + EXPECT_EQUAL(expect, fun3.dump()); + EXPECT_EQUAL(expect, fun4.dump()); +} + +TEST("require that convenient tensor create allows under-specified tensors") { + auto fun = Function::parse("tensor(x[2],y[2]):[[],[5]]"); + EXPECT_EQUAL("tensor(x[2],y[2]):{{x:1,y:0}:5}", fun.dump()); +} + +TEST("require that convenient tensor create detects invalid tensor type") { + TEST_DO(verify_error("tensor(x[,y}):ignored", + "[tensor(x[,y})]...[invalid tensor type]...[:ignored]")); +} + +TEST("require that convenient tensor create detects too large indexed dimensions") { + TEST_DO(verify_error("tensor(x[1]):[1,2]", + "[tensor(x[1]):[1,]...[dimension too large: 'x']...[2]]")); +} + +TEST("require that convenient tensor create detects under-specified cells") { + TEST_DO(verify_error("tensor(x[1],y[1]):[1]", + "[tensor(x[1],y[1]):[]...[expected '[', but got '1']...[1]]")); +} + +//----------------------------------------------------------------------------- + TEST("require that tensor concat can be parsed") { EXPECT_EQUAL("concat(a,b,d)", Function::parse({"a", "b"}, "concat(a,b,d)").dump()); EXPECT_EQUAL("concat(a,b,d)", Function::parse({"a", "b"}, " concat ( a , b , d ) ").dump()); diff --git a/eval/src/tests/eval/node_types/node_types_test.cpp b/eval/src/tests/eval/node_types/node_types_test.cpp index 5504bb33137..b2ad107f2aa 100644 --- a/eval/src/tests/eval/node_types/node_types_test.cpp +++ b/eval/src/tests/eval/node_types/node_types_test.cpp @@ -7,25 +7,10 @@ using namespace vespalib::eval; -/** - * Hack to avoid parse-conflict between tensor type expressions and - * lambda-generated tensors. This will patch leading identifier 'T' to - * 't' directly in the input stream after we have concluded that this - * is not a lambda-generated tensor in order to parse it out as a - * valid tensor type. This may be reverted later if we add support for - * parser rollback when we fail to parse a lambda-generated tensor. - **/ -void tensor_type_hack(const char *pos_in, const char *end_in) { - if ((pos_in < end_in) && (*pos_in == 'T')) { - const_cast<char *>(pos_in)[0] = 't'; - } -} - struct TypeSpecExtractor : public vespalib::eval::SymbolExtractor { void extract_symbol(const char *pos_in, const char *end_in, const char *&pos_out, vespalib::string &symbol_out) const override { - tensor_type_hack(pos_in, end_in); ValueType type = value_type::parse_spec(pos_in, end_in, pos_out); if (pos_out != nullptr) { symbol_out = type.to_spec(); @@ -33,21 +18,7 @@ struct TypeSpecExtractor : public vespalib::eval::SymbolExtractor { } }; -void verify(const vespalib::string &type_expr_in, const vespalib::string &type_spec, bool replace_first = true) { - vespalib::string type_expr = type_expr_in; - // replace 'tensor' with 'Tensor' in type expression, see hack above - size_t tensor_cnt = 0; - for (size_t idx = type_expr.find("tensor"); - idx != type_expr.npos; - idx = type_expr.find("tensor", idx + 1)) - { - // setting 'replace_first' to false will avoid replacing the - // first 'tensor' instance to let the parser handle it as an - // actual tensor generator. - if ((tensor_cnt++ > 0) || replace_first) { - type_expr[idx] = 'T'; - } - } +void verify(const vespalib::string &type_expr, const vespalib::string &type_spec) { Function function = Function::parse(type_expr, TypeSpecExtractor()); if (!EXPECT_TRUE(!function.has_error())) { fprintf(stderr, "parse error: %s\n", function.get_error().c_str()); @@ -225,22 +196,22 @@ TEST("require that join resolves correct type") { } TEST("require that lambda tensor resolves correct type") { - TEST_DO(verify("tensor(x[5])(1.0)", "tensor(x[5])", false)); - TEST_DO(verify("tensor(x[5],y[10])(1.0)", "tensor(x[5],y[10])", false)); - TEST_DO(verify("tensor(x[5],y[10],z[15])(1.0)", "tensor(x[5],y[10],z[15])", false)); - TEST_DO(verify("tensor<double>(x[5],y[10],z[15])(1.0)", "tensor(x[5],y[10],z[15])", false)); - TEST_DO(verify("tensor<float>(x[5],y[10],z[15])(1.0)", "tensor<float>(x[5],y[10],z[15])", false)); + TEST_DO(verify("tensor(x[5])(1.0)", "tensor(x[5])")); + TEST_DO(verify("tensor(x[5],y[10])(1.0)", "tensor(x[5],y[10])")); + TEST_DO(verify("tensor(x[5],y[10],z[15])(1.0)", "tensor(x[5],y[10],z[15])")); + TEST_DO(verify("tensor<double>(x[5],y[10],z[15])(1.0)", "tensor(x[5],y[10],z[15])")); + TEST_DO(verify("tensor<float>(x[5],y[10],z[15])(1.0)", "tensor<float>(x[5],y[10],z[15])")); } TEST("require that tensor create resolves correct type") { - TEST_DO(verify("tensor(x[3]):{{x:0}:double,{x:1}:double,{x:2}:double}", "tensor(x[3])", false)); - TEST_DO(verify("tensor(x{}):{{x:a}:double,{x:b}:double,{x:c}:double}", "tensor(x{})", false)); - TEST_DO(verify("tensor(x{},y[2]):{{x:a,y:0}:double,{x:a,y:1}:double}", "tensor(x{},y[2])", false)); - TEST_DO(verify("tensor<float>(x[3]):{{x:0}:double,{x:1}:double,{x:2}:double}", "tensor<float>(x[3])", false)); - TEST_DO(verify("tensor(x[3]):{{x:0}:double+double,{x:1}:double-double,{x:2}:double/double}", "tensor(x[3])", false)); - TEST_DO(verify("tensor(x[3]):{{x:0}:double,{x:1}:reduce(tensor(x[2]),sum),{x:2}:double}", "tensor(x[3])", false)); - TEST_DO(verify("tensor(x[3]):{{x:0}:double,{x:1}:tensor(x[2]),{x:2}:double}", "error", false)); - TEST_DO(verify("tensor(x[3]):{{x:0}:double,{x:1}:error,{x:2}:double}", "error", false)); + TEST_DO(verify("tensor(x[3]):{{x:0}:double,{x:1}:double,{x:2}:double}", "tensor(x[3])")); + TEST_DO(verify("tensor(x{}):{{x:a}:double,{x:b}:double,{x:c}:double}", "tensor(x{})")); + TEST_DO(verify("tensor(x{},y[2]):{{x:a,y:0}:double,{x:a,y:1}:double}", "tensor(x{},y[2])")); + TEST_DO(verify("tensor<float>(x[3]):{{x:0}:double,{x:1}:double,{x:2}:double}", "tensor<float>(x[3])")); + TEST_DO(verify("tensor(x[3]):{{x:0}:double+double,{x:1}:double-double,{x:2}:double/double}", "tensor(x[3])")); + TEST_DO(verify("tensor(x[3]):{{x:0}:double,{x:1}:reduce(tensor(x[2]),sum),{x:2}:double}", "tensor(x[3])")); + TEST_DO(verify("tensor(x[3]):{{x:0}:double,{x:1}:tensor(x[2]),{x:2}:double}", "error")); + TEST_DO(verify("tensor(x[3]):{{x:0}:double,{x:1}:error,{x:2}:double}", "error")); } TEST("require that tensor concat resolves correct type") { diff --git a/eval/src/tests/eval/value_type/value_type_test.cpp b/eval/src/tests/eval/value_type/value_type_test.cpp index 42b34c0d3ea..85ff7613775 100644 --- a/eval/src/tests/eval/value_type/value_type_test.cpp +++ b/eval/src/tests/eval/value_type/value_type_test.cpp @@ -162,6 +162,37 @@ TEST("require that value type spec can be parsed with extra whitespace") { EXPECT_EQUAL(ValueType::tensor_type({{"y", 10}}, CellType::FLOAT), ValueType::from_spec(" tensor < float > ( y [ 10 ] ) ")); } +TEST("require that the unsorted dimension list can be obtained when parsing type spec") { + std::vector<ValueType::Dimension> unsorted; + auto type = ValueType::from_spec("tensor(y[10],z[5],x{})", unsorted); + EXPECT_EQUAL(ValueType::tensor_type({{"x"}, {"y", 10}, {"z", 5}}), type); + ASSERT_EQUAL(unsorted.size(), 3u); + EXPECT_EQUAL(unsorted[0].name, "y"); + EXPECT_EQUAL(unsorted[0].size, 10u); + EXPECT_EQUAL(unsorted[1].name, "z"); + EXPECT_EQUAL(unsorted[1].size, 5u); + EXPECT_EQUAL(unsorted[2].name, "x"); + EXPECT_EQUAL(unsorted[2].size, npos); +} + +TEST("require that the unsorted dimension list can be obtained also when the type spec is invalid") { + std::vector<ValueType::Dimension> unsorted; + auto type = ValueType::from_spec("tensor(x[10],x[5])...", unsorted); + EXPECT_TRUE(type.is_error()); + ASSERT_EQUAL(unsorted.size(), 2u); + EXPECT_EQUAL(unsorted[0].name, "x"); + EXPECT_EQUAL(unsorted[0].size, 10u); + EXPECT_EQUAL(unsorted[1].name, "x"); + EXPECT_EQUAL(unsorted[1].size, 5u); +} + +TEST("require that the unsorted dimension list can not be obtained if the parse itself fails") { + std::vector<ValueType::Dimension> unsorted; + auto type = ValueType::from_spec("tensor(x[10],x[5]", unsorted); + EXPECT_TRUE(type.is_error()); + EXPECT_EQUAL(unsorted.size(), 0u); +} + TEST("require that malformed value type spec is parsed as error") { EXPECT_TRUE(ValueType::from_spec("").is_error()); EXPECT_TRUE(ValueType::from_spec(" ").is_error()); @@ -270,6 +301,20 @@ TEST("require that type-related predicate functions work as expected") { TEST_DO(verify_predicates(type("tensor<float>(x[5],y{})"), false, false, true, false, false)); } +TEST("require that dense subspace size calculation works as expected") { + EXPECT_EQUAL(type("error").dense_subspace_size(), 1u); + EXPECT_EQUAL(type("double").dense_subspace_size(), 1u); + EXPECT_EQUAL(type("tensor()").dense_subspace_size(), 1u); + EXPECT_EQUAL(type("tensor(x{})").dense_subspace_size(), 1u); + EXPECT_EQUAL(type("tensor(x{},y{})").dense_subspace_size(), 1u); + EXPECT_EQUAL(type("tensor(x[5])").dense_subspace_size(), 5u); + EXPECT_EQUAL(type("tensor(x[5],y[10])").dense_subspace_size(), 50u); + EXPECT_EQUAL(type("tensor(x[5],y{})").dense_subspace_size(), 5u); + EXPECT_EQUAL(type("tensor<float>(x{})").dense_subspace_size(), 1u); + EXPECT_EQUAL(type("tensor<float>(x[5])").dense_subspace_size(), 5u); + EXPECT_EQUAL(type("tensor<float>(x[5],y{})").dense_subspace_size(), 5u); +} + TEST("require that dimension predicates work as expected") { ValueType::Dimension x("x"); ValueType::Dimension y("y", 10); diff --git a/eval/src/vespa/eval/eval/function.cpp b/eval/src/vespa/eval/eval/function.cpp index 1cdd0478181..8b49278a8f0 100644 --- a/eval/src/vespa/eval/eval/function.cpp +++ b/eval/src/vespa/eval/eval/function.cpp @@ -613,11 +613,7 @@ TensorSpec::Address get_tensor_address(ParseContext &ctx, const ValueType &type) // pre: 'tensor<float>(a{},x[3]):' -> type // expect: '{{a:w,x:0}:1,{a:w,x:1}:2,{a:w,x:2}:3}' -void parse_tensor_create(ParseContext &ctx, const ValueType &type) { - if (type.is_error()) { - ctx.fail("invalid tensor type"); - return; - } +void parse_tensor_create_verbose(ParseContext &ctx, const ValueType &type) { ctx.skip_spaces(); ctx.eat('{'); nodes::TensorCreate::Spec create_spec; @@ -634,11 +630,78 @@ void parse_tensor_create(ParseContext &ctx, const ValueType &type) { ctx.push_expression(std::make_unique<nodes::TensorCreate>(type, std::move(create_spec))); } -void parse_tensor_lambda(ParseContext &ctx, const ValueType &type) { - if (!type.is_dense()) { - ctx.fail("invalid tensor type"); - return; +// pre: 'tensor<float>(a{},x[3]):' -> type +// expect: '{w:[0,1,2]}' +void parse_tensor_create_convenient(ParseContext &ctx, const ValueType &type, + const std::vector<ValueType::Dimension> &dim_list) +{ + nodes::TensorCreate::Spec create_spec; + using Label = TensorSpec::Label; + std::vector<Label> addr; + for (;;) { + if (addr.size() == dim_list.size()) { + TensorSpec::Address address; + for (size_t i = 0; i < addr.size(); ++i) { + if (addr[i].is_mapped()) { + address.emplace(dim_list[i].name, addr[i]); + } else { + address.emplace(dim_list[i].name, Label(addr[i].index-1)); + } + } + create_spec.emplace(std::move(address), get_expression(ctx)); + } else { + bool mapped = dim_list[addr.size()].is_mapped(); + addr.push_back(mapped ? Label("") : Label(size_t(0))); + ctx.skip_spaces(); + ctx.eat(mapped ? '{' : '['); + } + while (ctx.find_list_end()) { + bool mapped = addr.back().is_mapped(); + ctx.eat(mapped ? '}' : ']'); + addr.pop_back(); + if (addr.empty()) { + return ctx.push_expression(std::make_unique<nodes::TensorCreate>(type, std::move(create_spec))); + } + } + if (addr.back().is_mapped()) { + if (addr.back().name != "") { + ctx.eat(','); + } + addr.back().name = get_ident(ctx, false); + ctx.skip_spaces(); + ctx.eat(':'); + } else { + if (addr.back().index != 0) { + ctx.eat(','); + } + if (++addr.back().index > dim_list[addr.size()-1].size) { + return ctx.fail(make_string("dimension too large: '%s'", + dim_list[addr.size()-1].name.c_str())); + } + } + } +} + +void parse_tensor_create(ParseContext &ctx, const ValueType &type, + const std::vector<ValueType::Dimension> &dim_list) +{ + ctx.skip_spaces(); + ctx.eat(':'); + ParseContext::InputMark before_cells = ctx.get_input_mark(); + ctx.skip_spaces(); + ctx.eat('{'); + ctx.skip_spaces(); + ctx.eat('{'); + bool is_verbose = !ctx.failed(); + ctx.restore_input_mark(before_cells); + if (is_verbose) { + parse_tensor_create_verbose(ctx, type); + } else { + parse_tensor_create_convenient(ctx, type, dim_list); } +} + +void parse_tensor_lambda(ParseContext &ctx, const ValueType &type) { auto param_names = type.dimension_names(); ExplicitParams params(param_names); ctx.push_resolve_context(params, nullptr); @@ -650,7 +713,8 @@ void parse_tensor_lambda(ParseContext &ctx, const ValueType &type) { ctx.push_expression(std::make_unique<nodes::TensorLambda>(std::move(type), std::move(lambda))); } -void parse_tensor_generator(ParseContext &ctx) { +bool maybe_parse_tensor_generator(ParseContext &ctx) { + ParseContext::InputMark my_mark = ctx.get_input_mark(); vespalib::string type_spec("tensor"); while(!ctx.eos() && (ctx.get() != ')')) { type_spec.push_back(ctx.get()); @@ -658,14 +722,24 @@ void parse_tensor_generator(ParseContext &ctx) { } ctx.eat(')'); type_spec.push_back(')'); - ValueType type = ValueType::from_spec(type_spec); + std::vector<ValueType::Dimension> dim_list; + ValueType type = ValueType::from_spec(type_spec, dim_list); ctx.skip_spaces(); - if (ctx.get() == ':') { - ctx.eat(':'); - parse_tensor_create(ctx, type); - } else { + bool is_tensor_generate = ((ctx.get() == ':') || (ctx.get() == '(')); + if (!is_tensor_generate) { + ctx.restore_input_mark(my_mark); + return false; + } + bool is_create = (type.is_tensor() && (ctx.get() == ':')); + bool is_lambda = (type.is_dense() && (ctx.get() == '(')); + if (is_create) { + parse_tensor_create(ctx, type, dim_list); + } else if (is_lambda) { parse_tensor_lambda(ctx, type); + } else { + ctx.fail("invalid tensor type"); } + return true; } void parse_tensor_concat(ParseContext &ctx) { @@ -678,7 +752,7 @@ void parse_tensor_concat(ParseContext &ctx) { ctx.push_expression(std::make_unique<nodes::TensorConcat>(std::move(lhs), std::move(rhs), dimension)); } -bool try_parse_call(ParseContext &ctx, const vespalib::string &name) { +bool maybe_parse_call(ParseContext &ctx, const vespalib::string &name) { ctx.skip_spaces(); if (ctx.get() == '(') { ctx.eat('('); @@ -717,9 +791,8 @@ size_t parse_symbol(ParseContext &ctx, vespalib::string &name, ParseContext::Inp void parse_symbol_or_call(ParseContext &ctx) { ParseContext::InputMark before_name = ctx.get_input_mark(); vespalib::string name = get_ident(ctx, true); - if (name == "tensor") { - parse_tensor_generator(ctx); - } else if (!try_parse_call(ctx, name)) { + bool was_tensor_generate = ((name == "tensor") && maybe_parse_tensor_generator(ctx)); + if (!was_tensor_generate && !maybe_parse_call(ctx, name)) { size_t id = parse_symbol(ctx, name, before_name); if (name.empty()) { ctx.fail("missing value"); diff --git a/eval/src/vespa/eval/eval/value_type.cpp b/eval/src/vespa/eval/eval/value_type.cpp index 688112bbb8a..211a9c305a3 100644 --- a/eval/src/vespa/eval/eval/value_type.cpp +++ b/eval/src/vespa/eval/eval/value_type.cpp @@ -260,6 +260,12 @@ ValueType::from_spec(const vespalib::string &spec) return value_type::from_spec(spec); } +ValueType +ValueType::from_spec(const vespalib::string &spec, std::vector<ValueType::Dimension> &unsorted) +{ + return value_type::from_spec(spec, unsorted); +} + vespalib::string ValueType::to_spec() const { diff --git a/eval/src/vespa/eval/eval/value_type.h b/eval/src/vespa/eval/eval/value_type.h index df9c582aee8..b02053be3cb 100644 --- a/eval/src/vespa/eval/eval/value_type.h +++ b/eval/src/vespa/eval/eval/value_type.h @@ -77,6 +77,7 @@ public: static ValueType double_type() { return ValueType(Type::DOUBLE); } static ValueType tensor_type(std::vector<Dimension> dimensions_in, CellType cell_type = CellType::DOUBLE); static ValueType from_spec(const vespalib::string &spec); + static ValueType from_spec(const vespalib::string &spec, std::vector<ValueType::Dimension> &unsorted); vespalib::string to_spec() const; static ValueType join(const ValueType &lhs, const ValueType &rhs); static CellType unify_cell_types(const ValueType &a, const ValueType &b); diff --git a/eval/src/vespa/eval/eval/value_type_spec.cpp b/eval/src/vespa/eval/eval/value_type_spec.cpp index bbfa6f4fa28..0246800ca2a 100644 --- a/eval/src/vespa/eval/eval/value_type_spec.cpp +++ b/eval/src/vespa/eval/eval/value_type_spec.cpp @@ -176,7 +176,8 @@ CellType parse_cell_type(ParseContext &ctx) { } // namespace vespalib::eval::value_type::<anonymous> ValueType -parse_spec(const char *pos_in, const char *end_in, const char *&pos_out) +parse_spec(const char *pos_in, const char *end_in, const char *&pos_out, + std::vector<ValueType::Dimension> *unsorted) { ParseContext ctx(pos_in, end_in, pos_out); vespalib::string type_name = parse_ident(ctx); @@ -188,6 +189,9 @@ parse_spec(const char *pos_in, const char *end_in, const char *&pos_out) ValueType::CellType cell_type = parse_cell_type(ctx); std::vector<ValueType::Dimension> list = parse_dimension_list(ctx); if (!ctx.failed()) { + if (unsorted != nullptr) { + *unsorted = list; + } return ValueType::tensor_type(std::move(list), cell_type); } } else { @@ -208,6 +212,18 @@ from_spec(const vespalib::string &spec) return type; } +ValueType +from_spec(const vespalib::string &spec, std::vector<ValueType::Dimension> &unsorted) +{ + const char *after = nullptr; + const char *end = spec.data() + spec.size(); + ValueType type = parse_spec(spec.data(), end, after, &unsorted); + if (after != end) { + return ValueType::error_type(); + } + return type; +} + vespalib::string to_spec(const ValueType &type) { diff --git a/eval/src/vespa/eval/eval/value_type_spec.h b/eval/src/vespa/eval/eval/value_type_spec.h index f2609f59f32..ff5113c769a 100644 --- a/eval/src/vespa/eval/eval/value_type_spec.h +++ b/eval/src/vespa/eval/eval/value_type_spec.h @@ -6,9 +6,11 @@ namespace vespalib::eval::value_type { -ValueType parse_spec(const char *pos_in, const char *end_in, const char *&pos_out); +ValueType parse_spec(const char *pos_in, const char *end_in, const char *&pos_out, + std::vector<ValueType::Dimension> *unsorted = nullptr); -ValueType from_spec(const vespalib::string &str); +ValueType from_spec(const vespalib::string &spec); +ValueType from_spec(const vespalib::string &spec, std::vector<ValueType::Dimension> &unsorted); vespalib::string to_spec(const ValueType &type); } diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index 272e96903f8..4038fd49b02 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -173,6 +173,12 @@ public class Flags { "Takes effect on restart of Docker container", NODE_TYPE, APPLICATION_ID, HOSTNAME); + public static final UnboundStringFlag TLS_FOR_ZOOKEEPER_QUORUM_COMMUNICATION = defineStringFlag( + "tls-for-zookeeper-quorum-communication", "OFF", + "How to setup TLS for ZooKeeper quorum communication. Valid values are OFF, PORT_UNIFICATION, TLS_WITH_PORT_UNIFICATION, TLS_ONLY", + "Takes effect on restart of config server", + NODE_TYPE, HOSTNAME); + /** WARNING: public for testing: All flags should be defined in {@link Flags}. */ public static UnboundBooleanFlag defineFeatureFlag(String flagId, boolean defaultValue, String description, String modificationEffect, FetchVector.Dimension... dimensions) { diff --git a/jaxrs_client_utils/src/main/java/ai/vespa/util/http/VespaClientBuilderFactory.java b/jaxrs_client_utils/src/main/java/ai/vespa/util/http/VespaClientBuilderFactory.java index d55128069c4..50e555e7acd 100644 --- a/jaxrs_client_utils/src/main/java/ai/vespa/util/http/VespaClientBuilderFactory.java +++ b/jaxrs_client_utils/src/main/java/ai/vespa/util/http/VespaClientBuilderFactory.java @@ -10,9 +10,13 @@ import javax.ws.rs.client.ClientRequestContext; import javax.ws.rs.client.ClientRequestFilter; import javax.ws.rs.core.UriBuilder; import java.net.URI; +import java.util.Optional; +import java.util.logging.Filter; import java.util.logging.Level; import java.util.logging.Logger; +import static java.util.logging.Level.CONFIG; + /** * Factory for JAX-RS http client builder for internal Vespa communications over http/https. * @@ -26,6 +30,26 @@ public class VespaClientBuilderFactory implements AutoCloseable { private static final Logger log = Logger.getLogger(VespaClientBuilderFactory.class.getName()); + static { + // CONFIG log message are logged repeatedly from these classes. + disableConfigLogging("org.glassfish.jersey.client.internal.HttpUrlConnector"); + disableConfigLogging("org.glassfish.jersey.process.internal.ExecutorProviders"); + } + + // This method will hook a filter into the Jersey logger removing unwanted messages. + private static void disableConfigLogging(String className) { + @SuppressWarnings("LoggerInitializedWithForeignClass") + Logger logger = Logger.getLogger(className); + Optional<Filter> currentFilter = Optional.ofNullable(logger.getFilter()); + Filter filter = logRecord -> + !logRecord.getMessage().startsWith("Restricted headers are not enabled") + && !logRecord.getMessage().startsWith("Selected ExecutorServiceProvider implementation") + && !logRecord.getLevel().equals(CONFIG) + && currentFilter.map(f -> f.isLoggable(logRecord)).orElse(true); // Honour existing filter if exists + logger.setFilter(filter); + } + + private final TlsContext tlsContext = TransportSecurityUtils.createTlsContext().orElse(null); private final MixedMode mixedMode = TransportSecurityUtils.getInsecureMixedMode(); diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/ConfigServerApiImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/ConfigServerApiImpl.java index 3f6909b8ea8..4dadcb359ea 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/ConfigServerApiImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/ConfigServerApiImpl.java @@ -47,6 +47,7 @@ import java.util.logging.Logger; * @author bjorncs */ public class ConfigServerApiImpl implements ConfigServerApi { + private static final Logger logger = Logger.getLogger(ConfigServerApiImpl.class.getName()); private final ObjectMapper mapper = new ObjectMapper(); @@ -225,4 +226,5 @@ public class ConfigServerApiImpl implements ConfigServerApi { Collections.shuffle(shuffledConfigServerHosts); return shuffledConfigServerHosts; } + } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeSpec.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeSpec.java index c2e68bdb329..a9bb42e4dde 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeSpec.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeSpec.java @@ -20,6 +20,7 @@ import static com.yahoo.config.provision.NodeResources.DiskSpeed.slow; * @author stiankri */ public class NodeSpec { + private final String hostname; private final NodeState state; private final NodeType type; @@ -514,7 +515,7 @@ public class NodeSpec { } public Builder fastDisk(boolean fastDisk) { - return resources(resources.withDiskSpeed(fastDisk ? fast : slow)); + return resources(resources.with(fastDisk ? fast : slow)); } public Builder bandwidthGbps(double bandwidthGbps) { diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepository.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepository.java index 49222222a18..deb7f5f9549 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepository.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepository.java @@ -28,11 +28,9 @@ import java.util.logging.Logger; import java.util.stream.Collectors; import java.util.stream.Stream; -import static com.yahoo.config.provision.NodeResources.DiskSpeed.fast; -import static com.yahoo.config.provision.NodeResources.DiskSpeed.slow; - /** - * @author stiankri, dybis + * @author stiankri + * @author dybis */ public class RealNodeRepository implements NodeRepository { private static final Logger logger = Logger.getLogger(RealNodeRepository.class.getName()); @@ -68,7 +66,7 @@ public class RealNodeRepository implements NodeRepository { public Optional<NodeSpec> getOptionalNode(String hostName) { try { NodeRepositoryNode nodeResponse = configServerApi.get("/nodes/v2/node/" + hostName, - NodeRepositoryNode.class); + NodeRepositoryNode.class); return Optional.ofNullable(nodeResponse).map(RealNodeRepository::createNodeSpec); } catch (HttpException.NotFoundException | HttpException.ForbiddenException e) { @@ -152,7 +150,6 @@ public class RealNodeRepository implements NodeRepository { Optional<NodeMembership> membership = Optional.ofNullable(node.membership) .map(m -> new NodeMembership(m.clusterType, m.clusterId, m.group, m.index, m.retired)); NodeReports reports = NodeReports.fromMap(Optional.ofNullable(node.reports).orElseGet(Map::of)); - return new NodeSpec( node.hostname, Optional.ofNullable(node.wantedDockerImage).map(DockerImage::fromString), @@ -180,13 +177,26 @@ public class RealNodeRepository implements NodeRepository { node.minMainMemoryAvailableGb, node.minDiskAvailableGb, node.bandwidthGbps, - node.fastDisk ? fast : slow), + toDiskSpeed(node.fastDisk), + toStorageType(node.remoteStorage)), node.ipAddresses, node.additionalIpAddresses, reports, Optional.ofNullable(node.parentHostname)); } + private static NodeResources.DiskSpeed toDiskSpeed(Boolean fastDisk) { + if (fastDisk == null) return NodeResources.DiskSpeed.any; + if (fastDisk) return NodeResources.DiskSpeed.fast; + else return NodeResources.DiskSpeed.slow; + } + + private static NodeResources.StorageType toStorageType(Boolean remoteStorage) { + if (remoteStorage == null) return NodeResources.StorageType.any; + if (remoteStorage) return NodeResources.StorageType.remote; + else return NodeResources.StorageType.local; + } + private static NodeRepositoryNode nodeRepositoryNodeFromAddNode(AddNode addNode) { NodeRepositoryNode node = new NodeRepositoryNode(); node.openStackId = "fake-" + addNode.hostname; @@ -200,6 +210,7 @@ public class RealNodeRepository implements NodeRepository { node.minDiskAvailableGb = resources.diskGb(); node.bandwidthGbps = resources.bandwidthGbps(); node.fastDisk = resources.diskSpeed() == NodeResources.DiskSpeed.fast; + node.remoteStorage = resources.storageType() == NodeResources.StorageType.remote; }); node.type = addNode.nodeType.name(); node.ipAddresses = addNode.ipAddresses; diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/bindings/NodeRepositoryNode.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/bindings/NodeRepositoryNode.java index 9919c3affa6..bf124f4ad86 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/bindings/NodeRepositoryNode.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/bindings/NodeRepositoryNode.java @@ -58,6 +58,8 @@ public class NodeRepositoryNode { public Integer failCount; @JsonProperty("fastDisk") public Boolean fastDisk; + @JsonProperty("remoteStorage") + public Boolean remoteStorage; @JsonProperty("bandwidthGbps") public Double bandwidthGbps; @JsonProperty("environment") @@ -110,6 +112,7 @@ public class NodeRepositoryNode { ", wantedFirmwareCheck=" + wantedFirmwareCheck + ", failCount=" + failCount + ", fastDisk=" + fastDisk + + ", remoteStorage=" + remoteStorage + ", bandwidthGbps=" + bandwidthGbps + ", environment='" + environment + '\'' + ", type='" + type + '\'' + diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepositoryTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepositoryTest.java index 108f29b11cd..9cdb815a8ec 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepositoryTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepositoryTest.java @@ -162,10 +162,13 @@ public class RealNodeRepositoryTest { @Test public void testAddNodes() { - AddNode host = AddNode.forHost("host123.domain.tld", "default", Optional.of(FlavorOverrides.ofDisk(123)), NodeType.confighost, - Set.of("::1"), Set.of("::2", "::3")); + AddNode host = AddNode.forHost("host123.domain.tld", + "default", + Optional.of(FlavorOverrides.ofDisk(123)), + NodeType.confighost, + Set.of("::1"), Set.of("::2", "::3")); - NodeResources nodeResources = new NodeResources(1, 2, 3, 4, NodeResources.DiskSpeed.slow); + NodeResources nodeResources = new NodeResources(1, 2, 3, 4, NodeResources.DiskSpeed.slow, NodeResources.StorageType.local); AddNode node = AddNode.forNode("host123-1.domain.tld", "host123.domain.tld", nodeResources, NodeType.config, Set.of("::2", "::3")); assertFalse(nodeRepositoryApi.getOptionalNode("host123.domain.tld").isPresent()); @@ -180,4 +183,5 @@ public class RealNodeRepositoryTest { assertEquals(nodeResources, nodeSpec.resources()); assertEquals(NodeType.config, nodeSpec.type()); } + } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java index 49f96d65c81..4adb8fc247f 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java @@ -338,8 +338,8 @@ public final class Node { /** Computes the allocation skew of a host node */ public static double skew(NodeResources totalHostCapacity, NodeResources freeHostCapacity) { - NodeResources all = totalHostCapacity.anySpeed(); - NodeResources allocated = all.subtract(freeHostCapacity.anySpeed()); + NodeResources all = totalHostCapacity.justNumbers(); + NodeResources allocated = all.subtract(freeHostCapacity.justNumbers()); return new Mean(allocated.vcpu() / all.vcpu(), allocated.memoryGb() / all.memoryGb(), diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityChecker.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityChecker.java index 83498aa8709..65fab51932a 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityChecker.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityChecker.java @@ -132,7 +132,7 @@ public class CapacityChecker { int occupiedIps = 0; Set<String> ipPool = host.ipAddressPool().asSet(); for (var child : nodeChildren.get(host)) { - hostResources = hostResources.subtract(child.flavor().resources().withDiskSpeed(NodeResources.DiskSpeed.any)); + hostResources = hostResources.subtract(child.flavor().resources().justNumbers()); occupiedIps += child.ipAddresses().stream().filter(ipPool::contains).count(); } availableResources.put(host, new AllocationResources(hostResources, host.ipAddressPool().asSet().size() - occupiedIps)); @@ -299,15 +299,20 @@ public class CapacityChecker { reason.violatesParentHostPolicy = violatesParentHostPolicy(node, host, containedAllocations); NodeResources l = availableHostResources.nodeResources; - NodeResources r = node.allocation() - .map(Allocation::requestedResources) - .orElse(node.flavor().resources()); - if (l.vcpu() < r.vcpu()) { reason.insufficientVcpu = true; } - if (l.memoryGb() < r.memoryGb()) { reason.insufficientMemoryGb = true; } - if (l.diskGb() < r.diskGb()) { reason.insufficientDiskGb = true; } + NodeResources r = node.allocation().map(Allocation::requestedResources).orElse(node.flavor().resources()); + + if (l.vcpu() < r.vcpu()) + reason.insufficientVcpu = true; + if (l.memoryGb() < r.memoryGb()) + reason.insufficientMemoryGb = true; + if (l.diskGb() < r.diskGb()) + reason.insufficientDiskGb = true; if (r.diskSpeed() != NodeResources.DiskSpeed.any && r.diskSpeed() != l.diskSpeed()) - { reason.incompatibleDiskSpeed = true; } - if (availableHostResources.availableIPs < 1) { reason.insufficientAvailableIPs = true; } + reason.incompatibleDiskSpeed = true; + if (r.storageType() != NodeResources.StorageType.any && r.storageType() != l.storageType()) + reason.incompatibleStorageType = true; + if (availableHostResources.availableIPs < 1) + reason.insufficientAvailableIPs = true; allocationFailureReasons.add(reason); } @@ -406,6 +411,7 @@ public class CapacityChecker { * Keeps track of the reason why a host rejected an allocation. */ private static class AllocationFailureReason { + Node host; public AllocationFailureReason (Node host) { this.host = host; @@ -414,6 +420,7 @@ public class CapacityChecker { public boolean insufficientMemoryGb = false; public boolean insufficientDiskGb = false; public boolean incompatibleDiskSpeed = false; + public boolean incompatibleStorageType = false; public boolean insufficientAvailableIPs = false; public boolean violatesParentHostPolicy = false; @@ -435,6 +442,7 @@ public class CapacityChecker { if (insufficientMemoryGb) reasons.add("insufficientMemoryGb"); if (insufficientDiskGb) reasons.add("insufficientDiskGb"); if (incompatibleDiskSpeed) reasons.add("incompatibleDiskSpeed"); + if (incompatibleStorageType) reasons.add("incompatibleStorageType"); if (insufficientAvailableIPs) reasons.add("insufficientAvailableIPs"); if (violatesParentHostPolicy) reasons.add("violatesParentHostPolicy"); @@ -446,7 +454,9 @@ public class CapacityChecker { * Provides convenient methods for tallying failures. */ public static class AllocationFailureReasonList { + private List<AllocationFailureReason> allocationFailureReasons; + public AllocationFailureReasonList(List<AllocationFailureReason> allocationFailureReasons) { this.allocationFailureReasons = allocationFailureReasons; } @@ -455,6 +465,7 @@ public class CapacityChecker { public long insufficientMemoryGb() { return allocationFailureReasons.stream().filter(r -> r.insufficientMemoryGb).count(); } public long insufficientDiskGb() { return allocationFailureReasons.stream().filter(r -> r.insufficientDiskGb).count(); } public long incompatibleDiskSpeed() { return allocationFailureReasons.stream().filter(r -> r.incompatibleDiskSpeed).count(); } + public long incompatibleStorageType() { return allocationFailureReasons.stream().filter(r -> r.incompatibleStorageType).count(); } public long insufficientAvailableIps() { return allocationFailureReasons.stream().filter(r -> r.insufficientAvailableIPs).count(); } public long violatesParentHostPolicy() { return allocationFailureReasons.stream().filter(r -> r.violatesParentHostPolicy).count(); } @@ -471,13 +482,14 @@ public class CapacityChecker { } @Override public String toString() { - return String.format("CPU (%3d), Memory (%3d), Disk size (%3d), Disk speed (%3d), IP (%3d), Parent-Host Policy (%3d)", - insufficientVcpu(), insufficientMemoryGb(), insufficientDiskGb(), - incompatibleDiskSpeed(), insufficientAvailableIps(), violatesParentHostPolicy()); + return String.format("CPU (%3d), Memory (%3d), Disk size (%3d), Disk speed (%3d), Storage type (%3d), IP (%3d), Parent-Host Policy (%3d)", + insufficientVcpu(), insufficientMemoryGb(), insufficientDiskGb(), incompatibleDiskSpeed(), + incompatibleStorageType(), insufficientAvailableIps(), violatesParentHostPolicy()); } } public static class AllocationHistory { + public static class Entry { public Node tenant; public Node newParent; @@ -533,6 +545,7 @@ public class CapacityChecker { return out.toString(); } + } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java index 5c8aab4d0aa..da0db1f1896 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java @@ -220,7 +220,7 @@ public class MetricsReporter extends Maintainer { .forEach( (applicationId, applicationNodes) -> { var allocatedCapacity = applicationNodes.stream() - .map(node -> node.allocation().get().requestedResources().withDiskSpeed(any)) + .map(node -> node.allocation().get().requestedResources().justNumbers()) .reduce(new NodeResources(0, 0, 0, 0, any), NodeResources::add); var context = getContextAt( @@ -238,20 +238,20 @@ public class MetricsReporter extends Maintainer { private static NodeResources getCapacityTotal(NodeList nodes) { return nodes.nodeType(NodeType.host).asList().stream() .map(host -> host.flavor().resources()) - .map(resources -> resources.withDiskSpeed(any)) + .map(resources -> resources.justNumbers()) .reduce(new NodeResources(0, 0, 0, 0, any), NodeResources::add); } private static NodeResources getFreeCapacityTotal(NodeList nodes) { return nodes.nodeType(NodeType.host).asList().stream() .map(n -> freeCapacityOf(nodes, n)) - .map(resources -> resources.withDiskSpeed(any)) + .map(resources -> resources.justNumbers()) .reduce(new NodeResources(0, 0, 0, 0, any), NodeResources::add); } private static NodeResources freeCapacityOf(NodeList nodes, Node dockerHost) { return nodes.childrenOf(dockerHost).asList().stream() - .map(node -> node.flavor().resources().withDiskSpeed(any)) - .reduce(dockerHost.flavor().resources().withDiskSpeed(any), NodeResources::subtract); + .map(node -> node.flavor().resources().justNumbers()) + .reduce(dockerHost.flavor().resources().justNumbers(), NodeResources::subtract); } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Rebalancer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Rebalancer.java index 1c7fcff52f7..3800e48b4bf 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Rebalancer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Rebalancer.java @@ -16,9 +16,7 @@ import com.yahoo.vespa.hosted.provision.provisioning.NodePrioritizer; import java.time.Clock; import java.time.Duration; -import java.util.List; import java.util.Optional; -import java.util.stream.Stream; public class Rebalancer extends Maintainer { @@ -112,14 +110,14 @@ public class Rebalancer extends Maintainer { private double skewReductionByRemoving(Node node, Node fromHost, DockerHostCapacity capacity) { NodeResources freeHostCapacity = capacity.freeCapacityOf(fromHost); double skewBefore = Node.skew(fromHost.flavor().resources(), freeHostCapacity); - double skewAfter = Node.skew(fromHost.flavor().resources(), freeHostCapacity.add(node.flavor().resources().anySpeed())); + double skewAfter = Node.skew(fromHost.flavor().resources(), freeHostCapacity.add(node.flavor().resources().justNumbers())); return skewBefore - skewAfter; } private double skewReductionByAdding(Node node, Node toHost, DockerHostCapacity capacity) { NodeResources freeHostCapacity = capacity.freeCapacityOf(toHost); double skewBefore = Node.skew(toHost.flavor().resources(), freeHostCapacity); - double skewAfter = Node.skew(toHost.flavor().resources(), freeHostCapacity.subtract(node.flavor().resources().anySpeed())); + double skewAfter = Node.skew(toHost.flavor().resources(), freeHostCapacity.subtract(node.flavor().resources().justNumbers())); return skewBefore - skewAfter; } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java index a7c45cf1cbc..ff5391ec433 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java @@ -86,6 +86,7 @@ public class NodeSerializer { private static final String diskKey = "disk"; private static final String bandwidthKey = "bandwidth"; private static final String diskSpeedKey = "diskSpeed"; + private static final String storageTypeKey = "storageType"; // Allocation fields private static final String tenantIdKey = "tenantId"; @@ -166,6 +167,7 @@ public class NodeSerializer { resourcesObject.setDouble(diskKey, resources.diskGb()); resourcesObject.setDouble(bandwidthKey, resources.bandwidthGbps()); resourcesObject.setString(diskSpeedKey, diskSpeedToString(resources.diskSpeed())); + resourcesObject.setString(storageTypeKey, storageTypeToString(resources.storageType())); } private void toSlime(Allocation allocation, Cursor object) { @@ -251,7 +253,8 @@ public class NodeSerializer { resources.field(memoryKey).asDouble(), resources.field(diskKey).asDouble(), resources.field(bandwidthKey).asDouble(), - diskSpeedFromSlime(resources.field(diskSpeedKey)))); + diskSpeedFromSlime(resources.field(diskSpeedKey)), + storageTypeFromSlime(resources.field(storageTypeKey)))); } private Optional<Allocation> allocationFromSlime(NodeResources assignedResources, Inspector object) { @@ -443,6 +446,25 @@ public class NodeSerializer { case any : return "any"; default: throw new IllegalStateException("Illegal disk-speed value '" + diskSpeed + "'"); } + } + private static NodeResources.StorageType storageTypeFromSlime(Inspector storageType) { + if ( ! storageType.valid()) return NodeResources.StorageType.getDefault(); // TODO: Remove this line after December 2019 + switch (storageType.asString()) { + case "remote" : return NodeResources.StorageType.remote; + case "local" : return NodeResources.StorageType.local; + case "any" : return NodeResources.StorageType.any; + default: throw new IllegalStateException("Illegal storage-type value '" + storageType.asString() + "'"); + } } + + private static String storageTypeToString(NodeResources.StorageType storageType) { + switch (storageType) { + case remote : return "remote"; + case local : return "local"; + case any : return "any"; + default: throw new IllegalStateException("Illegal storage-type value '" + storageType + "'"); + } + } + } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java index 98d06f7e01a..8ee9a92fa46 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java @@ -45,9 +45,9 @@ public class CapacityPolicies { NodeResources resources = requestedResources.orElse(defaultNodeResources(cluster.type())); - // Allow slow disks in zones which are not performance sensitive + // Allow slow storage in zones which are not performance sensitive if (zone.system().isCd() || zone.environment() == Environment.dev || zone.environment() == Environment.test) - resources = resources.withDiskSpeed(NodeResources.DiskSpeed.any); + resources = resources.with(NodeResources.DiskSpeed.any).with(NodeResources.StorageType.any); // Dev does not cap the cpu of containers since usage is spotty: Allocate just a small amount exclusively // Do not cap in AWS as hosts are allocated on demand and 1-to-1, so the node can use the entire host diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/DockerHostCapacity.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/DockerHostCapacity.java index 9713615f77e..781dae020b9 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/DockerHostCapacity.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/DockerHostCapacity.java @@ -3,7 +3,6 @@ package com.yahoo.vespa.hosted.provision.provisioning; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; -import com.yahoo.vespa.hosted.provision.LockedNodeList; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeList; @@ -75,14 +74,11 @@ public class DockerHostCapacity { if (host.type() != NodeType.host) return new NodeResources(0, 0, 0, 0); NodeResources hostResources = hostResourcesCalculator.availableCapacityOf(host.flavor().resources()); - // Subtract used resources without taking disk speed into account since existing allocations grandfathered in - // may not use reflect the actual disk speed (as of May 2019). This (the 3 diskSpeed assignments below) - // can be removed when all node allocations accurately reflect the true host disk speed return allNodes.childrenOf(host).asList().stream() .filter(node -> !(excludeInactive && isInactiveOrRetired(node))) - .map(node -> node.flavor().resources().anySpeed()) - .reduce(hostResources.anySpeed(), NodeResources::subtract) - .withDiskSpeed(host.flavor().resources().diskSpeed()); + .map(node -> node.flavor().resources().justNumbers()) + .reduce(hostResources.justNumbers(), NodeResources::subtract) + .with(host.flavor().resources().diskSpeed()).with(host.flavor().resources().storageType()); } private static boolean isInactiveOrRetired(Node node) { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodePrioritizer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodePrioritizer.java index a7081060f2e..f80ccff94e9 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodePrioritizer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodePrioritizer.java @@ -160,7 +160,8 @@ public class NodePrioritizer { Node newNode = Node.createDockerNode(allocation.get().addresses(), allocation.get().hostname(), host.hostname(), - resources(requestedNodes).withDiskSpeed(host.flavor().resources().diskSpeed()), + resources(requestedNodes).with(host.flavor().resources().diskSpeed()) + .with(host.flavor().resources().storageType()), NodeType.tenant); PrioritizableNode nodePri = toPrioritizable(newNode, false, true); if ( ! nodePri.violatesSpares || isAllocatingForReplacement) { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeResourceComparator.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeResourceComparator.java index 51174d42d4b..f11807b040a 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeResourceComparator.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeResourceComparator.java @@ -30,6 +30,10 @@ public class NodeResourceComparator { if (a.diskGb() < b.diskGb()) return -1; if (a.vcpu() > b.vcpu()) return 1; if (a.vcpu() < b.vcpu()) return -1; + + int storageTypeComparison = NodeResources.StorageType.compare(a.storageType(), b.storageType()); + if (storageTypeComparison != 0) return storageTypeComparison; + return compare(a.diskSpeed(), b.diskSpeed()); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/PrioritizableNode.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/PrioritizableNode.java index 72f8550d063..1f45e466c9d 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/PrioritizableNode.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/PrioritizableNode.java @@ -89,6 +89,11 @@ class PrioritizableNode implements Comparable<PrioritizableNode> { other.parent.get().flavor().resources().diskSpeed()); if (diskCostDifference != 0) return diskCostDifference; + + int storageCostDifference = NodeResources.StorageType.compare(this.parent.get().flavor().resources().storageType(), + other.parent.get().flavor().resources().storageType()); + if (storageCostDifference != 0) + return storageCostDifference; } int hostPriority = Double.compare(this.skewWithThis() - this.skewWithoutThis(), @@ -112,7 +117,7 @@ class PrioritizableNode implements Comparable<PrioritizableNode> { private double skewWith(NodeResources resources) { if (parent.isEmpty()) return 0; - NodeResources free = freeParentCapacity.anySpeed().subtract(resources.anySpeed()); + NodeResources free = freeParentCapacity.justNumbers().subtract(resources.justNumbers()); return Node.skew(parent.get().flavor().resources(), free); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodePatcher.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodePatcher.java index 072dc1172e1..81cf401d358 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodePatcher.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodePatcher.java @@ -33,6 +33,8 @@ import java.util.stream.Collectors; import static com.yahoo.config.provision.NodeResources.DiskSpeed.fast; import static com.yahoo.config.provision.NodeResources.DiskSpeed.slow; +import static com.yahoo.config.provision.NodeResources.StorageType.remote; +import static com.yahoo.config.provision.NodeResources.StorageType.local; /** * A class which can take a partial JSON node/v2 node JSON structure and apply it to a node object. @@ -152,7 +154,9 @@ public class NodePatcher { case "minCpuCores": return node.with(node.flavor().with(node.flavor().resources().withVcpu(value.asDouble()))); case "fastDisk": - return node.with(node.flavor().with(node.flavor().resources().withDiskSpeed(value.asBool() ? fast : slow))); + return node.with(node.flavor().with(node.flavor().resources().with(value.asBool() ? fast : slow))); + case "remoteStorage": + return node.with(node.flavor().with(node.flavor().resources().with(value.asBool() ? remote : local))); case "bandwidthGbps": return node.with(node.flavor().with(node.flavor().resources().withBandwidthGbps(value.asDouble()))); case "modelName": @@ -205,7 +209,7 @@ public class NodePatcher { Optional<Allocation> allocation = node.allocation(); if (allocation.isPresent()) return node.with(allocation.get().withRequestedResources( - allocation.get().requestedResources().withDiskSpeed(NodeResources.DiskSpeed.valueOf(value)))); + allocation.get().requestedResources().with(NodeResources.DiskSpeed.valueOf(value)))); else throw new IllegalArgumentException("Node is not allocated"); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodeSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodeSerializer.java index 5d31e262d2a..57695a0a22c 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodeSerializer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodeSerializer.java @@ -79,4 +79,13 @@ public class NodeSerializer { } } + public String toString(NodeResources.StorageType storageType) { + switch (storageType) { + case remote : return "remote"; + case local : return "local"; + case any : return "any"; + default: throw new IllegalArgumentException("Unknown storage type '" + storageType.name() + "'"); + } + } + } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesApiHandler.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesApiHandler.java index 64cc2691010..ee1e5c7fa45 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesApiHandler.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesApiHandler.java @@ -51,6 +51,8 @@ import java.util.stream.Collectors; import static com.yahoo.config.provision.NodeResources.DiskSpeed.fast; import static com.yahoo.config.provision.NodeResources.DiskSpeed.slow; +import static com.yahoo.config.provision.NodeResources.StorageType.remote; +import static com.yahoo.config.provision.NodeResources.StorageType.local; import static com.yahoo.vespa.config.SlimeUtils.optionalString; /** @@ -239,13 +241,14 @@ public class NodesApiHandler extends LoggingRequestHandler { private Flavor flavorFromSlime(Inspector inspector) { Inspector flavorInspector = inspector.field("flavor"); - if (!flavorInspector.valid()) { + if ( ! flavorInspector.valid()) { return new Flavor(new NodeResources( requiredField(inspector, "minCpuCores", Inspector::asDouble), requiredField(inspector, "minMainMemoryAvailableGb", Inspector::asDouble), requiredField(inspector, "minDiskAvailableGb", Inspector::asDouble), requiredField(inspector, "bandwidthGbps", Inspector::asDouble), - requiredField(inspector, "fastDisk", Inspector::asBool) ? fast : slow)); + requiredField(inspector, "fastDisk", Inspector::asBool) ? fast : slow, + requiredField(inspector, "remoteStorage", Inspector::asBool) ? remote : local)); } Flavor flavor = nodeFlavors.getFlavorOrThrow(flavorInspector.asString()); @@ -258,7 +261,9 @@ public class NodesApiHandler extends LoggingRequestHandler { if (inspector.field("bandwidthGbps").valid()) flavor = flavor.with(flavor.resources().withBandwidthGbps(inspector.field("bandwidthGbps").asDouble())); if (inspector.field("fastDisk").valid()) - flavor = flavor.with(flavor.resources().withDiskSpeed(inspector.field("fastDisk").asBool() ? fast : slow)); + flavor = flavor.with(flavor.resources().with(inspector.field("fastDisk").asBool() ? fast : slow)); + if (inspector.field("remoteStorage").valid()) + flavor = flavor.with(flavor.resources().with(inspector.field("remoteStorage").asBool() ? remote : local)); return flavor; } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesResponse.java index 018d14ef6e0..d8b5426fb75 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesResponse.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesResponse.java @@ -154,6 +154,8 @@ class NodesResponse extends HttpResponse { if (node.flavor().cost() > 0) object.setLong("cost", node.flavor().cost()); object.setBool("fastDisk", node.flavor().hasFastDisk()); + if (node.flavor().resources().storageType() != NodeResources.StorageType.any) + object.setBool("remoteStorage", node.flavor().resources().storageType() == NodeResources.StorageType.remote); object.setDouble("bandwidthGbps", node.flavor().getBandwidthGbps()); object.setString("environment", node.flavor().getType().name()); node.allocation().ifPresent(allocation -> { @@ -219,7 +221,10 @@ class NodesResponse extends HttpResponse { object.setDouble("memoryGb", resources.memoryGb()); object.setDouble("diskGb", resources.diskGb()); object.setDouble("bandwidthGbps", resources.bandwidthGbps()); - object.setString("diskSpeed", serializer.toString(resources.diskSpeed())); + if ( ! resources.diskSpeed().isDefault()) + object.setString("diskSpeed", serializer.toString(resources.diskSpeed())); + if ( ! resources.storageType().isDefault()) + object.setString("storageType", serializer.toString(resources.storageType())); } // Hack: For non-docker noder, return current docker image as default prefix + current Vespa version diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerHostCapacityTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerHostCapacityTest.java index b52380242d7..a41b61fd352 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerHostCapacityTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerHostCapacityTest.java @@ -89,16 +89,20 @@ public class DockerHostCapacityTest { @Test public void freeCapacityOf() { - assertEquals(new NodeResources(5, 4, 8, 2), capacity.freeCapacityOf(host1, false)); - assertEquals(new NodeResources(5, 6, 8, 4.5), capacity.freeCapacityOf(host3, false)); + assertEquals(new NodeResources(5, 4, 8, 2, NodeResources.DiskSpeed.fast, NodeResources.StorageType.remote), + capacity.freeCapacityOf(host1, false)); + assertEquals(new NodeResources(5, 6, 8, 4.5, NodeResources.DiskSpeed.fast, NodeResources.StorageType.remote), + capacity.freeCapacityOf(host3, false)); doAnswer(invocation -> { NodeResources totalHostResources = (NodeResources) invocation.getArguments()[0]; return totalHostResources.subtract(new NodeResources(1, 2, 3, 0.5, NodeResources.DiskSpeed.any)); }).when(hostResourcesCalculator).availableCapacityOf(any()); - assertEquals(new NodeResources(4, 2, 5, 1.5), capacity.freeCapacityOf(host1, false)); - assertEquals(new NodeResources(4, 4, 5, 4), capacity.freeCapacityOf(host3, false)); + assertEquals(new NodeResources(4, 2, 5, 1.5, NodeResources.DiskSpeed.fast, NodeResources.StorageType.remote), + capacity.freeCapacityOf(host1, false)); + assertEquals(new NodeResources(4, 4, 5, 4, NodeResources.DiskSpeed.fast, NodeResources.StorageType.remote), + capacity.freeCapacityOf(host3, false)); } private Set<String> generateIPs(int start, int count) { diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java index a8b534e02ef..5a00d9d827b 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java @@ -330,7 +330,7 @@ public class ProvisioningTest { { // Deploy with disk-speed any and make sure that information is retained SystemState state = prepare(application, 0, 0, 3, 3, - defaultResources.anySpeed(), + defaultResources.justNumbers(), tester); assertEquals(6, state.allHosts.size()); tester.activate(application, state.allHosts); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java index 59c3ef32dcb..1bae70c8d60 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java @@ -847,7 +847,7 @@ public class RestApiTest { Request.Method.POST), "{\"message\":\"Added 1 nodes to the provisioned state\"}"); assertResponseContains(new Request("http://localhost:8080/nodes/v2/node/" + hostname), - "\"minDiskAvailableGb\":1234.0,\"minMainMemoryAvailableGb\":128.0,\"minCpuCores\":64.0,\"fastDisk\":true,\"bandwidthGbps\":15.0,"); + "\"minDiskAvailableGb\":1234.0,\"minMainMemoryAvailableGb\":128.0,\"minCpuCores\":64.0,\"fastDisk\":true,\"remoteStorage\":true,\"bandwidthGbps\":15.0,"); // Test patching with overrides assertResponse(new Request("http://localhost:8080/nodes/v2/node/" + hostname, @@ -861,7 +861,7 @@ public class RestApiTest { Request.Method.PATCH), "{\"message\":\"Updated " + hostname + "\"}"); assertResponseContains(new Request("http://localhost:8080/nodes/v2/node/" + hostname), - "\"minDiskAvailableGb\":5432.0,\"minMainMemoryAvailableGb\":128.0,\"minCpuCores\":64.0,\"fastDisk\":true,\"bandwidthGbps\":15.0,"); + "\"minDiskAvailableGb\":5432.0,\"minMainMemoryAvailableGb\":128.0,\"minCpuCores\":64.0,\"fastDisk\":true,\"remoteStorage\":true,\"bandwidthGbps\":15.0,"); } @Test @@ -878,20 +878,20 @@ public class RestApiTest { assertResponse(new Request("http://localhost:8080/nodes/v2/node", ("[{\"hostname\":\"" + hostname + "\"," + createIpAddresses("::1") + "\"openStackId\":\"osid-123\"," + - "\"minDiskAvailableGb\":1234,\"minMainMemoryAvailableGb\":4321,\"minCpuCores\":5,\"fastDisk\":false,\"bandwidthGbps\":0.3}]") + "\"minDiskAvailableGb\":1234,\"minMainMemoryAvailableGb\":4321,\"minCpuCores\":5,\"fastDisk\":false,\"remoteStorage\":false,\"bandwidthGbps\":0.3}]") .getBytes(StandardCharsets.UTF_8), Request.Method.POST), "{\"message\":\"Added 1 nodes to the provisioned state\"}"); assertResponseContains(new Request("http://localhost:8080/nodes/v2/node/" + hostname), - "\"minDiskAvailableGb\":1234.0,\"minMainMemoryAvailableGb\":4321.0,\"minCpuCores\":5.0,\"fastDisk\":false,\"bandwidthGbps\":0.3,"); + "\"minDiskAvailableGb\":1234.0,\"minMainMemoryAvailableGb\":4321.0,\"minCpuCores\":5.0,\"fastDisk\":false,\"remoteStorage\":false,\"bandwidthGbps\":0.3,"); // Test patching with overrides assertResponse(new Request("http://localhost:8080/nodes/v2/node/" + hostname, - "{\"minDiskAvailableGb\":12,\"minMainMemoryAvailableGb\":34,\"minCpuCores\":56,\"fastDisk\":true,\"bandwidthGbps\":78.0}".getBytes(StandardCharsets.UTF_8), + "{\"minDiskAvailableGb\":12,\"minMainMemoryAvailableGb\":34,\"minCpuCores\":56,\"fastDisk\":true,\"remoteStorage\":true,\"bandwidthGbps\":78.0}".getBytes(StandardCharsets.UTF_8), Request.Method.PATCH), "{\"message\":\"Updated " + hostname + "\"}"); assertResponseContains(new Request("http://localhost:8080/nodes/v2/node/" + hostname), - "\"minDiskAvailableGb\":12.0,\"minMainMemoryAvailableGb\":34.0,\"minCpuCores\":56.0,\"fastDisk\":true,\"bandwidthGbps\":78.0"); + "\"minDiskAvailableGb\":12.0,\"minMainMemoryAvailableGb\":34.0,\"minCpuCores\":56.0,\"fastDisk\":true,\"remoteStorage\":true,\"bandwidthGbps\":78.0"); } private static String asDockerNodeJson(String hostname, String parentHostname, String... ipAddress) { diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/cfg1.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/cfg1.json index ba9467edb19..b48c719826b 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/cfg1.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/cfg1.json @@ -11,6 +11,7 @@ "minMainMemoryAvailableGb": 16.0, "minCpuCores": 2.0, "fastDisk": true, + "remoteStorage": true, "bandwidthGbps": 10.0, "environment": "BARE_METAL", "rebootGeneration": 1, diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/cfg2.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/cfg2.json index aff38ae5403..0c32768ff78 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/cfg2.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/cfg2.json @@ -11,6 +11,7 @@ "minMainMemoryAvailableGb": 16.0, "minCpuCores": 2.0, "fastDisk": true, + "remoteStorage": true, "bandwidthGbps": 10.0, "environment": "BARE_METAL", "rebootGeneration": 1, diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/controller1.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/controller1.json index 511853d980c..e7685b3195b 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/controller1.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/controller1.json @@ -11,6 +11,7 @@ "minMainMemoryAvailableGb": 16.0, "minCpuCores": 2.0, "fastDisk": true, + "remoteStorage": true, "bandwidthGbps": 10.0, "environment": "BARE_METAL", "rebootGeneration": 0, diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-container1.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-container1.json index e419d709490..9bb28813596 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-container1.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-container1.json @@ -6,12 +6,13 @@ "hostname": "test-node-pool-102-2", "parentHostname": "dockerhost3.yahoo.com", "openStackId": "fake-test-node-pool-102-2", - "flavor": "[vcpu: 1.0, memory: 4.0 Gb, disk 100.0 Gb, bandwidth: 1.0 Gbps]", - "canonicalFlavor": "[vcpu: 1.0, memory: 4.0 Gb, disk 100.0 Gb, bandwidth: 1.0 Gbps]", + "flavor": "[vcpu: 1.0, memory: 4.0 Gb, disk 100.0 Gb, bandwidth: 1.0 Gbps, storage type: remote]", + "canonicalFlavor": "[vcpu: 1.0, memory: 4.0 Gb, disk 100.0 Gb, bandwidth: 1.0 Gbps, storage type: remote]", "minDiskAvailableGb": 100.0, "minMainMemoryAvailableGb": 4.0, "minCpuCores": 1.0, "fastDisk": true, + "remoteStorage": true, "bandwidthGbps": 1.0, "environment": "DOCKER_CONTAINER", "owner": { @@ -30,7 +31,7 @@ "currentRestartGeneration": 0, "wantedDockerImage": "docker-registry.domain.tld:8080/dist/vespa:6.42.0", "wantedVespaVersion": "6.42.0", - "requestedResources": { "vcpu":1.0, "memoryGb":4.0, "diskGb":100.0, "bandwidthGbps":1.0, "diskSpeed":"fast" }, + "requestedResources": { "vcpu":1.0, "memoryGb":4.0, "diskGb":100.0, "bandwidthGbps":1.0 }, "allowedToBeDown": false, "rebootGeneration": 0, "currentRebootGeneration": 0, diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node1-os-upgrade-complete.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node1-os-upgrade-complete.json index bb4ebd3588c..c033369d937 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node1-os-upgrade-complete.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node1-os-upgrade-complete.json @@ -11,6 +11,7 @@ "minMainMemoryAvailableGb": 32.0, "minCpuCores": 4.0, "fastDisk": true, + "remoteStorage": true, "bandwidthGbps": 20.0, "environment": "BARE_METAL", "owner": { @@ -29,7 +30,7 @@ "currentRestartGeneration": 0, "wantedDockerImage": "docker-registry.domain.tld:8080/dist/vespa:6.42.0", "wantedVespaVersion": "6.42.0", - "requestedResources": { "vcpu":4.0, "memoryGb":32.0, "diskGb":1600.0, "bandwidthGbps":20.0, "diskSpeed":"fast" }, + "requestedResources": { "vcpu":4.0, "memoryGb":32.0, "diskGb":1600.0, "bandwidthGbps":20.0, "storageType":"remote" }, "allowedToBeDown": false, "rebootGeneration": 0, "currentRebootGeneration": 0, diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node1-os-upgrade.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node1-os-upgrade.json index 7a1c873040a..23ecec66ea0 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node1-os-upgrade.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node1-os-upgrade.json @@ -29,7 +29,7 @@ "currentRestartGeneration": 0, "wantedDockerImage": "docker-registry.domain.tld:8080/dist/vespa:6.42.0", "wantedVespaVersion": "6.42.0", - "requestedResources": { "vcpu":4.0, "memoryGb":32.0, "diskGb":1600.0, "bandwidthGbps":20.0, "diskSpeed":"fast" }, + "requestedResources": { "vcpu":4.0, "memoryGb":32.0, "diskGb":1600.0, "bandwidthGbps":20.0 }, "allowedToBeDown": false, "rebootGeneration": 0, "currentRebootGeneration": 0, diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node1.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node1.json index 7a15e49c4c1..56c0cc7b824 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node1.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node1.json @@ -11,6 +11,7 @@ "minMainMemoryAvailableGb": 32.0, "minCpuCores": 4.0, "fastDisk": true, + "remoteStorage": true, "bandwidthGbps": 20.0, "environment": "BARE_METAL", "owner": { @@ -29,7 +30,7 @@ "currentRestartGeneration": 0, "wantedDockerImage": "docker-registry.domain.tld:8080/dist/vespa:6.42.0", "wantedVespaVersion": "6.42.0", - "requestedResources": { "vcpu":4.0, "memoryGb":32.0, "diskGb":1600.0, "bandwidthGbps":20.0, "diskSpeed":"fast" }, + "requestedResources": { "vcpu":4.0, "memoryGb":32.0, "diskGb":1600.0, "bandwidthGbps":20.0, "storageType":"remote" }, "allowedToBeDown": false, "rebootGeneration": 0, "currentRebootGeneration": 0, diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node2.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node2.json index fb9d8675431..e56caea2a2e 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node2.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node2.json @@ -11,6 +11,7 @@ "minMainMemoryAvailableGb": 32.0, "minCpuCores": 4.0, "fastDisk": true, + "remoteStorage": true, "bandwidthGbps": 20.0, "environment": "BARE_METAL", "owner": { @@ -29,7 +30,7 @@ "currentRestartGeneration": 0, "wantedDockerImage": "docker-registry.domain.tld:8080/dist/vespa:6.42.0", "wantedVespaVersion": "6.42.0", - "requestedResources": { "vcpu":4.0, "memoryGb":32.0, "diskGb":1600.0, "bandwidthGbps":20.0, "diskSpeed":"fast" }, + "requestedResources": { "vcpu":4.0, "memoryGb":32.0, "diskGb":1600.0, "bandwidthGbps":20.0, "storageType":"remote" }, "allowedToBeDown": false, "rebootGeneration": 0, "currentRebootGeneration": 0, diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node3.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node3.json index 3e750c75403..aa41c7ca1ee 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node3.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node3.json @@ -11,6 +11,7 @@ "minMainMemoryAvailableGb": 32.0, "minCpuCores": 4.0, "fastDisk": true, + "remoteStorage": true, "bandwidthGbps": 20.0, "environment": "BARE_METAL", "owner": { @@ -29,7 +30,7 @@ "currentRestartGeneration": 0, "wantedDockerImage": "docker-registry.domain.tld:8080/dist/vespa:6.42.0", "wantedVespaVersion": "6.42.0", - "requestedResources": { "vcpu":4.0, "memoryGb":32.0, "diskGb":1600.0, "bandwidthGbps":20.0, "diskSpeed":"fast" }, + "requestedResources": { "vcpu":4.0, "memoryGb":32.0, "diskGb":1600.0, "bandwidthGbps":20.0, "storageType":"remote" }, "allowedToBeDown": false, "rebootGeneration": 0, "currentRebootGeneration": 0, diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node4.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node4.json index 58aa6c4f60e..2b3956b3ece 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node4.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node4.json @@ -11,6 +11,7 @@ "minMainMemoryAvailableGb": 32.0, "minCpuCores": 4.0, "fastDisk": true, + "remoteStorage": true, "bandwidthGbps": 20.0, "environment": "BARE_METAL", "owner": { @@ -29,7 +30,7 @@ "currentRestartGeneration": 0, "wantedDockerImage": "docker-registry.domain.tld:8080/dist/vespa:6.42.0", "wantedVespaVersion": "6.42.0", - "requestedResources": { "vcpu":4.0, "memoryGb":32.0, "diskGb":1600.0, "bandwidthGbps":20.0, "diskSpeed":"fast" }, + "requestedResources": { "vcpu":4.0, "memoryGb":32.0, "diskGb":1600.0, "bandwidthGbps":20.0, "storageType":"remote" }, "allowedToBeDown": false, "rebootGeneration": 0, "currentRebootGeneration": 0, diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node5.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node5.json index 80b09c7460c..ce10aba763a 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node5.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node5.json @@ -11,6 +11,7 @@ "minMainMemoryAvailableGb": 32.0, "minCpuCores": 4.0, "fastDisk": true, + "remoteStorage": true, "bandwidthGbps": 20.0, "environment": "BARE_METAL", "owner": { @@ -29,7 +30,7 @@ "currentRestartGeneration": 0, "wantedDockerImage": "docker-registry.domain.tld:8080/dist/vespa:6.42.0", "wantedVespaVersion": "6.42.0", - "requestedResources": { "vcpu":4.0, "memoryGb":32.0, "diskGb":1600.0, "bandwidthGbps":20.0, "diskSpeed":"fast" }, + "requestedResources": { "vcpu":4.0, "memoryGb":32.0, "diskGb":1600.0, "bandwidthGbps":20.0, "storageType":"remote" }, "allowedToBeDown": false, "rebootGeneration": 0, "currentRebootGeneration": 0, diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/dockerhost1-with-firmware-data.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/dockerhost1-with-firmware-data.json index a892b6ee142..4a1c7fcf4ac 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/dockerhost1-with-firmware-data.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/dockerhost1-with-firmware-data.json @@ -11,6 +11,7 @@ "minMainMemoryAvailableGb": 32.0, "minCpuCores": 4.0, "fastDisk": true, + "remoteStorage": true, "bandwidthGbps": 20.0, "environment": "BARE_METAL", "owner": { @@ -29,7 +30,7 @@ "currentRestartGeneration": 0, "wantedDockerImage": "docker-registry.domain.tld:8080/dist/vespa:6.42.0", "wantedVespaVersion": "6.42.0", - "requestedResources": { "vcpu":4.0, "memoryGb":32.0, "diskGb":1600.0, "bandwidthGbps":20.0, "diskSpeed":"fast" }, + "requestedResources": { "vcpu":4.0, "memoryGb":32.0, "diskGb":1600.0, "bandwidthGbps":20.0, "storageType":"remote" }, "allowedToBeDown": false, "rebootGeneration": 0, "currentRebootGeneration": 0, diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node1.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node1.json index e8a6f623b5f..48b0806e0d4 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node1.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node1.json @@ -29,7 +29,7 @@ "currentRestartGeneration": 0, "wantedDockerImage": "docker-registry.domain.tld:8080/dist/vespa:6.42.0", "wantedVespaVersion": "6.42.0", - "requestedResources": { "vcpu":2.0, "memoryGb":8.0, "diskGb":50.0, "bandwidthGbps":1.0, "diskSpeed":"fast" }, + "requestedResources": { "vcpu":2.0, "memoryGb":8.0, "diskGb":50.0, "bandwidthGbps":1.0 }, "allowedToBeDown": false, "rebootGeneration": 1, "currentRebootGeneration": 0, diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node10.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node10.json index 82a7d26d6c6..3eaf10a0813 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node10.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node10.json @@ -30,7 +30,7 @@ "currentRestartGeneration": 0, "wantedDockerImage": "docker-registry.domain.tld:8080/dist/vespa:6.42.0", "wantedVespaVersion": "6.42.0", - "requestedResources": { "vcpu":2.0, "memoryGb":8.0, "diskGb":50.0, "bandwidthGbps":1.0, "diskSpeed":"fast" }, + "requestedResources": { "vcpu":2.0, "memoryGb":8.0, "diskGb":50.0, "bandwidthGbps":1.0 }, "allowedToBeDown": false, "rebootGeneration": 1, "currentRebootGeneration": 0, diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node13.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node13.json index 2ffccb2cc92..6d5cc0cb2b5 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node13.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node13.json @@ -29,7 +29,7 @@ "currentRestartGeneration": 0, "wantedDockerImage": "docker-registry.domain.tld:8080/dist/vespa:6.42.0", "wantedVespaVersion": "6.42.0", - "requestedResources": { "vcpu":10.0, "memoryGb":48.0, "diskGb":500.0, "bandwidthGbps":1.0, "diskSpeed":"fast" }, + "requestedResources": { "vcpu":10.0, "memoryGb":48.0, "diskGb":500.0, "bandwidthGbps":1.0 }, "allowedToBeDown": false, "rebootGeneration": 0, "currentRebootGeneration": 0, diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node14.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node14.json index c406a2a9de0..50a6a0c8408 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node14.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node14.json @@ -29,7 +29,7 @@ "currentRestartGeneration": 0, "wantedDockerImage": "docker-registry.domain.tld:8080/dist/vespa:6.42.0", "wantedVespaVersion": "6.42.0", - "requestedResources": { "vcpu":10.0, "memoryGb":48.0, "diskGb":500.0, "bandwidthGbps":1.0, "diskSpeed":"fast" }, + "requestedResources": { "vcpu":10.0, "memoryGb":48.0, "diskGb":500.0, "bandwidthGbps":1.0 }, "allowedToBeDown": false, "rebootGeneration": 0, "currentRebootGeneration": 0, diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node2.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node2.json index 8fbd23e4d84..fb3e152f772 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node2.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node2.json @@ -29,7 +29,7 @@ "currentRestartGeneration": 0, "wantedDockerImage": "docker-registry.domain.tld:8080/dist/vespa:6.42.0", "wantedVespaVersion": "6.42.0", - "requestedResources": { "vcpu":2.0, "memoryGb":8.0, "diskGb":50.0, "bandwidthGbps":1.0, "diskSpeed":"fast" }, + "requestedResources": { "vcpu":2.0, "memoryGb":8.0, "diskGb":50.0, "bandwidthGbps":1.0 }, "allowedToBeDown": false, "rebootGeneration": 1, "currentRebootGeneration": 0, diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node4-after-changes.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node4-after-changes.json index 9d2cec94367..6a4e35a39f9 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node4-after-changes.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node4-after-changes.json @@ -12,6 +12,7 @@ "minMainMemoryAvailableGb": 8.0, "minCpuCores": 2.0, "fastDisk": true, + "remoteStorage": true, "bandwidthGbps": 1.0, "environment": "DOCKER_CONTAINER", "owner": { @@ -30,7 +31,7 @@ "currentRestartGeneration": 1, "wantedDockerImage": "docker.domain.tld/my/image:6.42.0", "wantedVespaVersion": "6.42.0", - "requestedResources": { "vcpu":1.0, "memoryGb":4.0, "diskGb":100.0, "bandwidthGbps":1.0, "diskSpeed":"fast" }, + "requestedResources": { "vcpu":1.0, "memoryGb":4.0, "diskGb":100.0, "bandwidthGbps":1.0 }, "allowedToBeDown": false, "rebootGeneration": 3, "currentRebootGeneration": 1, diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node4.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node4.json index 30d0e292b3e..56b0c00219c 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node4.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node4.json @@ -30,7 +30,7 @@ "currentRestartGeneration": 0, "wantedDockerImage": "docker-registry.domain.tld:8080/dist/vespa:6.42.0", "wantedVespaVersion": "6.42.0", - "requestedResources": { "vcpu":1.0, "memoryGb":4.0, "diskGb":100.0, "bandwidthGbps":1.0, "diskSpeed":"fast" }, + "requestedResources": { "vcpu":1.0, "memoryGb":4.0, "diskGb":100.0, "bandwidthGbps":1.0 }, "allowedToBeDown": false, "rebootGeneration": 1, "currentRebootGeneration": 0, diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node6-reports-2.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node6-reports-2.json index 01e2fcf7e09..29825b38cae 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node6-reports-2.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node6-reports-2.json @@ -29,7 +29,7 @@ "currentRestartGeneration": 0, "wantedDockerImage": "docker-registry.domain.tld:8080/dist/vespa:6.42.0", "wantedVespaVersion": "6.42.0", - "requestedResources": { "vcpu":2.0, "memoryGb":8.0, "diskGb":50.0, "bandwidthGbps":1.0, "diskSpeed":"fast" }, + "requestedResources": { "vcpu":2.0, "memoryGb":8.0, "diskGb":50.0, "bandwidthGbps":1.0 }, "allowedToBeDown": false, "rebootGeneration": 1, "currentRebootGeneration": 0, diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node6-reports-3.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node6-reports-3.json index 39cce29e38e..4d63c91ab3c 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node6-reports-3.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node6-reports-3.json @@ -29,7 +29,7 @@ "currentRestartGeneration": 0, "wantedDockerImage": "docker-registry.domain.tld:8080/dist/vespa:6.42.0", "wantedVespaVersion": "6.42.0", - "requestedResources": { "vcpu":2.0, "memoryGb":8.0, "diskGb":50.0, "bandwidthGbps":1.0, "diskSpeed":"fast" }, + "requestedResources": { "vcpu":2.0, "memoryGb":8.0, "diskGb":50.0, "bandwidthGbps":1.0 }, "allowedToBeDown": false, "rebootGeneration": 1, "currentRebootGeneration": 0, diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node6-reports.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node6-reports.json index 01fdb4d9e72..0edb3eee76a 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node6-reports.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node6-reports.json @@ -29,7 +29,7 @@ "currentRestartGeneration": 0, "wantedDockerImage": "docker-registry.domain.tld:8080/dist/vespa:6.42.0", "wantedVespaVersion": "6.42.0", - "requestedResources": { "vcpu":2.0, "memoryGb":8.0, "diskGb":50.0, "bandwidthGbps":1.0, "diskSpeed":"fast" }, + "requestedResources": { "vcpu":2.0, "memoryGb":8.0, "diskGb":50.0, "bandwidthGbps":1.0 }, "allowedToBeDown": false, "rebootGeneration": 1, "currentRebootGeneration": 0, diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node6.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node6.json index 5a74e65c43a..3bc3e1fb04c 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node6.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node6.json @@ -29,7 +29,7 @@ "currentRestartGeneration": 0, "wantedDockerImage": "docker-registry.domain.tld:8080/dist/vespa:6.42.0", "wantedVespaVersion": "6.42.0", - "requestedResources": { "vcpu":2.0, "memoryGb":8.0, "diskGb":50.0, "bandwidthGbps":1.0, "diskSpeed":"fast" }, + "requestedResources": { "vcpu":2.0, "memoryGb":8.0, "diskGb":50.0, "bandwidthGbps":1.0 }, "allowedToBeDown": false, "rebootGeneration": 1, "currentRebootGeneration": 0, diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node8.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node8.json index a6e1ced6c2f..c597d9be8b5 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node8.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node8.json @@ -11,6 +11,7 @@ "minMainMemoryAvailableGb": 16.0, "minCpuCores": 2.0, "fastDisk": true, + "remoteStorage": true, "bandwidthGbps": 10.0, "environment": "BARE_METAL", "rebootGeneration": 0, diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node9.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node9.json index c23f89050bb..1cca5079345 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node9.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node9.json @@ -11,6 +11,7 @@ "minMainMemoryAvailableGb": 128.0, "minCpuCores": 64.0, "fastDisk": true, + "remoteStorage": true, "bandwidthGbps": 15.0, "environment": "BARE_METAL", "rebootGeneration": 0, diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/parent2.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/parent2.json index ae4663edb7c..1a056fac185 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/parent2.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/parent2.json @@ -10,7 +10,8 @@ "minDiskAvailableGb": 2000.0, "minMainMemoryAvailableGb": 128.0, "minCpuCores": 64.0, - "fastDisk":true, + "fastDisk": true, + "remoteStorage": true, "bandwidthGbps": 15.0, "environment": "BARE_METAL", "rebootGeneration": 0, diff --git a/searchlib/src/vespa/searchlib/attribute/attributevector.cpp b/searchlib/src/vespa/searchlib/attribute/attributevector.cpp index 86f0c683497..833d831466b 100644 --- a/searchlib/src/vespa/searchlib/attribute/attributevector.cpp +++ b/searchlib/src/vespa/searchlib/attribute/attributevector.cpp @@ -663,8 +663,13 @@ AttributeVector::clearDocs(DocId lidLow, DocId lidLimit) { assert(lidLow <= lidLimit); assert(lidLimit <= getNumDocs()); + uint32_t count = 0; + constexpr uint32_t commit_interval = 1000; for (DocId lid = lidLow; lid < lidLimit; ++lid) { clearDoc(lid); + if ((++count % commit_interval) == 0) { + commit(); + } } } diff --git a/zookeeper-server/zookeeper-server-3.5/pom.xml b/zookeeper-server/zookeeper-server-3.5/pom.xml index 1d14411f543..d283221457c 100644 --- a/zookeeper-server/zookeeper-server-3.5/pom.xml +++ b/zookeeper-server/zookeeper-server-3.5/pom.xml @@ -24,6 +24,10 @@ </dependency> <dependency> <groupId>org.slf4j</groupId> + <artifactId>slf4j-jdk14</artifactId> + </dependency> + <dependency> + <groupId>org.slf4j</groupId> <artifactId>slf4j-log4j12</artifactId> <version>1.7.5</version> </dependency> |