diff options
Diffstat (limited to 'config-model')
14 files changed, 65 insertions, 138 deletions
diff --git a/config-model/CMakeLists.txt b/config-model/CMakeLists.txt index 309363d5be1..e5f8c388613 100644 --- a/config-model/CMakeLists.txt +++ b/config-model/CMakeLists.txt @@ -6,6 +6,6 @@ vespa_install_script(src/main/perl/vespa-expand-config.pl bin) vespa_install_script(src/main/sh/vespa-validate-application bin) install(DIRECTORY src/main/resources/schema DESTINATION share/vespa PATTERN ".gitignore" EXCLUDE) -install(DIRECTORY src/main/resources/schema DESTINATION share/vespa/schema/version/6.x PATTERN ".gitignore" EXCLUDE) +install(DIRECTORY src/main/resources/schema DESTINATION share/vespa/schema/version/7.x PATTERN ".gitignore" EXCLUDE) install_java_artifact(jing) diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java b/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java index 0f43be96e73..b286b94c699 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java @@ -22,7 +22,6 @@ import com.yahoo.config.model.application.provider.MockFileRegistry; import com.yahoo.config.model.provision.HostsXmlProvisioner; import com.yahoo.config.model.provision.SingleNodeProvisioner; import com.yahoo.config.model.test.MockApplicationPackage; -import com.yahoo.config.provision.Rotation; import com.yahoo.config.provision.Zone; import com.yahoo.io.IOUtils; import com.yahoo.io.reader.NamedReader; @@ -47,7 +46,6 @@ import java.io.UncheckedIOException; import java.time.Instant; import java.util.Collection; import java.util.Collections; -import java.util.HashSet; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; diff --git a/config-model/src/main/java/com/yahoo/config/model/provision/SingleNodeProvisioner.java b/config-model/src/main/java/com/yahoo/config/model/provision/SingleNodeProvisioner.java index b7035594c71..180a16f3c8f 100644 --- a/config-model/src/main/java/com/yahoo/config/model/provision/SingleNodeProvisioner.java +++ b/config-model/src/main/java/com/yahoo/config/model/provision/SingleNodeProvisioner.java @@ -32,7 +32,7 @@ public class SingleNodeProvisioner implements HostProvisioner { } public SingleNodeProvisioner(Flavor flavor) { host = new Host(HostName.getLocalhost()); - this.hostSpec = new HostSpec(host.hostname(), host.aliases(),flavor); + this.hostSpec = new HostSpec(host.hostname(), host.aliases(), flavor); } @Override diff --git a/config-model/src/main/java/com/yahoo/vespa/model/HostPorts.java b/config-model/src/main/java/com/yahoo/vespa/model/HostPorts.java index 51b41e19a27..a80982fe75b 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/HostPorts.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/HostPorts.java @@ -20,10 +20,6 @@ import java.util.logging.Level; */ public class HostPorts { - public HostPorts(String hostname) { - this.hostname = hostname; - } - final String hostname; public final static int BASE_PORT = 19100; final static int MAX_PORTS = 799; @@ -42,6 +38,10 @@ public class HostPorts { private Optional<NetworkPorts> networkPortsList = Optional.empty(); + public HostPorts(String hostname) { + this.hostname = hostname; + } + /** * Get the allocated network ports. * Should be called after allocation is complete and flushPortReservations has been called diff --git a/config-model/src/main/java/com/yahoo/vespa/model/HostResource.java b/config-model/src/main/java/com/yahoo/vespa/model/HostResource.java index 0d4c0aa28af..9dba6fde9d4 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/HostResource.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/HostResource.java @@ -38,8 +38,6 @@ public class HostResource implements Comparable<HostResource> { /** Map from "sentinel name" to service */ private final Map<String, Service> services = new LinkedHashMap<>(); - private Set<ClusterMembership> clusterMemberships = new LinkedHashSet<>(); - /** * Create a new {@link HostResource} bound to a specific {@link com.yahoo.vespa.model.Host}. * @@ -64,7 +62,7 @@ public class HostResource implements Comparable<HostResource> { public HostPorts ports() { return hostPorts; } - public HostSpec spec() { return spec; } + public HostSpec spec() { return spec.withPorts(hostPorts.networkPorts()); } /** * Adds service and allocates resources for it. @@ -109,31 +107,6 @@ public class HostResource implements Comparable<HostResource> { /** Returns the flavor of this resource. Empty for self-hosted Vespa. */ public Optional<Flavor> getFlavor() { return spec.flavor(); } - public void addClusterMembership(ClusterMembership clusterMembership) { - if (clusterMembership != null) - clusterMemberships.add(clusterMembership); - } - - public Set<ClusterMembership> clusterMemberships() { - return Collections.unmodifiableSet(clusterMemberships); - } - - /** - * Returns the "primary" cluster membership. - * Content clusters are preferred, then container clusters, and finally admin clusters. - * If there is more than one cluster of the preferred type, the cluster that was added first will be chosen. - */ - public Optional<ClusterMembership> primaryClusterMembership() { - return clusterMemberships().stream() - .sorted(HostResource::compareClusters) - .findFirst(); - } - - private static int compareClusters(ClusterMembership cluster1, ClusterMembership cluster2) { - // This depends on the declared order of enum constants. - return cluster2.cluster().type().compareTo(cluster1.cluster().type()); - } - @Override public String toString() { return "host '" + host.getHostname() + "'"; @@ -163,10 +136,8 @@ public class HostResource implements Comparable<HostResource> { * Compare by hostname otherwise. */ public int comparePrimarilyByIndexTo(HostResource other) { - Optional<ClusterMembership> thisMembership = this.primaryClusterMembership(); - Optional<ClusterMembership> otherMembership = other.primaryClusterMembership(); - if (thisMembership.isPresent() && otherMembership.isPresent()) - return Integer.compare(thisMembership.get().index(), otherMembership.get().index()); + if (this.spec.membership().isPresent() && other.spec.membership().isPresent()) + return Integer.compare(this.spec.membership().get().index(), other.spec.membership().get().index()); else return this.getHostname().compareTo(other.getHostname()); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/HostSystem.java b/config-model/src/main/java/com/yahoo/vespa/model/HostSystem.java index 7d986eed877..3ac5f794426 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/HostSystem.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/HostSystem.java @@ -116,19 +116,13 @@ public class HostSystem extends AbstractConfigProducer<Host> { public HostResource getHost(String hostAlias) { HostSpec hostSpec = provisioner.allocateHost(hostAlias); - for (HostResource resource : hostname2host.values()) { - if (resource.getHostname().equals(hostSpec.hostname())) { - hostSpec.membership().ifPresent(resource::addClusterMembership); - return resource; - } - } - return addNewHost(hostSpec); + HostResource resource = hostname2host.get(hostSpec.hostname()); + return resource != null ? resource : addNewHost(hostSpec); } private HostResource addNewHost(HostSpec hostSpec) { Host host = Host.createHost(this, hostSpec.hostname()); HostResource hostResource = new HostResource(host, hostSpec); - hostSpec.membership().ifPresent(hostResource::addClusterMembership); hostSpec.networkPorts().ifPresent(np -> hostResource.ports().addNetworkPorts(np)); hostname2host.put(host.getHostname(), hostResource); return hostResource; @@ -177,15 +171,7 @@ public class HostSystem extends AbstractConfigProducer<Host> { } Set<HostSpec> getHostSpecs() { - return getHosts().stream() - .map(host -> new HostSpec(host.getHostname(), - Collections.emptyList(), - host.getFlavor(), - host.primaryClusterMembership(), - host.spec().version(), - host.ports().networkPorts(), - host.spec().requestedResources())) - .collect(Collectors.toCollection(LinkedHashSet::new)); + return getHosts().stream().map(host -> host.spec()).collect(Collectors.toCollection(LinkedHashSet::new)); } /** A provision logger which forwards to a deploy logger */ diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainer.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainer.java index c0dc029e009..719d3256889 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainer.java @@ -120,7 +120,7 @@ public class MetricsProxyContainer extends Container implements public void getConfig(NodeDimensionsConfig.Builder builder) { Map<String, String> dimensions = new LinkedHashMap<>(); if (isHostedVespa) { - getHostResource().primaryClusterMembership().map(ClusterMembership::cluster).ifPresent(cluster -> { + getHostResource().spec().membership().map(ClusterMembership::cluster).ifPresent(cluster -> { dimensions.put(CLUSTER_TYPE, cluster.type().name()); dimensions.put(CLUSTER_ID, cluster.id().value()); }); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/NodesSpecification.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/NodesSpecification.java index 86c72221cab..54850dedbba 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/NodesSpecification.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/NodesSpecification.java @@ -171,7 +171,8 @@ public class NodesSpecification { Optional.ofNullable(resources.stringAttribute("bandwidth")) .map(b -> parseGbAmount(b, "BPS")) .orElse(0.3), - parseOptionalDiskSpeed(resources.stringAttribute("disk-speed")))); + parseOptionalDiskSpeed(resources.stringAttribute("disk-speed")), + parseOptionalStorageType(resources.stringAttribute("storage-type")))); } else if (nodesElement.stringAttribute("flavor") != null) { // legacy fallback return Optional.of(NodeResources.fromLegacyName(nodesElement.stringAttribute("flavor"))); @@ -217,16 +218,27 @@ public class NodesSpecification { } private static NodeResources.DiskSpeed parseOptionalDiskSpeed(String diskSpeedString) { - if (diskSpeedString == null) return NodeResources.DiskSpeed.fast; + if (diskSpeedString == null) return NodeResources.DiskSpeed.getDefault(); switch (diskSpeedString) { case "fast" : return NodeResources.DiskSpeed.fast; case "slow" : return NodeResources.DiskSpeed.slow; - case "any" : return NodeResources.DiskSpeed.any; + case "any" : return NodeResources.DiskSpeed.any; default: throw new IllegalArgumentException("Illegal disk-speed value '" + diskSpeedString + "': Legal values are 'fast', 'slow' and 'any')"); } } + private static NodeResources.StorageType parseOptionalStorageType(String storageTypeString) { + if (storageTypeString == null) return NodeResources.StorageType.getDefault(); + switch (storageTypeString) { + case "remote" : return NodeResources.StorageType.remote; + case "local" : return NodeResources.StorageType.local; + case "any" : return NodeResources.StorageType.any; + default: throw new IllegalArgumentException("Illegal storage-type value '" + storageTypeString + + "': Legal values are 'remote', 'local' and 'any')"); + } + } + @Override public String toString() { return "specification of " + count + (dedicated ? " dedicated " : " ") + "nodes" + diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/HostedSslConnectorFactory.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/HostedSslConnectorFactory.java index 93eaeb0565a..d00ce3974fa 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/HostedSslConnectorFactory.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/HostedSslConnectorFactory.java @@ -17,12 +17,15 @@ public class HostedSslConnectorFactory extends ConnectorFactory { private static final List<String> INSECURE_WHITELISTED_PATHS = List.of("/status.html"); + private final boolean enforceClientAuth; + public HostedSslConnectorFactory(String serverName, TlsSecrets tlsSecrets) { - this(serverName, tlsSecrets, null); + this(serverName, tlsSecrets, null, false); } - public HostedSslConnectorFactory(String serverName, TlsSecrets tlsSecrets, String tlsCaCertificates) { + public HostedSslConnectorFactory(String serverName, TlsSecrets tlsSecrets, String tlsCaCertificates, boolean enforceClientAuth) { super("tls4443", 4443, createSslProvider(serverName, tlsSecrets, tlsCaCertificates)); + this.enforceClientAuth = enforceClientAuth; } private static ConfiguredDirectSslProvider createSslProvider( @@ -41,7 +44,7 @@ public class HostedSslConnectorFactory extends ConnectorFactory { super.getConfig(connectorBuilder); connectorBuilder.tlsClientAuthEnforcer(new ConnectorConfig.TlsClientAuthEnforcer.Builder() .pathWhitelist(INSECURE_WHITELISTED_PATHS) - .enable(true)); + .enable(enforceClientAuth)); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java index 484021ad4d5..073503e9341 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java @@ -338,7 +338,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { JettyHttpServer server = cluster.getHttp().getHttpServer(); String serverName = server.getComponentId().getName(); HostedSslConnectorFactory connectorFactory = authorizeClient - ? new HostedSslConnectorFactory(serverName, deployState.tlsSecrets().get(), deployState.tlsClientAuthority().get()) + ? new HostedSslConnectorFactory(serverName, deployState.tlsSecrets().get(), deployState.tlsClientAuthority().get(), true) : new HostedSslConnectorFactory(serverName, deployState.tlsSecrets().get()); server.addConnector(connectorFactory); } diff --git a/config-model/src/main/resources/schema/common.rnc b/config-model/src/main/resources/schema/common.rnc index c5690f9c915..e3ad942e7b3 100644 --- a/config-model/src/main/resources/schema/common.rnc +++ b/config-model/src/main/resources/schema/common.rnc @@ -27,7 +27,8 @@ Resources = element resources { attribute vcpu { xsd:double { minExclusive = "0.0" } } & attribute memory { xsd:string } & attribute disk { xsd:string } & - attribute disk-speed { xsd:string }? + attribute disk-speed { xsd:string }? & + attribute storage-type { xsd:string }? } OptionalDedicatedNodes = element nodes { diff --git a/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java b/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java index b8b50ba0eaa..93c3c9ea2ea 100644 --- a/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java +++ b/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java @@ -219,10 +219,9 @@ public class ModelProvisioningTest { assertEquals(1, model.getHostSystem().getHosts().size()); HostResource host = model.getHostSystem().getHosts().iterator().next(); - assertEquals(1, host.clusterMemberships().size()); - ClusterMembership membership = host.clusterMemberships().iterator().next(); - assertEquals("container", membership.cluster().type().name()); - assertEquals("container1", membership.cluster().id().value()); + assertTrue(host.spec().membership().isPresent()); + assertEquals("container", host.spec().membership().get().cluster().type().name()); + assertEquals("container1", host.spec().membership().get().cluster().id().value()); } @Test @@ -813,45 +812,51 @@ public class ModelProvisioningTest { "<?xml version='1.0' encoding='utf-8' ?>\n" + "<services>" + " <container version='1.0' id='container'>" + - " <nodes count='3' flavor='container-node'/>" + + " <nodes count='3'>" + + " <resources vcpu='1' memory='1Gb' disk='1Gb'/>" + + " </nodes>" + " </container>" + " <content version='1.0' id='content1'>" + " <redundancy>2</redundancy>" + " <documents>" + " <document type='type1' mode='index'/>" + " </documents>" + - " <nodes count='2' flavor='content1-node'/>" + + " <nodes count='2'>" + + " <resources vcpu='2' memory='2Gb' disk='2Gb'/>" + + " </nodes>" + " </content>" + " <content version='1.0' id='content2'>" + " <redundancy>2</redundancy>" + " <documents>" + " <document type='type1' mode='index'/>" + " </documents>" + - " <nodes count='2' flavor='content2-node'/>" + + " <nodes count='2'>" + + " <resources vcpu='4' memory='4Gb' disk='4Gb'/>" + + " </nodes>" + " </content>" + "</services>"; VespaModelTester tester = new VespaModelTester(); // use different flavors to make the test clearer - tester.addHosts("container-node", 3); - tester.addHosts("content1-node", 2); - tester.addHosts("content2-node", 2); + tester.addHosts(new NodeResources(1, 1, 1, 0.3), 3); + tester.addHosts(new NodeResources(2, 2, 2, 0.3), 2); + tester.addHosts(new NodeResources(4, 4, 4, 0.3), 2); VespaModel model = tester.createModel(services, true); ContentCluster cluster1 = model.getContentClusters().get("content1"); ClusterControllerContainerCluster clusterControllers1 = cluster1.getClusterControllers(); assertEquals(1, clusterControllers1.getContainers().size()); - assertEquals("content1-node0", clusterControllers1.getContainers().get(0).getHostName()); - assertEquals("content1-node1", clusterControllers1.getContainers().get(1).getHostName()); - assertEquals("container-node0", clusterControllers1.getContainers().get(2).getHostName()); + assertEquals("node-2-2-2-02", clusterControllers1.getContainers().get(0).getHostName()); + assertEquals("node-2-2-2-01", clusterControllers1.getContainers().get(1).getHostName()); + assertEquals("node-1-1-1-02", clusterControllers1.getContainers().get(2).getHostName()); ContentCluster cluster2 = model.getContentClusters().get("content2"); ClusterControllerContainerCluster clusterControllers2 = cluster2.getClusterControllers(); assertEquals(3, clusterControllers2.getContainers().size()); - assertEquals("content2-node0", clusterControllers2.getContainers().get(0).getHostName()); - assertEquals("content2-node1", clusterControllers2.getContainers().get(1).getHostName()); + assertEquals("node-4-4-4-02", clusterControllers2.getContainers().get(0).getHostName()); + assertEquals("node-4-4-4-01", clusterControllers2.getContainers().get(1).getHostName()); assertEquals("We do not pick the container used to supplement another cluster", - "container-node1", clusterControllers2.getContainers().get(2).getHostName()); + "node-1-1-1-01", clusterControllers2.getContainers().get(2).getHostName()); } @Test diff --git a/config-model/src/test/java/com/yahoo/vespa/model/HostResourceTest.java b/config-model/src/test/java/com/yahoo/vespa/model/HostResourceTest.java index 0624c2cd23a..234841f2b6c 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/HostResourceTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/HostResourceTest.java @@ -6,9 +6,11 @@ import com.yahoo.config.model.producer.AbstractConfigProducer; import com.yahoo.config.model.test.MockRoot; import com.yahoo.config.provision.ClusterMembership; import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.HostSpec; import org.junit.Test; import java.util.Arrays; +import java.util.Optional; import static com.yahoo.config.provision.ClusterSpec.Type.admin; import static com.yahoo.config.provision.ClusterSpec.Type.container; @@ -38,50 +40,13 @@ public class HostResourceTest { } @Test - public void no_clusters_yields_no_primary_cluster_membership() { - HostResource host = hostResourceWithMemberships(); - assertTrue(host.clusterMemberships().isEmpty()); - - assertFalse(host.primaryClusterMembership().isPresent()); - } - - @Test - public void one_cluster_yields_that_primary_cluster_membership() { + public void host_witrh_membership() { HostResource host = hostResourceWithMemberships(ClusterMembership.from(clusterSpec(container, "jdisc"), 0)); assertClusterMembership(host, container, "jdisc"); } - @Test - public void content_cluster_membership_is_preferred_over_other_types() { - HostResource host = hostResourceWithMemberships( - ClusterMembership.from(clusterSpec(container, "jdisc"), 0), - ClusterMembership.from(clusterSpec(content, "search"), 0), - ClusterMembership.from(clusterSpec(admin, "admin"), 0)); - - assertClusterMembership(host, content, "search"); - } - - @Test - public void container_cluster_membership_is_preferred_over_admin() { - HostResource host = hostResourceWithMemberships( - ClusterMembership.from(clusterSpec(admin, "admin"), 0), - ClusterMembership.from(clusterSpec(container, "jdisc"), 0)); - - assertClusterMembership(host, container, "jdisc"); - } - - @Test - public void cluster_membership_that_was_added_first_is_preferred() { - HostResource host = hostResourceWithMemberships( - ClusterMembership.from(clusterSpec(content, "content1"), 0), - ClusterMembership.from(clusterSpec(content, "content0"), 0), - ClusterMembership.from(clusterSpec(content, "content2"), 0)); - - assertClusterMembership(host, content, "content1"); - } - private void assertClusterMembership(HostResource host, ClusterSpec.Type type, String id) { - ClusterSpec membership = host.primaryClusterMembership().map(ClusterMembership::cluster) + ClusterSpec membership = host.spec().membership().map(ClusterMembership::cluster) .orElseThrow(() -> new RuntimeException("No cluster membership!")); assertEquals(type, membership.type()); @@ -92,14 +57,9 @@ public class HostResourceTest { return ClusterSpec.from(type, ClusterSpec.Id.from(id), ClusterSpec.Group.from(0), Version.fromString("6.42"), false); } - private HostResource mockHostResource(MockRoot root) { - return new HostResource(new Host(root)); - } - - private static HostResource hostResourceWithMemberships(ClusterMembership... memberships) { - HostResource host = new HostResource(Host.createHost(null, "hostname")); - Arrays.asList(memberships).forEach(host::addClusterMembership); - return host; + private static HostResource hostResourceWithMemberships(ClusterMembership membership) { + return new HostResource(Host.createHost(null, "hostname"), + new HostSpec("hostname", Optional.of(membership))); } private static int counter = 0; diff --git a/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTester.java b/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTester.java index 63269c45e5f..b6180ab78b9 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTester.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTester.java @@ -60,15 +60,6 @@ public class VespaModelTester { /** Adds some nodes with resources 1, 3, 9 */ public Hosts addHosts(int count) { return addHosts(new NodeResources(1, 3, 9, 1), count); } - /** Adds some hosts to this system */ - public Hosts addHosts(String flavor, int count) { - return addHosts(Optional.empty(), NodeResources.fromLegacyName(flavor), count); - } - - public Hosts addHosts(Flavor flavor, int count) { - return addHosts(Optional.of(flavor), NodeResources.fromLegacyName(flavor.name()), count); - } - public Hosts addHosts(NodeResources resources, int count) { return addHosts(Optional.of(new Flavor(resources)), resources, count); } |