diff options
20 files changed, 266 insertions, 236 deletions
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java index ca29e348094..0c0701983a4 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java @@ -307,6 +307,14 @@ public class VespaMetricSet { metrics.add(new Metric("content.proton.transport.query.latency.count")); metrics.add(new Metric("content.proton.transport.query.latency.average")); // TODO: Remove in Vespa 8 + // Executors shared between all document dbs + metrics.add(new Metric("content.proton.executor.proton.maxpending.last")); + metrics.add(new Metric("content.proton.executor.flush.maxpending.last")); + metrics.add(new Metric("content.proton.executor.match.maxpending.last")); + metrics.add(new Metric("content.proton.executor.docsum.maxpending.last")); + metrics.add(new Metric("content.proton.executor.shared.maxpending.last")); + metrics.add(new Metric("content.proton.executor.warmup.maxpending.last")); + // jobs metrics.add(new Metric("content.proton.documentdb.job.total.average")); metrics.add(new Metric("content.proton.documentdb.job.attribute_flush.average")); @@ -318,7 +326,7 @@ public class VespaMetricSet { metrics.add(new Metric("content.proton.documentdb.job.lid_space_compact.average")); metrics.add(new Metric("content.proton.documentdb.job.removed_documents_prune.average")); - // Threading service + // Threading service (per document db) metrics.add(new Metric("content.proton.documentdb.threading_service.master.maxpending.last")); metrics.add(new Metric("content.proton.documentdb.threading_service.index.maxpending.last")); metrics.add(new Metric("content.proton.documentdb.threading_service.summary.maxpending.last")); diff --git a/config-provisioning/abi-spec.json b/config-provisioning/abi-spec.json index a9b45fe6bb4..64114389751 100644 --- a/config-provisioning/abi-spec.json +++ b/config-provisioning/abi-spec.json @@ -8,8 +8,6 @@ "methods": [ "public static com.yahoo.config.provision.AllocatedHosts withHosts(java.util.Set)", "public java.util.Set getHosts()", - "public byte[] toJson()", - "public static com.yahoo.config.provision.AllocatedHosts fromJson(byte[], java.util.Optional)", "public boolean equals(java.lang.Object)", "public int hashCode()", "public java.lang.String toString()" @@ -567,19 +565,6 @@ ], "fields": [] }, - "com.yahoo.config.provision.NetworkPortsSerializer": { - "superClass": "java.lang.Object", - "interfaces": [], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>()", - "public static void toSlime(com.yahoo.config.provision.NetworkPorts, com.yahoo.slime.Cursor)", - "public static java.util.Optional fromSlime(com.yahoo.slime.Inspector)" - ], - "fields": [] - }, "com.yahoo.config.provision.NodeFlavors": { "superClass": "java.lang.Object", "interfaces": [], diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/AllocatedHosts.java b/config-provisioning/src/main/java/com/yahoo/config/provision/AllocatedHosts.java index 96942c53a12..c6228dc71e9 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/AllocatedHosts.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/AllocatedHosts.java @@ -2,18 +2,7 @@ package com.yahoo.config.provision; import com.google.common.collect.ImmutableSet; -import com.yahoo.slime.ArrayTraverser; -import com.yahoo.slime.Cursor; -import com.yahoo.slime.Inspector; -import com.yahoo.slime.Slime; -import com.yahoo.vespa.config.SlimeUtils; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.Optional; import java.util.Set; /** @@ -26,38 +15,9 @@ import java.util.Set; */ public class AllocatedHosts { - // WARNING: Since there are multiple servers in a ZooKeeper cluster and they upgrade one by one - // (and rewrite all nodes on startup), changes to the serialized format must be made - // such that what is serialized on version N+1 can be read by version N: - // - ADDING FIELDS: Always ok - // - REMOVING FIELDS: Stop reading the field first. Stop writing it on a later version. - // - CHANGING THE FORMAT OF A FIELD: Don't do it bro. + private final Set<HostSpec> hosts; - private static final String mappingKey = "mapping"; - private static final String hostSpecKey = "hostSpec"; - private static final String hostSpecHostNameKey = "hostName"; - private static final String aliasesKey = "aliases"; - private static final String hostSpecMembershipKey = "membership"; - - // Flavor can be removed when all allocated nodes are docker nodes - private static final String flavorKey = "flavor"; - - private static final String resourcesKey = "resources"; - private static final String vcpuKey = "vcpu"; - private static final String memoryKey = "memory"; - private static final String diskKey = "disk"; - private static final String diskSpeedKey = "diskSpeed"; - - /** Wanted version */ - private static final String hostSpecVespaVersionKey = "vespaVersion"; - - /** Current version */ - private static final String hostSpecCurrentVespaVersionKey = "currentVespaVersion"; - private static final String hostSpecNetworkPortsKey = "ports"; - - private final ImmutableSet<HostSpec> hosts; - - AllocatedHosts(Set<HostSpec> hosts) { + private AllocatedHosts(Set<HostSpec> hosts) { this.hosts = ImmutableSet.copyOf(hosts); } @@ -65,128 +25,8 @@ public class AllocatedHosts { return new AllocatedHosts(hosts); } - private void toSlime(Cursor cursor) { - Cursor array = cursor.setArray(mappingKey); - for (HostSpec host : hosts) - toSlime(host, array.addObject().setObject(hostSpecKey)); - } - - private void toSlime(HostSpec host, Cursor cursor) { - cursor.setString(hostSpecHostNameKey, host.hostname()); - aliasesToSlime(host, cursor); - host.membership().ifPresent(membership -> { - cursor.setString(hostSpecMembershipKey, membership.stringValue()); - cursor.setString(hostSpecVespaVersionKey, membership.cluster().vespaVersion().toFullString()); - }); - host.flavor().ifPresent(flavor -> toSlime(flavor, cursor)); - host.version().ifPresent(version -> cursor.setString(hostSpecCurrentVespaVersionKey, version.toFullString())); - host.networkPorts().ifPresent(ports -> NetworkPortsSerializer.toSlime(ports, cursor.setArray(hostSpecNetworkPortsKey))); - } - - private void aliasesToSlime(HostSpec spec, Cursor cursor) { - if (spec.aliases().isEmpty()) return; - Cursor aliases = cursor.setArray(aliasesKey); - for (String alias : spec.aliases()) - aliases.addString(alias); - } - - private void toSlime(Flavor flavor, Cursor object) { - if (flavor.isConfigured()) { - object.setString(flavorKey, flavor.name()); - } - else { - NodeResources resources = flavor.resources(); - Cursor resourcesObject = object.setObject(resourcesKey); - resourcesObject.setDouble(vcpuKey, resources.vcpu()); - resourcesObject.setDouble(memoryKey, resources.memoryGb()); - resourcesObject.setDouble(diskKey, resources.diskGb()); - resourcesObject.setString(diskSpeedKey, diskSpeedToString(resources.diskSpeed())); - } - } - /** Returns the hosts of this allocation */ public Set<HostSpec> getHosts() { return hosts; } - - private static AllocatedHosts fromSlime(Inspector inspector, Optional<NodeFlavors> nodeFlavors) { - Inspector array = inspector.field(mappingKey); - Set<HostSpec> hosts = new LinkedHashSet<>(); - array.traverse((ArrayTraverser)(i, host) -> hosts.add(hostFromSlime(host.field(hostSpecKey), nodeFlavors))); - return new AllocatedHosts(hosts); - } - - static HostSpec hostFromSlime(Inspector object, Optional<NodeFlavors> nodeFlavors) { - Optional<ClusterMembership> membership = - object.field(hostSpecMembershipKey).valid() ? Optional.of(membershipFromSlime(object)) : Optional.empty(); - Optional<Flavor> flavor = flavorFromSlime(object, nodeFlavors); - Optional<com.yahoo.component.Version> version = - optionalString(object.field(hostSpecCurrentVespaVersionKey)).map(com.yahoo.component.Version::new); - Optional<NetworkPorts> networkPorts = - NetworkPortsSerializer.fromSlime(object.field(hostSpecNetworkPortsKey)); - return new HostSpec(object.field(hostSpecHostNameKey).asString(), aliasesFromSlime(object), flavor, membership, version, networkPorts); - } - - private static List<String> aliasesFromSlime(Inspector object) { - if ( ! object.field(aliasesKey).valid()) return Collections.emptyList(); - List<String> aliases = new ArrayList<>(); - object.field(aliasesKey).traverse((ArrayTraverser)(index, alias) -> aliases.add(alias.asString())); - return aliases; - } - - private static Optional<Flavor> flavorFromSlime(Inspector object, Optional<NodeFlavors> nodeFlavors) { - if (object.field(flavorKey).valid() && nodeFlavors.isPresent() && nodeFlavors.get().exists(object.field(flavorKey).asString())) { - return nodeFlavors.get().getFlavor(object.field(flavorKey).asString()); - } - else if (object.field(resourcesKey).valid()) { - Inspector resources = object.field(resourcesKey); - return Optional.of(new Flavor(new NodeResources(resources.field(vcpuKey).asDouble(), - resources.field(memoryKey).asDouble(), - resources.field(diskKey).asDouble(), - diskSpeedFromSlime(resources.field(diskSpeedKey))))); - } - else { - return Optional.empty(); - } - } - - private static NodeResources.DiskSpeed diskSpeedFromSlime(Inspector diskSpeed) { - if ( ! diskSpeed.valid()) return NodeResources.DiskSpeed.fast; // TODO: Remove this line after June 2019 - switch (diskSpeed.asString()) { - case "fast" : return NodeResources.DiskSpeed.fast; - case "slow" : return NodeResources.DiskSpeed.slow; - case "any" : return NodeResources.DiskSpeed.any; - default: throw new IllegalStateException("Illegal disk-speed value '" + diskSpeed.asString() + "'"); - } - } - - private static String diskSpeedToString(NodeResources.DiskSpeed diskSpeed) { - switch (diskSpeed) { - case fast : return "fast"; - case slow : return "slow"; - case any : return "any"; - default: throw new IllegalStateException("Illegal disk-speed value '" + diskSpeed + "'"); - } - - } - - private static ClusterMembership membershipFromSlime(Inspector object) { - return ClusterMembership.from(object.field(hostSpecMembershipKey).asString(), - com.yahoo.component.Version.fromString(object.field(hostSpecVespaVersionKey).asString())); - } - - private static Optional<String> optionalString(Inspector inspector) { - if ( ! inspector.valid()) return Optional.empty(); - return Optional.of(inspector.asString()); - } - - public byte[] toJson() throws IOException { - Slime slime = new Slime(); - toSlime(slime.setObject()); - return SlimeUtils.toJsonBytes(slime); - } - - public static AllocatedHosts fromJson(byte[] json, Optional<NodeFlavors> nodeFlavors) { - return fromSlime(SlimeUtils.jsonToSlime(json).get(), nodeFlavors); - } @Override public boolean equals(Object other) { diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/Flavor.java b/config-provisioning/src/main/java/com/yahoo/config/provision/Flavor.java index b393d9ee22a..48c84b8ecb7 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/Flavor.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/Flavor.java @@ -1,7 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.config.provision; -import com.google.common.collect.ImmutableList; import com.yahoo.config.provisioning.FlavorsConfig; import java.util.ArrayList; @@ -151,7 +150,7 @@ public class Flavor { /** Irreversibly freezes the content of this */ public void freeze() { - replacesFlavors = ImmutableList.copyOf(replacesFlavors); + replacesFlavors = List.copyOf(replacesFlavors); } @Override diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/HostFilter.java b/config-provisioning/src/main/java/com/yahoo/config/provision/HostFilter.java index 5615cdbd2f1..95eec2ac0fb 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/HostFilter.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/HostFilter.java @@ -1,7 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.config.provision; -import com.google.common.collect.ImmutableSet; import com.yahoo.text.StringUtilities; import java.util.Collection; @@ -78,10 +77,10 @@ public class HostFilter { Collection<String> flavors, Collection<ClusterSpec.Type> clusterTypes, Collection<ClusterSpec.Id> clusterIds) { - return new HostFilter(ImmutableSet.copyOf(hostNames), - ImmutableSet.copyOf(flavors), - ImmutableSet.copyOf(clusterTypes), - ImmutableSet.copyOf(clusterIds)); + return new HostFilter(Set.copyOf(hostNames), + Set.copyOf(flavors), + Set.copyOf(clusterTypes), + Set.copyOf(clusterIds)); } /** Returns a host filter from three comma and-or space separated string lists. The strings may be null or empty. */ diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/HostSpec.java b/config-provisioning/src/main/java/com/yahoo/config/provision/HostSpec.java index e5d4aadb988..ba7a3c2f06b 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/HostSpec.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/HostSpec.java @@ -1,8 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.config.provision; -import com.google.common.collect.ImmutableList; - import java.util.ArrayList; import java.util.List; import java.util.Objects; @@ -20,7 +18,7 @@ public class HostSpec implements Comparable<HostSpec> { private final String hostname; /** Aliases of this host */ - private final ImmutableList<String> aliases; + private final List<String> aliases; /** The current membership role of this host in the cluster it belongs to */ private final Optional<ClusterMembership> membership; @@ -67,7 +65,7 @@ public class HostSpec implements Comparable<HostSpec> { Objects.requireNonNull(version, "Version cannot be null but can be empty"); Objects.requireNonNull(networkPorts, "Network ports cannot be null but can be empty"); this.hostname = hostname; - this.aliases = ImmutableList.copyOf(aliases); + this.aliases = List.copyOf(aliases); this.flavor = flavor; this.membership = membership; this.version = version; diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/NodeFlavors.java b/config-provisioning/src/main/java/com/yahoo/config/provision/NodeFlavors.java index 762c7cbaba5..4d4d3c8cf86 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/NodeFlavors.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/NodeFlavors.java @@ -1,12 +1,12 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.config.provision; -import com.google.common.collect.ImmutableMap; import com.google.inject.Inject; import com.yahoo.config.provisioning.FlavorsConfig; import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -21,14 +21,14 @@ import java.util.stream.Collectors; public class NodeFlavors { /** Flavors which are configured in this zone */ - private final ImmutableMap<String, Flavor> configuredFlavors; + private final Map<String, Flavor> configuredFlavors; @Inject public NodeFlavors(FlavorsConfig config) { - ImmutableMap.Builder<String, Flavor> b = new ImmutableMap.Builder<>(); + HashMap<String, Flavor> b = new HashMap<>(); for (Flavor flavor : toFlavors(config)) b.put(flavor.name(), flavor); - this.configuredFlavors = b.build(); + this.configuredFlavors = Collections.unmodifiableMap(b); } public List<Flavor> getFlavors() { diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/Zone.java b/config-provisioning/src/main/java/com/yahoo/config/provision/Zone.java index eebedd15e2c..33a13b1aa9f 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/Zone.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/Zone.java @@ -1,7 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.config.provision; -import com.google.common.base.Strings; import com.google.inject.Inject; import com.yahoo.cloud.config.ConfigserverConfig; diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/serialization/AllocatedHostsSerializer.java b/config-provisioning/src/main/java/com/yahoo/config/provision/serialization/AllocatedHostsSerializer.java new file mode 100644 index 00000000000..9e01718bfc6 --- /dev/null +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/serialization/AllocatedHostsSerializer.java @@ -0,0 +1,185 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.provision.serialization; + +import com.yahoo.config.provision.AllocatedHosts; +import com.yahoo.config.provision.ClusterMembership; +import com.yahoo.config.provision.Flavor; +import com.yahoo.config.provision.HostSpec; +import com.yahoo.config.provision.NetworkPorts; +import com.yahoo.config.provision.NodeFlavors; +import com.yahoo.config.provision.NodeResources; +import com.yahoo.slime.ArrayTraverser; +import com.yahoo.slime.Cursor; +import com.yahoo.slime.Inspector; +import com.yahoo.slime.Slime; +import com.yahoo.vespa.config.SlimeUtils; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Optional; +import java.util.Set; + +/** + * The hosts allocated to an application. + * This can be serialized to/from JSON. + * This is immutable. + * + * @author Ulf Lilleengen + * @author bratseth + */ +public class AllocatedHostsSerializer { + + // WARNING: Since there are multiple servers in a ZooKeeper cluster and they upgrade one by one + // (and rewrite all nodes on startup), changes to the serialized format must be made + // such that what is serialized on version N+1 can be read by version N: + // - ADDING FIELDS: Always ok + // - REMOVING FIELDS: Stop reading the field first. Stop writing it on a later version. + // - CHANGING THE FORMAT OF A FIELD: Don't do it bro. + + private static final String mappingKey = "mapping"; + private static final String hostSpecKey = "hostSpec"; + private static final String hostSpecHostNameKey = "hostName"; + private static final String aliasesKey = "aliases"; + private static final String hostSpecMembershipKey = "membership"; + + // Flavor can be removed when all allocated nodes are docker nodes + private static final String flavorKey = "flavor"; + + private static final String resourcesKey = "resources"; + private static final String vcpuKey = "vcpu"; + private static final String memoryKey = "memory"; + private static final String diskKey = "disk"; + private static final String diskSpeedKey = "diskSpeed"; + + /** Wanted version */ + private static final String hostSpecVespaVersionKey = "vespaVersion"; + + /** Current version */ + private static final String hostSpecCurrentVespaVersionKey = "currentVespaVersion"; + private static final String hostSpecNetworkPortsKey = "ports"; + + + public static byte[] toJson(AllocatedHosts allocatedHosts) throws IOException { + Slime slime = new Slime(); + toSlime(allocatedHosts, slime.setObject()); + return SlimeUtils.toJsonBytes(slime); + } + + public static void toSlime(AllocatedHosts allocatedHosts, Cursor cursor) { + Cursor array = cursor.setArray(mappingKey); + for (HostSpec host : allocatedHosts.getHosts()) + toSlime(host, array.addObject().setObject(hostSpecKey)); + } + + private static void toSlime(HostSpec host, Cursor cursor) { + cursor.setString(hostSpecHostNameKey, host.hostname()); + aliasesToSlime(host, cursor); + host.membership().ifPresent(membership -> { + cursor.setString(hostSpecMembershipKey, membership.stringValue()); + cursor.setString(hostSpecVespaVersionKey, membership.cluster().vespaVersion().toFullString()); + }); + host.flavor().ifPresent(flavor -> toSlime(flavor, cursor)); + host.version().ifPresent(version -> cursor.setString(hostSpecCurrentVespaVersionKey, version.toFullString())); + host.networkPorts().ifPresent(ports -> NetworkPortsSerializer.toSlime(ports, cursor.setArray(hostSpecNetworkPortsKey))); + } + + private static void aliasesToSlime(HostSpec spec, Cursor cursor) { + if (spec.aliases().isEmpty()) return; + Cursor aliases = cursor.setArray(aliasesKey); + for (String alias : spec.aliases()) + aliases.addString(alias); + } + + private static void toSlime(Flavor flavor, Cursor object) { + if (flavor.isConfigured()) { + object.setString(flavorKey, flavor.name()); + } + else { + NodeResources resources = flavor.resources(); + Cursor resourcesObject = object.setObject(resourcesKey); + resourcesObject.setDouble(vcpuKey, resources.vcpu()); + resourcesObject.setDouble(memoryKey, resources.memoryGb()); + resourcesObject.setDouble(diskKey, resources.diskGb()); + resourcesObject.setString(diskSpeedKey, diskSpeedToString(resources.diskSpeed())); + } + } + + + public static AllocatedHosts fromJson(byte[] json, Optional<NodeFlavors> nodeFlavors) { + return fromSlime(SlimeUtils.jsonToSlime(json).get(), nodeFlavors); + } + + public static AllocatedHosts fromSlime(Inspector inspector, Optional<NodeFlavors> nodeFlavors) { + Inspector array = inspector.field(mappingKey); + Set<HostSpec> hosts = new LinkedHashSet<>(); + array.traverse((ArrayTraverser)(i, host) -> hosts.add(hostFromSlime(host.field(hostSpecKey), nodeFlavors))); + return AllocatedHosts.withHosts(hosts); + } + + private static HostSpec hostFromSlime(Inspector object, Optional<NodeFlavors> nodeFlavors) { + Optional<ClusterMembership> membership = + object.field(hostSpecMembershipKey).valid() ? Optional.of(membershipFromSlime(object)) : Optional.empty(); + Optional<Flavor> flavor = flavorFromSlime(object, nodeFlavors); + Optional<com.yahoo.component.Version> version = + optionalString(object.field(hostSpecCurrentVespaVersionKey)).map(com.yahoo.component.Version::new); + Optional<NetworkPorts> networkPorts = + NetworkPortsSerializer.fromSlime(object.field(hostSpecNetworkPortsKey)); + return new HostSpec(object.field(hostSpecHostNameKey).asString(), aliasesFromSlime(object), flavor, membership, version, networkPorts); + } + + private static List<String> aliasesFromSlime(Inspector object) { + if ( ! object.field(aliasesKey).valid()) return Collections.emptyList(); + List<String> aliases = new ArrayList<>(); + object.field(aliasesKey).traverse((ArrayTraverser)(index, alias) -> aliases.add(alias.asString())); + return aliases; + } + + private static Optional<Flavor> flavorFromSlime(Inspector object, Optional<NodeFlavors> nodeFlavors) { + if (object.field(flavorKey).valid() && nodeFlavors.isPresent() && nodeFlavors.get().exists(object.field(flavorKey).asString())) { + return nodeFlavors.get().getFlavor(object.field(flavorKey).asString()); + } + else if (object.field(resourcesKey).valid()) { + Inspector resources = object.field(resourcesKey); + return Optional.of(new Flavor(new NodeResources(resources.field(vcpuKey).asDouble(), + resources.field(memoryKey).asDouble(), + resources.field(diskKey).asDouble(), + diskSpeedFromSlime(resources.field(diskSpeedKey))))); + } + else { + return Optional.empty(); + } + } + + private static NodeResources.DiskSpeed diskSpeedFromSlime(Inspector diskSpeed) { + if ( ! diskSpeed.valid()) return NodeResources.DiskSpeed.fast; // TODO: Remove this line after June 2019 + switch (diskSpeed.asString()) { + case "fast" : return NodeResources.DiskSpeed.fast; + case "slow" : return NodeResources.DiskSpeed.slow; + case "any" : return NodeResources.DiskSpeed.any; + default: throw new IllegalStateException("Illegal disk-speed value '" + diskSpeed.asString() + "'"); + } + } + + private static String diskSpeedToString(NodeResources.DiskSpeed diskSpeed) { + switch (diskSpeed) { + case fast : return "fast"; + case slow : return "slow"; + case any : return "any"; + default: throw new IllegalStateException("Illegal disk-speed value '" + diskSpeed + "'"); + } + + } + + private static ClusterMembership membershipFromSlime(Inspector object) { + return ClusterMembership.from(object.field(hostSpecMembershipKey).asString(), + com.yahoo.component.Version.fromString(object.field(hostSpecVespaVersionKey).asString())); + } + + private static Optional<String> optionalString(Inspector inspector) { + if ( ! inspector.valid()) return Optional.empty(); + return Optional.of(inspector.asString()); + } +} diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/NetworkPortsSerializer.java b/config-provisioning/src/main/java/com/yahoo/config/provision/serialization/NetworkPortsSerializer.java index d3af337e9be..e800190b5f8 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/NetworkPortsSerializer.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/serialization/NetworkPortsSerializer.java @@ -1,7 +1,8 @@ // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.config.provision; +package com.yahoo.config.provision.serialization; +import com.yahoo.config.provision.NetworkPorts; import com.yahoo.slime.ArrayTraverser; import com.yahoo.slime.Cursor; import com.yahoo.slime.Inspector; diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/serialization/package-info.java b/config-provisioning/src/main/java/com/yahoo/config/provision/serialization/package-info.java new file mode 100644 index 00000000000..440228b9d3e --- /dev/null +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/serialization/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.config.provision.serialization; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/zone/ZoneList.java b/config-provisioning/src/main/java/com/yahoo/config/provision/zone/ZoneList.java index 776f925c424..4f9e335e4d8 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/zone/ZoneList.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/zone/ZoneList.java @@ -1,10 +1,8 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.config.provision.zone; -import com.google.common.collect.ImmutableList; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.RegionName; -import com.yahoo.config.provision.Zone; import java.util.List; import java.util.stream.Collectors; diff --git a/config-provisioning/src/test/java/com/yahoo/config/provision/AllocatedHostsTest.java b/config-provisioning/src/test/java/com/yahoo/config/provision/serialization/AllocatedHostsSerializerTest.java index 737c1047197..44aa6ed59b5 100644 --- a/config-provisioning/src/test/java/com/yahoo/config/provision/AllocatedHostsTest.java +++ b/config-provisioning/src/test/java/com/yahoo/config/provision/serialization/AllocatedHostsSerializerTest.java @@ -1,7 +1,14 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.config.provision; +package com.yahoo.config.provision.serialization; import com.yahoo.component.Version; +import com.yahoo.config.provision.AllocatedHosts; +import com.yahoo.config.provision.ClusterMembership; +import com.yahoo.config.provision.Flavor; +import com.yahoo.config.provision.HostSpec; +import com.yahoo.config.provision.NetworkPorts; +import com.yahoo.config.provision.NodeFlavors; +import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provisioning.FlavorsConfig; import org.junit.Test; @@ -12,12 +19,14 @@ import java.util.List; import java.util.Optional; import java.util.Set; +import static com.yahoo.config.provision.serialization.AllocatedHostsSerializer.fromJson; +import static com.yahoo.config.provision.serialization.AllocatedHostsSerializer.toJson; import static org.junit.Assert.assertEquals; /** * @author bratseth */ -public class AllocatedHostsTest { +public class AllocatedHostsSerializerTest { @Test public void testAllocatedHostsSerialization() throws IOException { @@ -29,7 +38,7 @@ public class AllocatedHostsTest { hosts.add(new HostSpec("with-aliases", List.of("alias1", "alias2"))); hosts.add(new HostSpec("allocated", - Optional.of(ClusterMembership.from("container/test/0/0", com.yahoo.component.Version.fromString("6.73.1"))))); + Optional.of(ClusterMembership.from("container/test/0/0", Version.fromString("6.73.1"))))); hosts.add(new HostSpec("flavor-from-resources-1", Collections.emptyList(), new Flavor(new NodeResources(0.5, 3.1, 4)))); hosts.add(new HostSpec("flavor-from-resources-2", @@ -47,8 +56,7 @@ public class AllocatedHostsTest { } private void assertAllocatedHosts(AllocatedHosts expectedHosts, NodeFlavors configuredFlavors) throws IOException { - AllocatedHosts deserializedHosts = AllocatedHosts.fromJson(expectedHosts.toJson(), - Optional.of(configuredFlavors)); + AllocatedHosts deserializedHosts = fromJson(toJson(expectedHosts), Optional.of(configuredFlavors)); assertEquals(expectedHosts, deserializedHosts); for (HostSpec expectedHost : expectedHosts.getHosts()) { @@ -69,7 +77,7 @@ public class AllocatedHostsTest { throw new IllegalArgumentException("No host " + hostname + " is present"); } - public NodeFlavors configuredFlavorsFrom(String flavorName, double cpu, double mem, double disk, Flavor.Type type) { + private NodeFlavors configuredFlavorsFrom(String flavorName, double cpu, double mem, double disk, Flavor.Type type) { FlavorsConfig.Builder b = new FlavorsConfig.Builder(); FlavorsConfig.Flavor.Builder flavor = new FlavorsConfig.Flavor.Builder(); flavor.name(flavorName); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClient.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClient.java index e5552de41f5..110e73bcdf9 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClient.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClient.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.deploy; +import com.yahoo.component.Version; import com.yahoo.config.application.api.ApplicationFile; import com.yahoo.config.application.api.ApplicationMetaData; import com.yahoo.config.application.api.ApplicationPackage; @@ -9,7 +10,7 @@ import com.yahoo.config.application.api.FileRegistry; import com.yahoo.config.application.api.UnparsedConfigDefinition; import com.yahoo.config.model.application.provider.PreGeneratedFileRegistry; import com.yahoo.config.provision.AllocatedHosts; -import com.yahoo.component.Version; +import com.yahoo.config.provision.serialization.AllocatedHostsSerializer; import com.yahoo.io.reader.NamedReader; import com.yahoo.log.LogLevel; import com.yahoo.path.Path; @@ -361,7 +362,9 @@ public class ZooKeeperClient { } public void write(AllocatedHosts hosts) throws IOException { - configCurator.putData(rootPath.append(ZKApplicationPackage.allocatedHostsNode).getAbsolute(), hosts.toJson()); + configCurator.putData( + rootPath.append(ZKApplicationPackage.allocatedHostsNode).getAbsolute(), + AllocatedHostsSerializer.toJson(hosts)); } public void write(Map<Version, FileRegistry> fileRegistryMap) { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackage.java b/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackage.java index 5c40f592a77..e013244c80c 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackage.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackage.java @@ -3,19 +3,20 @@ package com.yahoo.vespa.config.server.zookeeper; import com.google.common.base.Joiner; import com.yahoo.component.Version; +import com.yahoo.config.application.api.ApplicationFile; import com.yahoo.config.application.api.ApplicationMetaData; +import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.application.api.ComponentInfo; import com.yahoo.config.application.api.FileRegistry; import com.yahoo.config.application.api.UnparsedConfigDefinition; import com.yahoo.config.codegen.DefParser; -import com.yahoo.config.application.api.ApplicationFile; -import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.model.application.provider.PreGeneratedFileRegistry; -import com.yahoo.config.provision.NodeFlavors; import com.yahoo.config.provision.AllocatedHosts; +import com.yahoo.config.provision.NodeFlavors; +import com.yahoo.config.provision.serialization.AllocatedHostsSerializer; import com.yahoo.io.IOUtils; -import com.yahoo.path.Path; import com.yahoo.io.reader.NamedReader; +import com.yahoo.path.Path; import com.yahoo.vespa.config.ConfigDefinition; import com.yahoo.vespa.config.ConfigDefinitionBuilder; import com.yahoo.vespa.config.ConfigDefinitionKey; @@ -69,7 +70,7 @@ public class ZKApplicationPackage implements ApplicationPackage { */ private AllocatedHosts readAllocatedHosts(String allocatedHostsPath, Optional<NodeFlavors> nodeFlavors) { try { - return AllocatedHosts.fromJson(liveApp.getBytes(allocatedHostsPath), nodeFlavors); + return AllocatedHostsSerializer.fromJson(liveApp.getBytes(allocatedHostsPath), nodeFlavors); } catch (Exception e) { throw new RuntimeException("Unable to read allocated hosts", e); } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClientTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClientTest.java index e01b2eccb35..918670d71f2 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClientTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClientTest.java @@ -13,9 +13,9 @@ import com.yahoo.config.model.application.provider.MockFileRegistry; import com.yahoo.config.provision.AllocatedHosts; import com.yahoo.config.provision.HostSpec; import com.yahoo.path.Path; +import com.yahoo.vespa.config.server.zookeeper.ConfigCurator; import com.yahoo.vespa.config.server.zookeeper.ZKApplicationPackage; import com.yahoo.vespa.curator.mock.MockCurator; -import com.yahoo.vespa.config.server.zookeeper.ConfigCurator; import org.junit.Before; import org.junit.Ignore; import org.junit.Rule; @@ -30,10 +30,11 @@ import java.util.List; import java.util.Map; import java.util.Optional; +import static com.yahoo.config.provision.serialization.AllocatedHostsSerializer.fromJson; import static org.hamcrest.core.Is.is; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; /** * Unit tests for ZooKeeperClient. @@ -199,7 +200,7 @@ public class ZooKeeperClientTest { Path hostsPath = app.append(ZKApplicationPackage.allocatedHostsNode); assertTrue(zk.exists(hostsPath.getAbsolute())); - AllocatedHosts deserialized = AllocatedHosts.fromJson(zk.getBytes(hostsPath.getAbsolute()), Optional.empty()); + AllocatedHosts deserialized = fromJson(zk.getBytes(hostsPath.getAbsolute()), Optional.empty()); assertEquals(hosts, deserialized.getHosts()); } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackageTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackageTest.java index e8e2dd07756..8b8be1a27d7 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackageTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackageTest.java @@ -1,27 +1,15 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.zookeeper; -import static org.hamcrest.CoreMatchers.is; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.assertThat; - -import java.io.File; -import java.io.IOException; -import java.io.Reader; -import java.util.Collections; -import java.util.Optional; -import java.util.regex.Pattern; - +import com.yahoo.component.Version; import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.config.provision.AllocatedHosts; import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.HostSpec; import com.yahoo.config.provision.NodeFlavors; -import com.yahoo.config.provision.AllocatedHosts; -import com.yahoo.component.Version; import com.yahoo.config.provisioning.FlavorsConfig; +import com.yahoo.io.IOUtils; import com.yahoo.path.Path; import com.yahoo.text.Utf8; import com.yahoo.vespa.curator.mock.MockCurator; @@ -30,7 +18,19 @@ import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; -import com.yahoo.io.IOUtils; +import java.io.File; +import java.io.IOException; +import java.io.Reader; +import java.util.Collections; +import java.util.Optional; +import java.util.regex.Pattern; + +import static com.yahoo.config.provision.serialization.AllocatedHostsSerializer.toJson; +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; public class ZKApplicationPackageTest { @@ -77,7 +77,7 @@ public class ZKApplicationPackageTest { assertFalse(zkApp.getFileRegistries().containsKey(new Version(0, 0, 0))); assertThat(zkApp.getFileRegistries().get(goodVersion).fileSourceHost(), is("dummyfiles")); AllocatedHosts readInfo = zkApp.getAllocatedHosts().get(); - assertThat(Utf8.toString(readInfo.toJson()), is(Utf8.toString(ALLOCATED_HOSTS.toJson()))); + assertThat(Utf8.toString(toJson(readInfo)), is(Utf8.toString(toJson(ALLOCATED_HOSTS)))); assertThat(readInfo.getHosts().iterator().next().flavor(), is(TEST_FLAVOR)); assertEquals("6.0.1", readInfo.getHosts().iterator().next().version().get().toString()); assertTrue(zkApp.getDeployment().isPresent()); @@ -90,7 +90,7 @@ public class ZKApplicationPackageTest { String metaData = "{\"deploy\":{\"user\":\"foo\",\"from\":\"bar\",\"timestamp\":1},\"application\":{\"name\":\"foo\",\"checksum\":\"abc\",\"generation\":4,\"previousActiveGeneration\":3}}"; zk.putData("/0", ConfigCurator.META_ZK_PATH, metaData); zk.putData("/0/" + ZKApplicationPackage.fileRegistryNode + "/3.0.0", "dummyfiles"); - zk.putData("/0/" + ZKApplicationPackage.allocatedHostsNode, ALLOCATED_HOSTS.toJson()); + zk.putData("/0/" + ZKApplicationPackage.allocatedHostsNode, toJson(ALLOCATED_HOSTS)); } private static class MockNodeFlavors extends NodeFlavors{ diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java index 45fb1e050a7..cf6531c0748 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java @@ -9,11 +9,11 @@ import com.yahoo.config.provision.ClusterMembership; import com.yahoo.config.provision.DockerImage; import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.InstanceName; -import com.yahoo.config.provision.NetworkPortsSerializer; import com.yahoo.config.provision.NodeFlavors; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.TenantName; +import com.yahoo.config.provision.serialization.NetworkPortsSerializer; import com.yahoo.slime.ArrayTraverser; import com.yahoo.slime.Cursor; import com.yahoo.slime.Inspector; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesResponse.java index a591217f5d5..be34e11f585 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesResponse.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesResponse.java @@ -5,8 +5,8 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterMembership; import com.yahoo.config.provision.DockerImage; import com.yahoo.config.provision.Flavor; -import com.yahoo.config.provision.NetworkPortsSerializer; import com.yahoo.config.provision.NodeType; +import com.yahoo.config.provision.serialization.NetworkPortsSerializer; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.slime.Cursor; diff --git a/vespalog/src/logctl/logctl.cpp b/vespalog/src/logctl/logctl.cpp index 743847a1237..950b7929a50 100644 --- a/vespalog/src/logctl/logctl.cpp +++ b/vespalog/src/logctl/logctl.cpp @@ -35,13 +35,13 @@ usage(const char *name) " <level>=<on|off>[,<level>=<on|off>]...\n" "<level> is one of:\n" " all, fatal, error, warning, info, event, config, debug or spam\n\n" - "component-specification specicies which sub-components of the\n" + "component-specification specifies which sub-components of the\n" "service should be controlled. If it is empty, all components\n" "are controlled:\n" " x. : Matches only component x\n" " x : Matches component x and all its sub-components\n\n" - "Example: %s topleveldispatch:log all=on,spam=off,debug=off : For service\n" - "topleveldispatch, set log and all sub-components of log to enable all\n" + "Example: %s qrserver:log all=on,spam=off,debug=off : For service\n" + "qrserver, set log and all sub-components of log to enable all\n" "except spam and debug.\n\n", name, name, name); } |