diff options
338 files changed, 4363 insertions, 3933 deletions
diff --git a/.gitignore b/.gitignore index 0b2fba8c9ed..c77fe07eee7 100644 --- a/.gitignore +++ b/.gitignore @@ -34,12 +34,17 @@ CTestTestfile.cmake cmake_install.cmake Makefile Testing +/.mvn/ /.ninja_deps /.ninja_log /build.ninja /rules.ninja *_test_app /hadoop/dependency-reduced-pom.xml +/mvnw +/mvnw.cmd +/mvnwDebug +/mvnwDebug.cmd /vespa-hadoop/dependency-reduced-pom.xml .preprocessed/ .DS_Store diff --git a/application/pom.xml b/application/pom.xml index f990988c77e..61cea1a1826 100644 --- a/application/pom.xml +++ b/application/pom.xml @@ -180,6 +180,10 @@ <artifactId>maven-surefire-plugin</artifactId> <configuration> <forkCount>2</forkCount> + <!-- Illegal reflective access by LogFileHandler via com.yahoo.io.NativeIO --> + <argLine> + --add-opens=java.base/java.io=ALL-UNNAMED + </argLine> </configuration> </plugin> <plugin> diff --git a/athenz-identity-provider-service/pom.xml b/athenz-identity-provider-service/pom.xml index 65777957d86..08b436436e1 100644 --- a/athenz-identity-provider-service/pom.xml +++ b/athenz-identity-provider-service/pom.xml @@ -147,6 +147,16 @@ </compilerArgs> </configuration> </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-surefire-plugin</artifactId> + <configuration> + <!-- Illegal reflective access by guice. TODO: try to remove for guice >3.0 --> + <argLine> + --add-opens=java.base/java.lang=ALL-UNNAMED + </argLine> + </configuration> + </plugin> </plugins> </build> diff --git a/config-model-api/src/main/java/com/yahoo/config/application/api/xml/DeploymentSpecXmlReader.java b/config-model-api/src/main/java/com/yahoo/config/application/api/xml/DeploymentSpecXmlReader.java index 4ca9ee1dc2f..6f77dce8fc5 100644 --- a/config-model-api/src/main/java/com/yahoo/config/application/api/xml/DeploymentSpecXmlReader.java +++ b/config-model-api/src/main/java/com/yahoo/config/application/api/xml/DeploymentSpecXmlReader.java @@ -281,6 +281,7 @@ public class DeploymentSpecXmlReader { List<Endpoint.Target> targets = new ArrayList<>(); if (level == Endpoint.Level.application) { String region = requireStringAttribute("region", endpointElement); + int weightSum = 0; for (var instanceElement : XML.getChildren(endpointElement, "instance")) { String instanceName = instanceElement.getTextContent(); String weightFromAttribute = requireStringAttribute("weight", instanceElement); @@ -291,10 +292,12 @@ public class DeploymentSpecXmlReader { } catch (NumberFormatException e) { throw new IllegalArgumentException(msgPrefix + "invalid weight value '" + weightFromAttribute + "'"); } + weightSum += weight; targets.add(new Endpoint.Target(RegionName.from(region), InstanceName.from(instanceName), weight)); } + if (weightSum == 0) illegal(msgPrefix + "sum of all weights must be positive, got " + weightSum); } else { if (stringAttribute("region", endpointElement).isPresent()) illegal(msgPrefix + "invalid 'region' attribute"); for (var regionElement : XML.getChildren(endpointElement, "region")) { diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ApplicationClusterEndpoint.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ApplicationClusterEndpoint.java new file mode 100644 index 00000000000..a91f95d71b1 --- /dev/null +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ApplicationClusterEndpoint.java @@ -0,0 +1,172 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.config.model.api; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ClusterSpec; + +import java.util.List; +import java.util.Objects; +import java.util.Optional; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +/** + * Represents one endpoint for an application cluster + * + * @author mortent + */ +public class ApplicationClusterEndpoint { + public enum Scope {application, global, zone} + + public enum RoutingMethod {shared, sharedLayer4} + + private final DnsName dnsName; + private final Scope scope; + private final RoutingMethod routingMethod; + private final int weight; + private final List<String> hostNames; + + public ApplicationClusterEndpoint(DnsName dnsName, Scope scope, RoutingMethod routingMethod, int weight, List<String> hostNames) { + this.dnsName = dnsName; + this.scope = scope; + this.routingMethod = routingMethod; + this.weight = weight; + this.hostNames = List.copyOf(hostNames); + } + + public DnsName dnsName() { + return dnsName; + } + + public Scope scope() { + return scope; + } + + public RoutingMethod routingMethod() { + return routingMethod; + } + + public int weight() { + return weight; + } + + public List<String> hostNames() { + return hostNames; + } + + public static Builder builder() { + return new Builder(); + } + + public static class Builder { + + private DnsName dnsName; + private Scope scope; + private RoutingMethod routingMethod; + private int weigth = 0; + private List<String> hosts; + + public Builder dnsName(DnsName name) { + this.dnsName = name; + return this; + } + + public Builder zoneScope() { + this.scope = Scope.zone; + return this; + } + + public Builder scope(Scope scope) { + this.scope = scope; + return this; + } + + public Builder sharedRouting() { + this.routingMethod = RoutingMethod.shared; + return this; + } + + public Builder sharedL4Routing() { + this.routingMethod = RoutingMethod.sharedLayer4; + return this; + } + + public Builder weight(int weigth) { + this.weigth = weigth; + return this; + } + + public Builder hosts(List<String> hosts) { + this.hosts = List.copyOf(hosts); + return this; + } + + public ApplicationClusterEndpoint build() { + return new ApplicationClusterEndpoint(dnsName, scope, routingMethod, weigth, hosts); + } + } + + public static class DnsName { + private static final int MAX_LABEL_LENGTH = 63; + + private final String name; + + private DnsName(String name) { + this.name = name; + } + + public String value() { + return name; + } + + // TODO: remove + public static DnsName sharedNameFrom(ClusterSpec.Id cluster, ApplicationId applicationId, String suffix) { + String name = dnsParts(cluster, applicationId) + .filter(Objects::nonNull) // remove null values that were "default" + .collect(Collectors.joining("--")); + return new DnsName(sanitize(name) + suffix); // Need to sanitize name since it is considered one label + } + + public static DnsName sharedL4NameFrom(ClusterSpec.Id cluster, ApplicationId applicationId, String suffix) { + String name = dnsParts(cluster, applicationId) + .filter(Objects::nonNull) // remove null values that were "default" + .map(DnsName::sanitize) + .collect(Collectors.joining(".")); + return new DnsName(name + suffix); + } + + public static DnsName from(String name) { + return new DnsName(name); + } + + private static Stream<String> dnsParts(ClusterSpec.Id cluster, ApplicationId applicationId) { + return Stream.of( + nullIfDefault(cluster.value()), + nullIfDefault(applicationId.instance().value()), + applicationId.application().value(), + applicationId.tenant().value() + ); + } + + /** + * Remove any invalid characters from the hostnames + */ + private static String sanitize(String id) { + return shortenIfNeeded(id.toLowerCase() + .replace('_', '-') + .replaceAll("[^a-z0-9-]*", "")); + } + + /** + * Truncate the given string at the front so its length does not exceed 63 characters. + */ + private static String shortenIfNeeded(String id) { + return id.substring(Math.max(0, id.length() - MAX_LABEL_LENGTH)); + } + + private static String nullIfDefault(String string) { + return Optional.of(string).filter(s -> !s.equals("default")).orElse(null); + } + } +} diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ApplicationClusterInfo.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ApplicationClusterInfo.java new file mode 100644 index 00000000000..2cd2e980761 --- /dev/null +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ApplicationClusterInfo.java @@ -0,0 +1,9 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.config.model.api; + +import java.util.List; + +public interface ApplicationClusterInfo { + List<ApplicationClusterEndpoint> endpoints(); +} diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ContainerEndpoint.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ContainerEndpoint.java index 2b2b5e2c404..a114f9d40ef 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ContainerEndpoint.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ContainerEndpoint.java @@ -14,10 +14,12 @@ import java.util.Objects; public class ContainerEndpoint { private final String clusterId; + private final ApplicationClusterEndpoint.Scope scope; private final List<String> names; - public ContainerEndpoint(String clusterId, List<String> names) { + public ContainerEndpoint(String clusterId, ApplicationClusterEndpoint.Scope scope, List<String> names) { this.clusterId = Objects.requireNonNull(clusterId); + this.scope = Objects.requireNonNull(scope); this.names = List.copyOf(Objects.requireNonNull(names)); } @@ -29,23 +31,28 @@ public class ContainerEndpoint { return names; } + public ApplicationClusterEndpoint.Scope scope() { + return scope; + } + @Override public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; ContainerEndpoint that = (ContainerEndpoint) o; return Objects.equals(clusterId, that.clusterId) && + Objects.equals(scope, that.scope) && Objects.equals(names, that.names); } @Override public int hashCode() { - return Objects.hash(clusterId, names); + return Objects.hash(clusterId, names, scope); } @Override public String toString() { - return String.format("container endpoint %s -> %s", clusterId, names); + return String.format("container endpoint %s -> %s [scope=%s]", clusterId, names, scope); } } diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/Model.java b/config-model-api/src/main/java/com/yahoo/config/model/api/Model.java index c1248ff556c..a7fd48bfea8 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/Model.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/Model.java @@ -79,4 +79,6 @@ public interface Model { /** Returns the set of document types in each cluster, that have an index for one of more fields. */ default Map<String, Set<String>> indexedDocumentTypesByCluster() { return Map.of(); } + /** Returns the set of container clusters */ + default Set<ApplicationClusterInfo> applicationClusterInfo() { return Set.of(); } } diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java index ff612ffc2b0..c5781c2805d 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java @@ -148,6 +148,8 @@ public interface ModelContext { default List<X509Certificate> operatorCertificates() { return List.of(); } default List<String> tlsCiphersOverride() { return List.of(); } + + default List<String> zoneDnsSuffixes() { return List.of(); } } @Retention(RetentionPolicy.RUNTIME) diff --git a/config-model-api/src/test/java/com/yahoo/config/application/api/DeploymentSpecTest.java b/config-model-api/src/test/java/com/yahoo/config/application/api/DeploymentSpecTest.java index 8f9c9bccfcd..74e79d5e8cf 100644 --- a/config-model-api/src/test/java/com/yahoo/config/application/api/DeploymentSpecTest.java +++ b/config-model-api/src/test/java/com/yahoo/config/application/api/DeploymentSpecTest.java @@ -1200,7 +1200,7 @@ public class DeploymentSpecTest { } @Test - public void applicationLevelEndpointRequiresAttributes() { + public void applicationLevelEndpointValidation() { String xmlForm = "<deployment>\n" + " <instance id=\"beta\">\n" + " <prod>\n" + @@ -1226,6 +1226,7 @@ public class DeploymentSpecTest { assertInvalid(String.format(xmlForm, "region='invalid'", "weight='1'", "main", ""), "Application-level endpoint 'foo': targets undeclared region 'invalid' in instance 'main'"); assertInvalid(String.format(xmlForm, "region='us-west-1'", "weight='foo'", "main", ""), "Application-level endpoint 'foo': invalid weight value 'foo'"); assertInvalid(String.format(xmlForm, "region='us-west-1'", "weight='1'", "main", "<region>us-east-3</region>"), "Application-level endpoint 'foo': invalid element 'region'"); + assertInvalid(String.format(xmlForm, "region='us-west-1'", "weight='0'", "main", ""), "Application-level endpoint 'foo': sum of all weights must be positive, got 0"); } @Test diff --git a/config-model/src/main/java/com/yahoo/config/model/CommonConfigsProducer.java b/config-model/src/main/java/com/yahoo/config/model/CommonConfigsProducer.java index fad097d1edb..094b11dcbc7 100644 --- a/config-model/src/main/java/com/yahoo/config/model/CommonConfigsProducer.java +++ b/config-model/src/main/java/com/yahoo/config/model/CommonConfigsProducer.java @@ -21,7 +21,6 @@ import com.yahoo.vespa.config.content.AllClustersBucketSpacesConfig; * This interface describes the configs that are produced by the model producer root. * * @author Ulf Lilleengen - * @since 5.1 */ public interface CommonConfigsProducer extends DocumentmanagerConfig.Producer, DocumenttypesConfig.Producer, diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java index 5183a3ca587..49c968a1d91 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java @@ -72,6 +72,7 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea private double diskBloatFactor = 0.2; private boolean distributorEnhancedMaintenanceScheduling = false; private boolean asyncApplyBucketDiff = false; + private List<String> zoneDnsSuffixes = List.of(); @Override public ModelContext.FeatureFlags featureFlags() { return this; } @Override public boolean multitenant() { return multitenant; } @@ -124,6 +125,7 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea @Override public boolean distributorEnhancedMaintenanceScheduling() { return distributorEnhancedMaintenanceScheduling; } @Override public int maxUnCommittedMemory() { return maxUnCommittedMemory; } @Override public boolean asyncApplyBucketDiff() { return asyncApplyBucketDiff; } + @Override public List<String> zoneDnsSuffixes() { return zoneDnsSuffixes; } public TestProperties maxUnCommittedMemory(int maxUnCommittedMemory) { this.maxUnCommittedMemory = maxUnCommittedMemory; @@ -320,6 +322,11 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea return this; } + public TestProperties setZoneDnsSuffixes(List<String> zoneDnsSuffixes) { + this.zoneDnsSuffixes = List.copyOf(zoneDnsSuffixes); + return this; + } + public static class Spec implements ConfigServerSpec { private final String hostName; diff --git a/config-model/src/main/java/com/yahoo/documentmodel/NewDocumentType.java b/config-model/src/main/java/com/yahoo/documentmodel/NewDocumentType.java index 08a0f8b9882..d98869e9dd3 100644 --- a/config-model/src/main/java/com/yahoo/documentmodel/NewDocumentType.java +++ b/config-model/src/main/java/com/yahoo/documentmodel/NewDocumentType.java @@ -368,9 +368,9 @@ public final class NewDocumentType extends StructuredDataType implements DataTyp @Override public String toString() { return name; } - public final String getName() { return name; } + public String getName() { return name; } - public final int getId() { return id; } + public int getId() { return id; } @Override public int hashCode() { return name.hashCode(); } diff --git a/config-model/src/main/java/com/yahoo/documentmodel/VespaDocumentType.java b/config-model/src/main/java/com/yahoo/documentmodel/VespaDocumentType.java index 4899029c4b0..b29e4704f62 100644 --- a/config-model/src/main/java/com/yahoo/documentmodel/VespaDocumentType.java +++ b/config-model/src/main/java/com/yahoo/documentmodel/VespaDocumentType.java @@ -16,7 +16,7 @@ public class VespaDocumentType { public static NewDocumentType INSTANCE = newInstance(); - public static DataTypeName NAME = new DataTypeName("document"); + public static final DataTypeName NAME = new DataTypeName("document"); private static NewDocumentType newInstance() { NewDocumentType vespa = new NewDocumentType(new NewDocumentType.Name(8, "document")); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java index b5d86f5bb38..52fd5286bcf 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java @@ -98,6 +98,8 @@ public class RankProfile implements Cloneable { private String inheritedSummaryFeatures; private Set<ReferenceNode> matchFeatures; + private String inheritedMatchFeatures; + private Set<ReferenceNode> rankFeatures; /** The properties of this - a multimap */ @@ -519,8 +521,30 @@ public class RankProfile implements Cloneable { this.inheritedSummaryFeatures = parentProfile; } + /** + * Sets the name of a profile this should inherit the match features of. + * Without setting this, this will either have the match features of the parent, + * or if match features are set in this, only have the match features in this. + * With this set the resulting match features of this will be the superset of those defined in this and + * the final (with inheritance included) match features of the given parent. + * The profile must be the profile which is directly inherited by this. + * + */ + public void setInheritedMatchFeatures(String parentProfile) { + if ( ! parentProfile.equals(inheritedName)) + throw new IllegalArgumentException("This rank profile ("+name+") can only inherit the match features of its parent, '" + + inheritedName + ", but attemtping to inherit '" + parentProfile); + this.inheritedMatchFeatures = parentProfile; + } + /** Returns a read-only view of the match features to use in this profile. This is never null */ public Set<ReferenceNode> getMatchFeatures() { + if (inheritedMatchFeatures != null && matchFeatures != null) { + Set<ReferenceNode> combined = new HashSet<>(); + combined.addAll(getInherited().getMatchFeatures()); + combined.addAll(matchFeatures); + return Collections.unmodifiableSet(combined); + } if (matchFeatures != null) return Collections.unmodifiableSet(matchFeatures); if (getInherited() != null) return getInherited().getMatchFeatures(); return Set.of(); @@ -532,7 +556,7 @@ public class RankProfile implements Cloneable { matchFeatures.add(feature); } - /** Adds the content of the given feature list to the internal list of summary features. */ + /** Adds the content of the given feature list to the internal list of match features. */ public void addMatchFeatures(FeatureList features) { for (ReferenceNode feature : features) { addMatchFeature(feature); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/SDDocumentTypeOrderer.java b/config-model/src/main/java/com/yahoo/searchdefinition/SDDocumentTypeOrderer.java index 8a6f16586d2..aa43c00f461 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/SDDocumentTypeOrderer.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/SDDocumentTypeOrderer.java @@ -4,6 +4,7 @@ package com.yahoo.searchdefinition; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.document.*; import com.yahoo.document.annotation.AnnotationReferenceDataType; +import com.yahoo.documentmodel.NewDocumentType; import com.yahoo.searchdefinition.document.SDDocumentType; import com.yahoo.searchdefinition.document.TemporarySDDocumentType; @@ -15,10 +16,10 @@ import java.util.logging.Level; */ public class SDDocumentTypeOrderer { - private Map<DataTypeName, SDDocumentType> createdSDTypes = new LinkedHashMap<>(); - private Set<Integer> seenTypes = new LinkedHashSet<>(); + private final Map<DataTypeName, SDDocumentType> createdSDTypes = new LinkedHashMap<>(); + private final Set<Integer> seenTypes = new LinkedHashSet<>(); List<SDDocumentType> processingOrder = new LinkedList<>(); - private DeployLogger deployLogger; + private final DeployLogger deployLogger; public SDDocumentTypeOrderer(List<SDDocumentType> sdTypes, DeployLogger deployLogger) { this.deployLogger = deployLogger; @@ -36,28 +37,12 @@ public class SDDocumentTypeOrderer { public void process() { for (SDDocumentType type : createdSDTypes.values()) { - process(type); + process(type, type); } } - private void process(SDDocumentType type) { - List<DataTypeName> toReplace = new ArrayList<>(); - for (SDDocumentType sdoc : type.getInheritedTypes()) { - if (sdoc instanceof TemporarySDDocumentType) { - toReplace.add(sdoc.getDocumentName()); - } - } - for (DataTypeName name : toReplace) { - SDDocumentType inherited = createdSDTypes.get(name); - if (inherited == null) { - throw new IllegalStateException("Document type '" + name + "' not found."); - } - process(inherited); - type.inherit(inherited); - } - visit(type); - } - private void visit(SDDocumentType docOrStruct) { + private void process(SDDocumentType docOrStruct, SDDocumentType owningDocument) { + resolveAndProcessInheritedTemporaryTypes(docOrStruct, owningDocument); int id; if (docOrStruct.isStruct()) { id = new StructDataType(docOrStruct.getName()).getId(); @@ -71,16 +56,38 @@ public class SDDocumentTypeOrderer { seenTypes.add((new StructDataType(docOrStruct.getName()).getId())); } - for (Field field : docOrStruct.fieldSet()) { if (!seenTypes.contains(field.getDataType().getId())) { //we haven't seen this before, do it - visit(field.getDataType()); + visit(field.getDataType(), owningDocument); } } processingOrder.add(docOrStruct); } + private void resolveAndProcessInheritedTemporaryTypes(SDDocumentType type, SDDocumentType owningDocument) { + List<DataTypeName> toReplace = new ArrayList<>(); + for (SDDocumentType sdoc : type.getInheritedTypes()) { + if (sdoc instanceof TemporarySDDocumentType) { + toReplace.add(sdoc.getDocumentName()); + } + } + for (DataTypeName name : toReplace) { + SDDocumentType inherited; + if (type.isStruct()) { + inherited = owningDocument.allTypes().get(new NewDocumentType.Name(name.getName())); + if (inherited == null) throw new IllegalStateException("Struct '" + name + "' not found in " + owningDocument); + process(inherited, owningDocument); + } + else { + inherited = createdSDTypes.get(name); + if (inherited == null) throw new IllegalStateException("Document type '" + name + "' not found"); + process(inherited, inherited); + } + type.inherit(inherited); + } + } + private SDDocumentType find(String name) { SDDocumentType sdDocType = createdSDTypes.get(new DataTypeName(name)); if (sdDocType != null) { @@ -95,27 +102,28 @@ public class SDDocumentTypeOrderer { } return null; } - private void visit(DataType type) { + + private void visit(DataType type, SDDocumentType owningDocument) { if (type instanceof StructuredDataType) { StructuredDataType structType = (StructuredDataType) type; SDDocumentType sdDocType = find(structType.getName()); if (sdDocType == null) { - throw new IllegalArgumentException("Could not find struct '" + type.getName() + "'."); + throw new IllegalArgumentException("Could not find struct '" + type.getName() + "'"); } - visit(sdDocType); + process(sdDocType, owningDocument); return; } if (type instanceof MapDataType) { MapDataType mType = (MapDataType) type; - visit(mType.getValueType()); - visit(mType.getKeyType()); + visit(mType.getValueType(), owningDocument); + visit(mType.getKeyType(), owningDocument); } else if (type instanceof WeightedSetDataType) { WeightedSetDataType wType = (WeightedSetDataType) type; - visit(wType.getNestedType()); + visit(wType.getNestedType(), owningDocument); } else if (type instanceof CollectionDataType) { CollectionDataType cType = (CollectionDataType) type; - visit(cType.getNestedType()); + visit(cType.getNestedType(), owningDocument); } else if (type instanceof AnnotationReferenceDataType) { //do nothing } else if (type instanceof PrimitiveDataType) { @@ -128,4 +136,5 @@ public class SDDocumentTypeOrderer { deployLogger.logApplicationPackage(Level.WARNING, "Unknown type : " + type); } } + } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/Schema.java b/config-model/src/main/java/com/yahoo/searchdefinition/Schema.java index 512d908ce5b..c7a7ecd1d08 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/Schema.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/Schema.java @@ -514,21 +514,14 @@ public class Schema implements ImmutableSchema { return Collections.unmodifiableList(allIndices); } - /** - * Adds an explicitly defined summary to this search definition - * - * @param summary The summary to add. - */ + /** Adds an explicitly defined summary to this search definition */ public void addSummary(DocumentSummary summary) { summaries.put(summary.getName(), summary); } /** - * <p>Returns a summary class defined by this search definition, or null if no summary with this name is defined. - * The default summary, named "default" is always present.</p> - * - * @param name the name of the summary to get. - * @return Summary found. + * Returns a summary class defined by this search definition, or null if no summary with this name is defined. + * The default summary, named "default" is always present. */ public DocumentSummary getSummary(String name) { var summary = summaries.get(name); @@ -540,9 +533,6 @@ public class Schema implements ImmutableSchema { /** * Returns the first explicit instance found of a summary field with this name, or null if not present (implicitly * or explicitly) in any summary class. - * - * @param name The name of the summaryfield to get. - * @return SummaryField to return. */ public SummaryField getSummaryField(String name) { for (DocumentSummary summary : summaries.values()) { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/SummaryClass.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/SummaryClass.java index 68966d39d7d..6c233aacf30 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/SummaryClass.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/SummaryClass.java @@ -31,9 +31,9 @@ public class SummaryClass extends Derived { private final boolean omitSummaryFeatures; /** The summary fields of this indexed by name */ - private Map<String,SummaryClassField> fields = new java.util.LinkedHashMap<>(); + private final Map<String,SummaryClassField> fields = new java.util.LinkedHashMap<>(); - private DeployLogger deployLogger; + private final DeployLogger deployLogger; private final Random random = new Random(7); @@ -78,9 +78,9 @@ public class SummaryClass extends Derived { private void addField(String name, DataType type, SummaryTransform transform) { if (fields.containsKey(name)) { SummaryClassField sf = fields.get(name); - if (!SummaryClassField.convertDataType(type, transform, rawAsBase64).equals(sf.getType())) { - deployLogger.logApplicationPackage(Level.WARNING, "Conflicting definition of field " + name + ". " + - "Declared as type " + sf.getType() + " and " + type); + if ( SummaryClassField.convertDataType(type, transform, rawAsBase64) != sf.getType()) { + deployLogger.logApplicationPackage(Level.WARNING, "Conflicting definition of field " + name + + ". " + "Declared as type " + sf.getType() + " and " + type); } } else { fields.put(name, new SummaryClassField(name, type, transform, rawAsBase64)); @@ -106,6 +106,7 @@ public class SummaryClass extends Derived { public int getFieldCount() { return fields.size(); } + @Override public int hashCode() { int number = 1; int hash = getName().hashCode(); @@ -143,6 +144,7 @@ public class SummaryClass extends Derived { @Override protected String getDerivedName() { return "summary"; } + @Override public String toString() { return "summary class " + getName(); } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/SDDocumentType.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/SDDocumentType.java index 6424db1c2dd..60b06d3655c 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/document/SDDocumentType.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/SDDocumentType.java @@ -1,7 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.searchdefinition.document; -import com.yahoo.document.CompressionConfig; import com.yahoo.document.DataType; import com.yahoo.document.DataTypeName; import com.yahoo.document.DocumentType; @@ -127,6 +126,14 @@ public class SDDocumentType implements Cloneable, Serializable { return list; } + public Map<NewDocumentType.Name, SDDocumentType> allTypes() { + Map<NewDocumentType.Name, SDDocumentType> map = new LinkedHashMap<>(); + for (SDDocumentType inherited : inheritedTypes.values()) + map.putAll(inherited.allTypes()); + map.putAll(ownedTypes); + return map; + } + /** * Creates a new document type. * The document type id will be generated as a hash from the document type name. @@ -145,13 +152,11 @@ public class SDDocumentType implements Cloneable, Serializable { * Creates a new document type. * The document type id will be generated as a hash from the document type name. * - * @param name The name of the new document type + * @param name the name of the new document type * @param schema check for type ID collisions in this search definition */ - @SuppressWarnings("deprecation") public SDDocumentType(String name, Schema schema) { docType = new DocumentType(name); - docType.contentStruct().setCompressionConfig(new CompressionConfig()); validateId(schema); inherit(VESPA_DOCUMENT); } @@ -161,7 +166,7 @@ public class SDDocumentType implements Cloneable, Serializable { public SDDocumentType setStruct(DataType structType) { if (structType != null) { this.structType = structType; - inheritedTypes.clear(); + inheritedTypes.remove(VESPA_DOCUMENT.getDocumentName()); } else { if (docType.contentStruct() != null) { this.structType = docType.contentStruct(); @@ -200,7 +205,7 @@ public class SDDocumentType implements Cloneable, Serializable { if (schema.getDocument(getName()) == null) return; SDDocumentType doc = schema.getDocument(); throw new IllegalArgumentException("Failed creating document type '" + getName() + "', " + - "document type '" + doc.getName() + "' already uses ID '" + doc.getName() + "'"); + "document type '" + doc.getName() + "' already uses ID '" + doc.getName() + "'"); } public void setFieldId(SDField field, int id) { @@ -293,6 +298,8 @@ public class SDDocumentType implements Cloneable, Serializable { return fieldSet().iterator(); } + /** Returns the number of fields in this only, not including inherited fields */ + // TODO: Remove public int getFieldCount() { return docType.getFieldCount(); } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImplicitSummaries.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImplicitSummaries.java index 6b09234c469..9eb8b921e81 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImplicitSummaries.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImplicitSummaries.java @@ -29,17 +29,16 @@ public class ImplicitSummaries extends Processor { @Override public void process(boolean validate, boolean documentsOnly) { - DocumentSummary defaultSummary = schema.getSummary("default"); + DocumentSummary defaultSummary = schema.getSummariesInThis().get("default"); if (defaultSummary == null) { defaultSummary = new DocumentSummary("default", schema); - defaultSummary.setFromDisk(true); + defaultSummary.setFromDisk(true); // As we add documentid to this schema.addSummary(defaultSummary); } for (SDField field : schema.allConcreteFields()) { collectSummaries(field, schema, validate); } - for (DocumentSummary documentSummary : schema.getSummaries().values()) { documentSummary.purgeImplicits(); } @@ -50,7 +49,7 @@ public class ImplicitSummaries extends Processor { } private void collectSummaries(SDField field , Schema schema, boolean validate) { - SummaryField addedSummaryField=null; + SummaryField addedSummaryField = null; // Implicit String fieldName = field.getName(); @@ -65,7 +64,7 @@ public class ImplicitSummaries extends Processor { } if (fieldSummaryField != null) { for (String dest : fieldSummaryField.getDestinations()) { - DocumentSummary summary = schema.getSummary(dest); + DocumentSummary summary = schema.getSummariesInThis().get(dest); if (summary != null) { summary.add(fieldSummaryField); } @@ -114,7 +113,7 @@ public class ImplicitSummaries extends Processor { } private DocumentSummary getOrCreateAttributePrefetchSummary(Schema schema) { - DocumentSummary summary = schema.getSummary("attributeprefetch"); + DocumentSummary summary = schema.getSummariesInThis().get("attributeprefetch"); if (summary == null) { summary = new DocumentSummary("attributeprefetch", schema); schema.addSummary(summary); @@ -165,13 +164,13 @@ public class ImplicitSummaries extends Processor { throw newProcessException(schema, summaryField, "Source field '" + fieldName + "' does not exist."); } if (! sourceField.doesSummarying() && - ! summaryField.getTransform().equals(SummaryTransform.ATTRIBUTE) && - ! summaryField.getTransform().equals(SummaryTransform.GEOPOS)) + summaryField.getTransform() != SummaryTransform.ATTRIBUTE && + summaryField.getTransform() != SummaryTransform.GEOPOS) { // Summary transform attribute may indicate that the ilscript was rewritten to remove summary // by another search that uses this same field in inheritance. deployLogger.logApplicationPackage(Level.WARNING, "Ignoring " + summaryField + ": " + sourceField + - " is not creating a summary value in its indexing statement"); + " is not creating a summary value in its indexing statement"); return false; } @@ -210,7 +209,7 @@ public class ImplicitSummaries extends Processor { } private void addToDestination(String destinationName, SummaryField summaryField, Schema schema) { - DocumentSummary destination = schema.getSummary(destinationName); + DocumentSummary destination = schema.getSummariesInThis().get(destinationName); if (destination == null) { destination = new DocumentSummary(destinationName, schema); schema.addSummary(destination); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/MakeDefaultSummaryTheSuperSet.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/MakeDefaultSummaryTheSuperSet.java index a82c8d0c6be..49a56bafe2a 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/MakeDefaultSummaryTheSuperSet.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/MakeDefaultSummaryTheSuperSet.java @@ -10,20 +10,20 @@ import com.yahoo.vespa.documentmodel.SummaryTransform; import com.yahoo.vespa.model.container.search.QueryProfiles; /** - * <p>All summary fields which are not attributes + * All summary fields which are not attributes * must currently be present in the default summary class, * since the default summary class also defines the docsum.dat format. * This processor adds any missing summaries to the default summary. * When that is decoupled from the actual summaries returned, this * processor can be removed. Note: the StreamingSummary also takes advantage of - * the fact that default is the superset.</p> + * the fact that default is the superset. * - * <p>All other summary logic should work unchanged without this processing step + * All other summary logic should work unchanged without this processing step * except that IndexStructureValidator.validateSummaryFields must be changed to * consider all summaries, not just the default, i.e change to - * if (search.getSummaryField(expr.getFieldName()) == null)</p> + * if (search.getSummaryField(expr.getFieldName()) == null) * - * <p>This must be done after other summary processors.</p> + * This must be done after other summary processors. * * @author bratseth */ @@ -35,7 +35,7 @@ public class MakeDefaultSummaryTheSuperSet extends Processor { @Override public void process(boolean validate, boolean documentsOnly) { - DocumentSummary defaultSummary= schema.getSummary("default"); + DocumentSummary defaultSummary= schema.getSummariesInThis().get("default"); for (SummaryField summaryField : schema.getUniqueNamedSummaryFields().values() ) { if (defaultSummary.getSummaryField(summaryField.getName()) != null) continue; if (summaryField.getTransform() == SummaryTransform.ATTRIBUTE) continue; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/PredicateProcessor.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/PredicateProcessor.java index 73b75d6e23a..493bf9b5251 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/PredicateProcessor.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/PredicateProcessor.java @@ -68,7 +68,7 @@ public class PredicateProcessor extends Processor { attribute.setDensePostingListThreshold(threshold); addPredicateOptimizationIlScript(field, booleanDefinition); } - DocumentSummary summary = schema.getSummary("attributeprefetch"); + DocumentSummary summary = schema.getSummariesInThis().get("attributeprefetch"); if (summary != null) { summary.remove(attribute.getName()); } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ReferenceFieldsProcessor.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ReferenceFieldsProcessor.java index 19bfb41289d..57833fecc7a 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ReferenceFieldsProcessor.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ReferenceFieldsProcessor.java @@ -51,7 +51,7 @@ public class ReferenceFieldsProcessor extends Processor { } private void removeFromAttributePrefetchSummaryClass(SDField field) { - DocumentSummary summary = schema.getSummary("attributeprefetch"); + DocumentSummary summary = schema.getSummariesInThis().get("attributeprefetch"); if (summary != null) { summary.remove(field.getName()); } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryConsistency.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryConsistency.java index 8b86674e4d0..e2e0cf94bb8 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryConsistency.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryConsistency.java @@ -43,7 +43,7 @@ public class SummaryConsistency extends Processor { private void assertConsistency(SummaryField summaryField, Schema schema, boolean validate) { // Compare to default: - SummaryField existingDefault = schema.getSummary("default").getSummaryField(summaryField.getName()); + SummaryField existingDefault = schema.getSummariesInThis().get("default").getSummaryField(summaryField.getName()); if (existingDefault != null) { if (validate) assertConsistentTypes(existingDefault, summaryField); diff --git a/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentManager.java b/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentManager.java index 64b528ad858..8605389e1c2 100644 --- a/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentManager.java +++ b/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentManager.java @@ -117,13 +117,6 @@ public class DocumentManager { Datatype.Structtype.Builder structBuilder = new Datatype.Structtype.Builder(); builder.structtype(structBuilder); structBuilder.name(structType.getName()); - if (structType.getCompressionConfig().type.getCode() != 0) { - structBuilder. - compresstype(Datatype.Structtype.Compresstype.Enum.valueOf(structType.getCompressionConfig().type.toString())). - compresslevel(structType.getCompressionConfig().compressionLevel). - compressthreshold((int)structType.getCompressionConfig().threshold). - compressminsize((int)structType.getCompressionConfig().minsize); - } for (com.yahoo.document.Field field : structType.getFieldsThisTypeOnly()) { Datatype.Structtype.Field.Builder fieldBuilder = new Datatype.Structtype.Field.Builder(); structBuilder.field(fieldBuilder); diff --git a/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentTypes.java b/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentTypes.java index a89ef74a749..3b0b63f277e 100644 --- a/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentTypes.java +++ b/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentTypes.java @@ -136,13 +136,6 @@ public class DocumentTypes { DocumenttypesConfig.Documenttype.Datatype.Sstruct.Builder structBuilder = new DocumenttypesConfig.Documenttype.Datatype.Sstruct.Builder(); dataTypeBuilder.sstruct(structBuilder); structBuilder.name(type.getName()); - if (type.getCompressionConfig().type.getCode() != 0) { - structBuilder.compression(new DocumenttypesConfig.Documenttype.Datatype.Sstruct.Compression.Builder(). - type(DocumenttypesConfig.Documenttype.Datatype.Sstruct.Compression.Type.Enum.valueOf(type.getCompressionConfig().type.toString())). - level(type.getCompressionConfig().compressionLevel). - threshold((int) type.getCompressionConfig().threshold). - minsize((int) type.getCompressionConfig().minsize)); - } for (com.yahoo.document.Field field : type.getFields()) { DocumenttypesConfig.Documenttype.Datatype.Sstruct.Field.Builder builder = new DocumenttypesConfig.Documenttype.Datatype.Sstruct.Field.Builder(); diff --git a/config-model/src/main/java/com/yahoo/vespa/documentmodel/DocumentSummary.java b/config-model/src/main/java/com/yahoo/vespa/documentmodel/DocumentSummary.java index 3dda498b0be..59b4e1a2f9b 100644 --- a/config-model/src/main/java/com/yahoo/vespa/documentmodel/DocumentSummary.java +++ b/config-model/src/main/java/com/yahoo/vespa/documentmodel/DocumentSummary.java @@ -88,7 +88,7 @@ public class DocumentSummary extends FieldView { String sourceName = j.next().getName(); if (sourceName.equals(summaryField.getName())) continue; SummaryField sourceField=getSummaryField(sourceName); - if (sourceField==null) continue; + if (sourceField == null) continue; if (!sourceField.isImplicit()) continue; falseImplicits.add(sourceField); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java b/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java index d6c66a635d4..f584b4cd207 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java @@ -18,6 +18,7 @@ import com.yahoo.config.model.ApplicationConfigProducerRoot; import com.yahoo.config.model.ConfigModelRegistry; import com.yahoo.config.model.ConfigModelRepo; import com.yahoo.config.model.NullConfigModelRegistry; +import com.yahoo.config.model.api.ApplicationClusterInfo; import com.yahoo.config.model.api.HostInfo; import com.yahoo.config.model.api.Model; import com.yahoo.config.model.api.Provisioned; @@ -696,4 +697,8 @@ public final class VespaModel extends AbstractConfigProducerRoot implements Seri .collect(Collectors.toSet()); } + @Override + public Set<ApplicationClusterInfo> applicationClusterInfo() { + return Set.copyOf(getContainerClusters().values()); + } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java index c3897f49c44..a5dc26e19e3 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java @@ -7,10 +7,14 @@ import com.yahoo.component.ComponentId; import com.yahoo.component.ComponentSpecification; import com.yahoo.config.FileReference; import com.yahoo.config.application.api.ComponentInfo; +import com.yahoo.config.model.api.ApplicationClusterEndpoint; +import com.yahoo.config.model.api.ApplicationClusterInfo; +import com.yahoo.config.model.api.ContainerEndpoint; import com.yahoo.config.model.api.Model; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.producer.AbstractConfigProducer; import com.yahoo.config.provision.AllocatedHosts; +import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.HostSpec; import com.yahoo.container.bundle.BundleInstantiationSpecification; import com.yahoo.container.di.config.ApplicationBundlesConfig; @@ -26,6 +30,7 @@ import com.yahoo.vespa.config.search.RankProfilesConfig; import com.yahoo.vespa.config.search.core.OnnxModelsConfig; import com.yahoo.vespa.config.search.core.RankingConstantsConfig; import com.yahoo.vespa.config.search.core.RankingExpressionsConfig; +import com.yahoo.vespa.model.AbstractService; import com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyContainer; import com.yahoo.vespa.model.container.component.BindingPattern; import com.yahoo.vespa.model.container.component.Component; @@ -39,6 +44,7 @@ import com.yahoo.vespa.model.utils.FileSender; import java.util.ArrayList; import java.util.Collection; import java.util.LinkedHashSet; +import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; @@ -60,7 +66,8 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat ServletPathsConfig.Producer, ContainerMbusConfig.Producer, MetricsProxyApiConfig.Producer, - ZookeeperServerConfig.Producer { + ZookeeperServerConfig.Producer, + ApplicationClusterInfo { public static final String METRICS_V2_HANDLER_CLASS = MetricsV2Handler.class.getName(); public static final BindingPattern METRICS_V2_HANDLER_BINDING_1 = SystemBindingPattern.fromHttpPath(MetricsV2Handler.V2_PATH); @@ -88,6 +95,8 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat private Integer memoryPercentage = null; + private List<ApplicationClusterEndpoint> endpointList = List.of(); + public ApplicationContainerCluster(AbstractConfigProducer<?> parent, String configSubId, String clusterId, DeployState deployState) { super(parent, configSubId, clusterId, deployState, true); this.tlsClientAuthority = deployState.tlsClientAuthority(); @@ -115,6 +124,7 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat protected void doPrepare(DeployState deployState) { addAndSendApplicationBundles(deployState); sendUserConfiguredFiles(deployState); + createEndpointList(deployState); } private void addAndSendApplicationBundles(DeployState deployState) { @@ -184,6 +194,60 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat */ public Optional<Integer> getMemoryPercentage() { return Optional.ofNullable(memoryPercentage); } + /* + Create list of endpoints, these will be consumed later by the LBservicesProducer + */ + private void createEndpointList(DeployState deployState) { + if(!deployState.isHosted()) return; + if(deployState.getProperties().applicationId().instance().isTester()) return; + List<ApplicationClusterEndpoint> endpoints = new ArrayList<>(); + // Add zone local endpoints using zone dns suffixes, tenant, application and cluster id. + // For now support both L7 and L4 routing + + List<String> hosts = getContainers().stream() + .map(AbstractService::getHostName) + .collect(Collectors.toList()); + for(String suffix : deployState.getProperties().zoneDnsSuffixes()) { + // L4 + ApplicationClusterEndpoint.DnsName l4Name = ApplicationClusterEndpoint.DnsName.sharedL4NameFrom( + ClusterSpec.Id.from(getName()), + deployState.getProperties().applicationId(), + suffix); + endpoints.add(ApplicationClusterEndpoint.builder() + .zoneScope() + .sharedL4Routing() + .dnsName(l4Name) + .hosts(hosts) + .build()); + + // L7 + ApplicationClusterEndpoint.DnsName l7Name = ApplicationClusterEndpoint.DnsName.sharedNameFrom( + ClusterSpec.Id.from(getName()), + deployState.getProperties().applicationId(), + suffix); + endpoints.add(ApplicationClusterEndpoint.builder() + .zoneScope() + .sharedRouting() + .dnsName(l7Name) + .hosts(hosts) + .build()); + } + + // Then get all endpoints provided by controller. Can be created with L4 routing only + Set<ContainerEndpoint> endpointsFromController = deployState.getEndpoints(); + endpointsFromController.stream() + .filter(ce -> ce.clusterId().equals(getName())) + .forEach(ce -> ce.names().forEach( + name -> endpoints.add(ApplicationClusterEndpoint.builder() + .scope(ce.scope()) + .sharedL4Routing() + .dnsName(ApplicationClusterEndpoint.DnsName.from(name)) + .hosts(hosts) + .build()) + )); + endpointList = List.copyOf(endpoints); + } + @Override public void getConfig(ApplicationBundlesConfig.Builder builder) { applicationBundles.stream().map(FileReference::value) @@ -293,6 +357,11 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat null)))); } + @Override + public List<ApplicationClusterEndpoint> endpoints() { + return endpointList; + } + public static class MbusParams { // the amount of the maxpendingbytes to process concurrently, typically 0.2 (20%) final Double maxConcurrentFactor; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java index 39d4d7ec6c8..527897a3266 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java @@ -8,8 +8,10 @@ import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.application.api.DeploymentInstanceSpec; import com.yahoo.config.application.api.DeploymentSpec; +import com.yahoo.config.application.api.Endpoint; import com.yahoo.config.model.ConfigModelContext; import com.yahoo.config.model.ConfigModelContext.ApplicationType; +import com.yahoo.config.model.api.ApplicationClusterEndpoint; import com.yahoo.config.model.api.ConfigServerSpec; import com.yahoo.config.model.api.ContainerEndpoint; import com.yahoo.config.model.api.EndpointCertificateSecrets; @@ -281,6 +283,8 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { private void addCloudSecretStore(ApplicationContainerCluster cluster, Element secretStoreElement, DeployState deployState) { if ( ! deployState.isHosted()) return; + if ( ! cluster.getZone().system().isPublic()) + throw new RuntimeException("cloud secret store is not supported in non-public system, please see documentation"); CloudSecretStore cloudSecretStore = new CloudSecretStore(); Map<String, TenantSecretStore> secretStoresByName = deployState.getProperties().tenantSecretStores() .stream() @@ -338,9 +342,11 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { private void setRotations(Container container, Set<ContainerEndpoint> endpoints, String containerClusterName) { var rotationsProperty = endpoints.stream() - .filter(endpoint -> endpoint.clusterId().equals(containerClusterName)) - .flatMap(endpoint -> endpoint.names().stream()) - .collect(Collectors.toUnmodifiableSet()); + .filter(endpoint -> endpoint.clusterId().equals(containerClusterName)) + // Only consider global endpoints. + .filter(endpoint -> endpoint.scope() == ApplicationClusterEndpoint.Scope.global) + .flatMap(endpoint -> endpoint.names().stream()) + .collect(Collectors.toUnmodifiableSet()); // Build the comma delimited list of endpoints this container should be known as. // Confusingly called 'rotations' for legacy reasons. diff --git a/config-model/src/main/java/com/yahoo/vespa/model/package-info.java b/config-model/src/main/java/com/yahoo/vespa/model/package-info.java index 48a19037569..be1506c9418 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/package-info.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/package-info.java @@ -36,7 +36,7 @@ com.yahoo.config.model.producer.AbstractConfigProducer allocation</a>. </p> - <h3>Config Generation</h3> + <h2>Config Generation</h2> <p>The method {@link com.yahoo.vespa.model.VespaModel#getConfig(com.yahoo.config.ConfigInstance.Builder, String) @@ -61,7 +61,7 @@ com.yahoo.config.model.producer.AbstractConfigProducer method and by user defined config. </p> - <h4>Example:</h4> + <h3>Example:</h3> <p> Say we have a config named 'sample' with an integer parameter named 'v'. If the VespaModel root node's {@link @@ -99,7 +99,7 @@ com.yahoo.config.model.producer.AbstractConfigProducer </p> - <h3 id="plugin_loading">Plugin Loading</h3> + <h2 id="plugin_loading">Plugin Loading</h2> <p>Each highest-level node in the setup file from the user's application specification corresponds to a {@link @@ -116,7 +116,7 @@ com.yahoo.config.model.builder.xml.ConfigModelBuilder ConfigModelBuilder}. The <p>The built models are given to other models that depends on it. </p> - <h4>Important notes for plugin developers:</h4> + <h3>Important notes for plugin developers:</h3> <ul> <li>The constructors of all child classes of {@link @@ -138,7 +138,7 @@ com.yahoo.config.model.producer.AbstractConfigProducer </ul> - <h3 id="port_allocation">Port Allocation</h3> + <h2 id="port_allocation">Port Allocation</h2> <p>Each {@link com.yahoo.vespa.model.Host Host} has an available dynamic port range running from {@link diff --git a/config-model/src/main/javacc/SDParser.jj b/config-model/src/main/javacc/SDParser.jj index 5bcc033fa37..c3851f2918a 100644 --- a/config-model/src/main/javacc/SDParser.jj +++ b/config-model/src/main/javacc/SDParser.jj @@ -154,26 +154,6 @@ public class SDParser { token_source.input_stream.getBeginColumn() + ".").initCause(e); } } - - /** - * Sets the compression threshold in each item in the compression config array. - * - * @param cfg The array of configs to modify. - * @param val The compression threshold to set. - */ - private void setCompressionThreshold(CompressionConfig cfg, int val) { - cfg.threshold = val; - } - - /** - * Sets the compression level in each item in the compression config array. - * - * @param cfg The array of configs to modify. - * @param val The compression level to set. - */ - private void setCompressionLevel(CompressionConfig cfg, int val) { - cfg.compressionLevel = val; - } } PARSER_END(SDParser) @@ -358,6 +338,7 @@ TOKEN : | < MULTITHREADEDINDEXING: "multi-threaded-indexing" > | < MATCHFEATURES_SL: "match-features" (" ")* ":" (~["}","\n"])* ("\n")? > | < MATCHFEATURES_ML: "match-features" (<SEARCHLIB_SKIP>)? "{" (~["}"])* "}" > +| < MATCHFEATURES_ML_INHERITS: "match-features inherits " (<IDENTIFIER>) (<SEARCHLIB_SKIP>)? "{" (~["}"])* "}" > | < SUMMARYFEATURES_SL: "summary-features" (" ")* ":" (~["}","\n"])* ("\n")? > | < SUMMARYFEATURES_ML: "summary-features" (<SEARCHLIB_SKIP>)? "{" (~["}"])* "}" > | < SUMMARYFEATURES_ML_INHERITS: "summary-features inherits " (<IDENTIFIER>) (<SEARCHLIB_SKIP>)? "{" (~["}"])* "}" > @@ -568,7 +549,7 @@ Object documentBody(SDDocumentType document, Schema schema) : } { ( annotation(schema, document) - | compression(document, null) + | compression(document) | headercfg(document) | bodycfg(document) | structInside(document, schema) @@ -589,7 +570,7 @@ void rawAsBase64(Schema schema) : */ void headercfg(SDDocumentType document) : { } { - <HEADER> lbrace() [compression(document, "header") (<NL>)*] <RBRACE> + <HEADER> lbrace() [compression(document) (<NL>)*] <RBRACE> } /** @@ -599,46 +580,33 @@ void headercfg(SDDocumentType document) : { } */ void bodycfg(SDDocumentType document) : { } { - <BODY> lbrace() [compression(document, "body") (<NL>)*] <RBRACE> + <BODY> lbrace() [compression(document) (<NL>)*] <RBRACE> } /** * Consumes a compression block. This can be set in both document header and -body block. * * @param document The document type to modify. - * @param name The name of the document block to modify. */ -void compression(SDDocumentType document, String name) : +void compression(SDDocumentType document) : { deployLogger.logApplicationPackage(Level.WARNING, "'compression' for a document is deprecated and ignored"); - CompressionConfig cfg = new CompressionConfig(CompressionType.LZ4); } { - <COMPRESSION> lbrace() (cfg = compressionItem(cfg) (<NL>)*)* <RBRACE> - { - if (name == null || name.equals("header")) { - document.getDocumentType().contentStruct().setCompressionConfig(cfg); - } - } + <COMPRESSION> lbrace() ( compressionItem() (<NL>)*)* <RBRACE> { } } /** * Consumes the body of a compression block. * - * @param cfg The compression config to modify. */ -CompressionConfig compressionItem(CompressionConfig cfg) : +void compressionItem() : +{ } { - int val = -1; -} -{ - ( ( <TYPE> <COLON> <LZ4> { cfg = new CompressionConfig(CompressionType.LZ4, cfg.compressionLevel, cfg.threshold); } ) - | (<COMPRESSIONTHRESHOLD> <COLON> val = integer()) { setCompressionThreshold(cfg, val); } - | (<COMPRESSIONLEVEL> <COLON> val = integer()) { setCompressionLevel(cfg, val); } - ) - { - return cfg; - } + ( ( <TYPE> <COLON> <LZ4> ) + | (<COMPRESSIONTHRESHOLD> <COLON> <INTEGER>) + | (<COMPRESSIONLEVEL> <COLON> <INTEGER>) + ) { } } /** @@ -839,14 +807,13 @@ void structOutside(Schema schema) : SDDocumentType structDefinition(Schema schema, SDDocumentType repo) : { String name; + String inherited = null; SDDocumentType struct; } { - <STRUCT> name = identifier() - { - struct = new SDDocumentType(name, schema); - } - lbrace() (structFieldDefinition(struct) (<NL>)*)* <RBRACE> + ( <STRUCT> name = identifier() (<NL>)* { struct = new SDDocumentType(name, schema); } + [ inheritsDocument(struct) (<NL>)* ] + lbrace() (structFieldDefinition(struct) (<NL>)*)* <RBRACE> ) { try { docMan.getDataType(name); @@ -2322,7 +2289,14 @@ Object matchFeatures(RankProfile profile) : { ( <MATCHFEATURES_SL> { features = token.image.substring(token.image.indexOf(":") + 1).trim(); } | <MATCHFEATURES_ML> { features = token.image.substring(token.image.indexOf("{") + 1, - token.image.lastIndexOf("}")).trim(); } ) + token.image.lastIndexOf("}")).trim(); } | + <MATCHFEATURES_ML_INHERITS> { + int inheritsIndex = token.image.indexOf("inherits "); + String rest = token.image.substring(inheritsIndex + "inherits ".length()); + profile.setInheritedMatchFeatures(rest.substring(0, rest.indexOf(" ")).trim()); + features = token.image.substring(token.image.indexOf("{") + 1, token.image.lastIndexOf("}")).trim(); + } + ) { profile.addMatchFeatures(getFeatureList(features)); return null; diff --git a/config-model/src/test/derived/rankprofileinheritance/child.sd b/config-model/src/test/derived/rankprofileinheritance/child.sd index 9369472cb23..a6e0787a659 100644 --- a/config-model/src/test/derived/rankprofileinheritance/child.sd +++ b/config-model/src/test/derived/rankprofileinheritance/child.sd @@ -20,6 +20,10 @@ schema child { attribute(field3) } + match-features inherits profile1 { + function3 + } + } rank-profile profile4 inherits profile2 { diff --git a/config-model/src/test/derived/rankprofileinheritance/parent1.sd b/config-model/src/test/derived/rankprofileinheritance/parent1.sd index d4375427e11..d25182fde4c 100644 --- a/config-model/src/test/derived/rankprofileinheritance/parent1.sd +++ b/config-model/src/test/derived/rankprofileinheritance/parent1.sd @@ -15,11 +15,18 @@ schema parent1 { expression: attribute(field1) + 5 } + function function1b() { + expression: attribute(field1) + 42 + } + summary-features { function1 attribute(field1) } + match-features { + function1b + } } } diff --git a/config-model/src/test/derived/rankprofileinheritance/rank-profiles.cfg b/config-model/src/test/derived/rankprofileinheritance/rank-profiles.cfg index 88788f5a93a..440b0ad2b97 100644 --- a/config-model/src/test/derived/rankprofileinheritance/rank-profiles.cfg +++ b/config-model/src/test/derived/rankprofileinheritance/rank-profiles.cfg @@ -11,12 +11,18 @@ rankprofile[].fef.property[].value "true" rankprofile[].name "profile3" rankprofile[].fef.property[].name "rankingExpression(function3).rankingScript" rankprofile[].fef.property[].value "attribute(field3) + 5" +rankprofile[].fef.property[].name "rankingExpression(function1b).rankingScript" +rankprofile[].fef.property[].value "attribute(field1) + 42" rankprofile[].fef.property[].name "rankingExpression(function1).rankingScript" rankprofile[].fef.property[].value "attribute(field1) + 5" rankprofile[].fef.property[].name "vespa.summary.feature" rankprofile[].fef.property[].value "attribute(field3)" rankprofile[].fef.property[].name "vespa.summary.feature" rankprofile[].fef.property[].value "rankingExpression(function3)" +rankprofile[].fef.property[].name "vespa.match.feature" +rankprofile[].fef.property[].value "rankingExpression(function3)" +rankprofile[].fef.property[].name "vespa.match.feature" +rankprofile[].fef.property[].value "rankingExpression(function1b)" rankprofile[].name "profile4" rankprofile[].fef.property[].name "rankingExpression(function2).rankingScript" rankprofile[].fef.property[].value "attribute(field2) + 5" diff --git a/config-model/src/test/derived/schemainheritance/summary.cfg b/config-model/src/test/derived/schemainheritance/summary.cfg index 451894ec2c1..d3286961007 100644 --- a/config-model/src/test/derived/schemainheritance/summary.cfg +++ b/config-model/src/test/derived/schemainheritance/summary.cfg @@ -1,19 +1,19 @@ -defaultsummaryid 1044042981 -classes[].id 1044042981 +defaultsummaryid 1313596701 +classes[].id 1313596701 classes[].name "default" classes[].omitsummaryfeatures false classes[].fields[].name "parent_field" classes[].fields[].type "longstring" +classes[].fields[].name "child_field" +classes[].fields[].type "longstring" classes[].fields[].name "pf1" classes[].fields[].type "longstring" +classes[].fields[].name "cf1" +classes[].fields[].type "longstring" classes[].fields[].name "rankfeatures" classes[].fields[].type "featuredata" classes[].fields[].name "summaryfeatures" classes[].fields[].type "featuredata" -classes[].fields[].name "child_field" -classes[].fields[].type "longstring" -classes[].fields[].name "cf1" -classes[].fields[].type "longstring" classes[].fields[].name "documentid" classes[].fields[].type "longstring" classes[].id 2134223620 @@ -25,17 +25,17 @@ classes[].fields[].name "rankfeatures" classes[].fields[].type "featuredata" classes[].fields[].name "summaryfeatures" classes[].fields[].type "featuredata" -classes[].id 178044032 +classes[].id 524210908 classes[].name "attributeprefetch" classes[].omitsummaryfeatures false classes[].fields[].name "parent_field" classes[].fields[].type "longstring" +classes[].fields[].name "child_field" +classes[].fields[].type "longstring" classes[].fields[].name "rankfeatures" classes[].fields[].type "featuredata" classes[].fields[].name "summaryfeatures" classes[].fields[].type "featuredata" -classes[].fields[].name "child_field" -classes[].fields[].type "longstring" classes[].id 1486475170 classes[].name "child_summary" classes[].omitsummaryfeatures false @@ -46,4 +46,4 @@ classes[].fields[].type "featuredata" classes[].fields[].name "summaryfeatures" classes[].fields[].type "featuredata" classes[].fields[].name "cf1" -classes[].fields[].type "longstring" +classes[].fields[].type "longstring"
\ No newline at end of file diff --git a/config-model/src/test/java/com/yahoo/config/model/deploy/DeployStateTest.java b/config-model/src/test/java/com/yahoo/config/model/deploy/DeployStateTest.java index 36da5c2f142..031f25306d5 100644 --- a/config-model/src/test/java/com/yahoo/config/model/deploy/DeployStateTest.java +++ b/config-model/src/test/java/com/yahoo/config/model/deploy/DeployStateTest.java @@ -2,6 +2,7 @@ package com.yahoo.config.model.deploy; import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.config.model.api.ApplicationClusterEndpoint; import com.yahoo.config.model.api.ConfigDefinitionRepo; import com.yahoo.config.model.api.ContainerEndpoint; import com.yahoo.config.model.api.HostProvisioner; @@ -111,7 +112,7 @@ public class DeployStateTest { @Test public void testContainerEndpoints() { assertTrue(new DeployState.Builder().endpoints(Set.of()).build().getEndpoints().isEmpty()); - var endpoints = Set.of(new ContainerEndpoint("c1", List.of("c1.example.com", "c1-alias.example.com"))); + var endpoints = Set.of(new ContainerEndpoint("c1", ApplicationClusterEndpoint.Scope.global, List.of("c1.example.com", "c1-alias.example.com"))); assertEquals(endpoints, new DeployState.Builder().endpoints(endpoints).build().getEndpoints()); } diff --git a/config-model/src/test/java/com/yahoo/document/test/SDDocumentTypeTestCase.java b/config-model/src/test/java/com/yahoo/document/test/SDDocumentTypeTestCase.java index 4714f4396dd..f1bebdb0a29 100644 --- a/config-model/src/test/java/com/yahoo/document/test/SDDocumentTypeTestCase.java +++ b/config-model/src/test/java/com/yahoo/document/test/SDDocumentTypeTestCase.java @@ -5,12 +5,18 @@ import com.yahoo.document.DataType; import com.yahoo.document.DataTypeName; import com.yahoo.documentmodel.VespaDocumentType; import com.yahoo.searchdefinition.AbstractSchemaTestCase; +import com.yahoo.searchdefinition.SchemaBuilder; import com.yahoo.searchdefinition.document.SDDocumentType; import com.yahoo.searchdefinition.document.SDField; +import com.yahoo.searchdefinition.parser.ParseException; +import com.yahoo.searchdefinition.processing.ImportedFieldsResolver; +import com.yahoo.searchdefinition.processing.OnnxModelTypeResolver; +import com.yahoo.vespa.model.test.utils.DeployLoggerStub; import org.junit.Test; import java.util.Iterator; +import static com.yahoo.config.model.test.TestUtil.joinLines; import static org.junit.Assert.*; /** @@ -95,8 +101,43 @@ public class SDDocumentTypeTestCase extends AbstractSchemaTestCase { field = (SDField) fields.next(); assertEquals("childfield", field.getName()); + } - // TODO: Test uninheriting + @Test + public void testStructInheritance() throws ParseException { + String schemaLines = joinLines( + "schema test {" + + " document test {" + + " struct parent_struct {" + + " field parent_struct_field_1 type string {}" + + " }" + + " struct child_struct inherits parent_struct {" + + " field child_struct_field_1 type string {}" + + " }" + + " field child_array type array<child_struct> {" + + " indexing: summary\n" + + " struct-field child_struct_field_1 { indexing: attribute }" + + " struct-field parent_struct_field_1 { indexing: attribute }" + + " }" + + " }" + + "}"); + + SchemaBuilder builder = new SchemaBuilder(new DeployLoggerStub()); + builder.importString(schemaLines); + builder.build(true); + var application = builder.application(); + + SDDocumentType type = application.schemas().get("test").getDocument(); + + SDDocumentType parent_struct = type.getOwnedType("parent_struct"); + assertEquals(1, parent_struct.fieldSet().size()); + assertNotNull(parent_struct.getField("parent_struct_field_1")); + + SDDocumentType child_struct = type.getOwnedType("child_struct"); + assertTrue(child_struct.inheritedTypes().containsKey(parent_struct.getDocumentName())); + assertEquals(2, child_struct.fieldSet().size()); + assertNotNull(child_struct.getField("child_struct_field_1")); + assertNotNull(child_struct.getField("parent_struct_field_1")); } } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/SchemaTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/SchemaTestCase.java index 860b741a1f2..d906685d502 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/SchemaTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/SchemaTestCase.java @@ -5,6 +5,7 @@ import com.yahoo.searchdefinition.document.Stemming; import com.yahoo.searchdefinition.parser.ParseException; import com.yahoo.searchdefinition.processing.ImportedFieldsResolver; import com.yahoo.searchdefinition.processing.OnnxModelTypeResolver; +import com.yahoo.vespa.documentmodel.DocumentSummary; import com.yahoo.vespa.model.test.utils.DeployLoggerStub; import org.junit.Test; @@ -102,81 +103,157 @@ public class SchemaTestCase { " import field parentschema_ref.name as parent_imported {}" + " raw-as-base64-in-summary" + "}"); - String childLines = joinLines( - "schema child inherits parent {" + - " document child inherits parent {" + - " field cf1 type string {" + + String child1Lines = joinLines( + "schema child1 inherits parent {" + + " document child1 inherits parent {" + + " field c1f1 type string {" + " indexing: summary" + " }" + " }" + - " fieldset child_set {" + - " fields: cf1, pf1" + + " fieldset child1_set {" + + " fields: c1f1, pf1" + " }" + " stemming: shortest" + - " index child_index {" + + " index child1_index {" + " stemming: shortest" + " }" + - " field child_field type string {" + + " field child1_field type string {" + " indexing: input pf1 | lowercase | index | attribute | summary" + " }" + - " rank-profile child_profile inherits parent_profile {" + + " rank-profile child1_profile inherits parent_profile {" + " }" + - " constant child_constant {" + + " constant child1_constant {" + " file: constants/my_constant_tensor_file.json" + " type: tensor<float>(x{},y{})" + " }" + - " onnx-model child_model {" + + " onnx-model child1_model {" + " file: models/my_model.onnx" + " }" + - " document-summary child_summary inherits parent_summary {" + - " summary cf1 type string {}" + + " document-summary child1_summary inherits parent_summary {" + + " summary c1f1 type string {}" + " }" + - " import field parentschema_ref.name as child_imported {}" + + " import field parentschema_ref.name as child1_imported {}" + + "}"); + String child2Lines = joinLines( + "schema child2 inherits parent {" + + " document child2 inherits parent {" + + " field c2f1 type string {" + + " indexing: summary" + + " }" + + " }" + + " fieldset child2_set {" + + " fields: c2f1, pf1" + + " }" + + " stemming: shortest" + + " index child2_index {" + + " stemming: shortest" + + " }" + + " field child2_field type string {" + + " indexing: input pf1 | lowercase | index | attribute | summary" + + " }" + + " rank-profile child2_profile inherits parent_profile {" + + " }" + + " constant child2_constant {" + + " file: constants/my_constant_tensor_file.json" + + " type: tensor<float>(x{},y{})" + + " }" + + " onnx-model child2_model {" + + " file: models/my_model.onnx" + + " }" + + " document-summary child2_summary inherits parent_summary {" + + " summary c2f1 type string {}" + + " }" + + " import field parentschema_ref.name as child2_imported {}" + "}"); - System.out.println(parentLines); SchemaBuilder builder = new SchemaBuilder(new DeployLoggerStub()); builder.processorsToSkip().add(OnnxModelTypeResolver.class); // Avoid discovering the Onnx model referenced does not exist builder.processorsToSkip().add(ImportedFieldsResolver.class); // Avoid discovering the document reference leads nowhere builder.importString(parentLines); - builder.importString(childLines); + builder.importString(child1Lines); + builder.importString(child2Lines); builder.build(true); var application = builder.application(); - var child = application.schemas().get("child"); - assertEquals("pf1", child.fieldSets().userFieldSets().get("parent_set").getFieldNames().stream().findFirst().get()); - assertEquals("[cf1, pf1]", child.fieldSets().userFieldSets().get("child_set").getFieldNames().toString()); - assertEquals(Stemming.SHORTEST, child.getStemming()); - assertEquals(Stemming.BEST, child.getIndex("parent_index").getStemming()); - assertEquals(Stemming.SHORTEST, child.getIndex("child_index").getStemming()); - assertNotNull(child.getField("parent_field")); - assertNotNull(child.getField("child_field")); - assertNotNull(child.getExtraField("parent_field")); - assertNotNull(child.getExtraField("child_field")); - assertNotNull(builder.getRankProfileRegistry().get(child, "parent_profile")); - assertNotNull(builder.getRankProfileRegistry().get(child, "child_profile")); - assertEquals("parent_profile", builder.getRankProfileRegistry().get(child, "child_profile").getInheritedName()); - assertNotNull(child.rankingConstants().get("parent_constant")); - assertNotNull(child.rankingConstants().get("child_constant")); - assertTrue(child.rankingConstants().asMap().containsKey("parent_constant")); - assertTrue(child.rankingConstants().asMap().containsKey("child_constant")); - assertNotNull(child.onnxModels().get("parent_model")); - assertNotNull(child.onnxModels().get("child_model")); - assertTrue(child.onnxModels().asMap().containsKey("parent_model")); - assertTrue(child.onnxModels().asMap().containsKey("child_model")); - assertNotNull(child.getSummary("parent_summary")); - assertNotNull(child.getSummary("child_summary")); - assertEquals("parent_summary", child.getSummary("child_summary").inherited().get().getName()); - assertTrue(child.getSummaries().containsKey("parent_summary")); - assertTrue(child.getSummaries().containsKey("child_summary")); - assertNotNull(child.getSummaryField("pf1")); - assertNotNull(child.getSummaryField("cf1")); - assertNotNull(child.getExplicitSummaryField("pf1")); - assertNotNull(child.getExplicitSummaryField("cf1")); - assertNotNull(child.getUniqueNamedSummaryFields().get("pf1")); - assertNotNull(child.getUniqueNamedSummaryFields().get("cf1")); - assertNotNull(child.temporaryImportedFields().get().fields().get("parent_imported")); - assertNotNull(child.temporaryImportedFields().get().fields().get("child_imported")); + var child1 = application.schemas().get("child1"); + assertEquals("pf1", child1.fieldSets().userFieldSets().get("parent_set").getFieldNames().stream().findFirst().get()); + assertEquals("[c1f1, pf1]", child1.fieldSets().userFieldSets().get("child1_set").getFieldNames().toString()); + assertEquals(Stemming.SHORTEST, child1.getStemming()); + assertEquals(Stemming.BEST, child1.getIndex("parent_index").getStemming()); + assertEquals(Stemming.SHORTEST, child1.getIndex("child1_index").getStemming()); + assertNotNull(child1.getField("parent_field")); + assertNotNull(child1.getField("child1_field")); + assertNotNull(child1.getExtraField("parent_field")); + assertNotNull(child1.getExtraField("child1_field")); + assertNotNull(builder.getRankProfileRegistry().get(child1, "parent_profile")); + assertNotNull(builder.getRankProfileRegistry().get(child1, "child1_profile")); + assertEquals("parent_profile", builder.getRankProfileRegistry().get(child1, "child1_profile").getInheritedName()); + assertNotNull(child1.rankingConstants().get("parent_constant")); + assertNotNull(child1.rankingConstants().get("child1_constant")); + assertTrue(child1.rankingConstants().asMap().containsKey("parent_constant")); + assertTrue(child1.rankingConstants().asMap().containsKey("child1_constant")); + assertNotNull(child1.onnxModels().get("parent_model")); + assertNotNull(child1.onnxModels().get("child1_model")); + assertTrue(child1.onnxModels().asMap().containsKey("parent_model")); + assertTrue(child1.onnxModels().asMap().containsKey("child1_model")); + assertNotNull(child1.getSummary("parent_summary")); + assertNotNull(child1.getSummary("child1_summary")); + assertEquals("parent_summary", child1.getSummary("child1_summary").inherited().get().getName()); + assertTrue(child1.getSummaries().containsKey("parent_summary")); + assertTrue(child1.getSummaries().containsKey("child1_summary")); + assertNotNull(child1.getSummaryField("pf1")); + assertNotNull(child1.getSummaryField("c1f1")); + assertNotNull(child1.getExplicitSummaryField("pf1")); + assertNotNull(child1.getExplicitSummaryField("c1f1")); + assertNotNull(child1.getUniqueNamedSummaryFields().get("pf1")); + assertNotNull(child1.getUniqueNamedSummaryFields().get("c1f1")); + assertNotNull(child1.temporaryImportedFields().get().fields().get("parent_imported")); + assertNotNull(child1.temporaryImportedFields().get().fields().get("child1_imported")); + + var child2 = application.schemas().get("child2"); + assertEquals("pf1", child2.fieldSets().userFieldSets().get("parent_set").getFieldNames().stream().findFirst().get()); + assertEquals("[c2f1, pf1]", child2.fieldSets().userFieldSets().get("child2_set").getFieldNames().toString()); + assertEquals(Stemming.SHORTEST, child2.getStemming()); + assertEquals(Stemming.BEST, child2.getIndex("parent_index").getStemming()); + assertEquals(Stemming.SHORTEST, child2.getIndex("child2_index").getStemming()); + assertNotNull(child2.getField("parent_field")); + assertNotNull(child2.getField("child2_field")); + assertNotNull(child2.getExtraField("parent_field")); + assertNotNull(child2.getExtraField("child2_field")); + assertNotNull(builder.getRankProfileRegistry().get(child2, "parent_profile")); + assertNotNull(builder.getRankProfileRegistry().get(child2, "child2_profile")); + assertEquals("parent_profile", builder.getRankProfileRegistry().get(child2, "child2_profile").getInheritedName()); + assertNotNull(child2.rankingConstants().get("parent_constant")); + assertNotNull(child2.rankingConstants().get("child2_constant")); + assertTrue(child2.rankingConstants().asMap().containsKey("parent_constant")); + assertTrue(child2.rankingConstants().asMap().containsKey("child2_constant")); + assertNotNull(child2.onnxModels().get("parent_model")); + assertNotNull(child2.onnxModels().get("child2_model")); + assertTrue(child2.onnxModels().asMap().containsKey("parent_model")); + assertTrue(child2.onnxModels().asMap().containsKey("child2_model")); + assertNotNull(child2.getSummary("parent_summary")); + assertNotNull(child2.getSummary("child2_summary")); + assertEquals("parent_summary", child2.getSummary("child2_summary").inherited().get().getName()); + assertTrue(child2.getSummaries().containsKey("parent_summary")); + assertTrue(child2.getSummaries().containsKey("child2_summary")); + assertNotNull(child2.getSummaryField("pf1")); + assertNotNull(child2.getSummaryField("c2f1")); + assertNotNull(child2.getExplicitSummaryField("pf1")); + assertNotNull(child2.getExplicitSummaryField("c2f1")); + assertNotNull(child2.getUniqueNamedSummaryFields().get("pf1")); + assertNotNull(child2.getUniqueNamedSummaryFields().get("c2f1")); + assertNotNull(child2.temporaryImportedFields().get().fields().get("parent_imported")); + assertNotNull(child2.temporaryImportedFields().get().fields().get("child2_imported")); + DocumentSummary child2DefaultSummary = child2.getSummary("default"); + assertEquals(6, child2DefaultSummary.getSummaryFields().size()); + assertTrue(child2DefaultSummary.getSummaryFields().containsKey("child2_field")); + assertTrue(child2DefaultSummary.getSummaryFields().containsKey("parent_field")); + assertTrue(child2DefaultSummary.getSummaryFields().containsKey("pf1")); + assertTrue(child2DefaultSummary.getSummaryFields().containsKey("c2f1")); + DocumentSummary child2AttributeprefetchSummary = child2.getSummary("attributeprefetch"); + assertEquals(4, child2AttributeprefetchSummary.getSummaryFields().size()); + assertTrue(child2AttributeprefetchSummary.getSummaryFields().containsKey("child2_field")); + assertTrue(child2AttributeprefetchSummary.getSummaryFields().containsKey("parent_field")); } @Test diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/InheritanceTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/InheritanceTestCase.java index 97cfc48580d..79df1fc9501 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/InheritanceTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/InheritanceTestCase.java @@ -50,7 +50,7 @@ public class InheritanceTestCase extends AbstractExportingTestCase { try { assertCorrectDeriving("inheritfromnull"); } catch (IllegalStateException e) { - assertEquals("Document type 'foo' not found.", e.getMessage()); + assertEquals("Document type 'foo' not found", e.getMessage()); } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java index 7190a7d4db1..2016cea02a9 100755 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java @@ -7,6 +7,8 @@ import com.yahoo.cloud.config.CuratorConfig; import com.yahoo.cloud.config.ZookeeperServerConfig; import com.yahoo.component.ComponentId; import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.config.model.api.ApplicationClusterEndpoint; +import com.yahoo.config.model.api.ContainerEndpoint; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.deploy.TestProperties; import com.yahoo.config.model.test.MockApplicationPackage; @@ -36,8 +38,11 @@ import java.util.ArrayList; import java.util.Collection; import java.util.Iterator; import java.util.List; +import java.util.Objects; +import java.util.Set; import java.util.stream.Collectors; +import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.hasItem; import static org.hamcrest.Matchers.hasKey; import static org.junit.Assert.assertEquals; @@ -351,6 +356,49 @@ public class ContainerClusterTest { } + @Test + public void generatesCorrectRoutingInfo() { + + assertNames(ApplicationId.from("t1", "a1", "i1"), + Set.of(), + List.of("search-cluster.i1.a1.t1.endpoint.suffix", "search-cluster--i1--a1--t1.endpoint.suffix")); + + assertNames(ApplicationId.from("t1", "a1", "default"), + Set.of(), + List.of("search-cluster.a1.t1.endpoint.suffix", "search-cluster--a1--t1.endpoint.suffix")); + + assertNames(ApplicationId.from("t1", "default", "default"), + Set.of(), + List.of("search-cluster.default.t1.endpoint.suffix", "search-cluster--default--t1.endpoint.suffix")); + + assertNames(ApplicationId.from("t1", "a1", "default"), + Set.of(new ContainerEndpoint("not-in-this-cluster", ApplicationClusterEndpoint.Scope.global, List.of("foo", "bar"))), + List.of("search-cluster.a1.t1.endpoint.suffix", "search-cluster--a1--t1.endpoint.suffix")); + + assertNames(ApplicationId.from("t1", "a1", "default"), + Set.of(new ContainerEndpoint("search-cluster", ApplicationClusterEndpoint.Scope.global, List.of("rotation-1.x.y.z", "rotation-2.x.y.z")), + new ContainerEndpoint("search-cluster", ApplicationClusterEndpoint.Scope.application, List.of("app-rotation.x.y.z"))), + List.of("search-cluster.a1.t1.endpoint.suffix", "search-cluster--a1--t1.endpoint.suffix", "rotation-1.x.y.z", "rotation-2.x.y.z", "app-rotation.x.y.z")); + } + + private void assertNames(ApplicationId appId, Set<ContainerEndpoint> globalEndpoints, List<String> expectedNames) { + DeployState state = new DeployState.Builder() + .zone(Zone.defaultZone()) + .endpoints(globalEndpoints) + .properties(new TestProperties() + .setHostedVespa(true) + .setApplicationId(appId) + .setZoneDnsSuffixes(List.of(".endpoint.suffix"))) + .build(); + MockRoot root = new MockRoot("foo", state); + ApplicationContainerCluster cluster = new ApplicationContainerCluster(root, "container", "search-cluster", state); + addContainer(root, cluster, "c1", "host-c1"); + cluster.doPrepare(state); + List<ApplicationClusterEndpoint> endpoints = cluster.endpoints(); + assertEquals(expectedNames.size(), endpoints.size()); + expectedNames.forEach(expected -> assertTrue("Endpoint not matched " + expected, endpoints.stream().anyMatch(e -> Objects.equals(e.dnsName().value(), expected)))); + } + private void verifyTesterApplicationInstalledBundles(Zone zone, List<String> expectedBundleNames) { ApplicationId appId = ApplicationId.from("tenant", "application", "instance-t"); DeployState state = new DeployState.Builder().properties( diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/http/BlockFeedGlobalEndpointsFilterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/http/BlockFeedGlobalEndpointsFilterTest.java index 6d51c3a50b6..1d774526a9b 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/http/BlockFeedGlobalEndpointsFilterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/http/BlockFeedGlobalEndpointsFilterTest.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.model.container.http; +import com.yahoo.config.model.api.ApplicationClusterEndpoint; import com.yahoo.config.model.api.ContainerEndpoint; import com.yahoo.jdisc.http.filter.security.rule.RuleBasedFilterConfig; import org.hamcrest.Matchers; @@ -19,7 +20,7 @@ public class BlockFeedGlobalEndpointsFilterTest { @Test public void setup_blocking_rule_when_endpoints_is_non_empty() { - var endpoints = Set.of(new ContainerEndpoint("default", List.of("foo", "bar"))); + var endpoints = Set.of(new ContainerEndpoint("default", ApplicationClusterEndpoint.Scope.global, List.of("foo", "bar"))); var filter = new BlockFeedGlobalEndpointsFilter(endpoints, true); var config = getConfig(filter); assertEquals(1, config.rule().size()); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java index be6d8ca5d0a..5516c74f9a6 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java @@ -5,6 +5,7 @@ import com.yahoo.cloud.config.ZookeeperServerConfig; import com.yahoo.component.ComponentId; import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.model.NullConfigModelRegistry; +import com.yahoo.config.model.api.ApplicationClusterEndpoint; import com.yahoo.config.model.api.ContainerEndpoint; import com.yahoo.config.model.api.EndpointCertificateSecrets; import com.yahoo.config.model.api.ModelContext; @@ -613,7 +614,7 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase { final var deployState = new DeployState.Builder() .applicationPackage(applicationPackage) .zone(new Zone(Environment.prod, RegionName.from("us-east-1"))) - .endpoints(Set.of(new ContainerEndpoint("comics-search", List.of("nalle", "balle")))) + .endpoints(Set.of(new ContainerEndpoint("comics-search", ApplicationClusterEndpoint.Scope.global, List.of("nalle", "balle")))) .properties(new TestProperties().setHostedVespa(true)) .build(); @@ -796,6 +797,34 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase { } @Test + public void cloud_secret_store_fails_to_set_up_in_non_public_zone() { + try { + Element clusterElem = DomBuilderTest.parse( + "<container version='1.0'>", + " <secret-store type='cloud'>", + " <store id='store'>", + " <aws-parameter-store account='store1' region='eu-north-1'/>", + " </store>", + " </secret-store>", + "</container>"); + + DeployState state = new DeployState.Builder() + .properties( + new TestProperties() + .setHostedVespa(true) + .setTenantSecretStores(List.of(new TenantSecretStore("store1", "1234", "role", Optional.of("externalid"))))) + .zone(new Zone(SystemName.main, Environment.prod, RegionName.defaultName())) + .build(); + createModel(root, state, null, clusterElem); + } catch (RuntimeException e) { + assertEquals("cloud secret store is not supported in non-public system, please see documentation", + e.getMessage()); + return; + } + fail(); + } + + @Test public void missing_security_clients_pem_fails_in_public() { Element clusterElem = DomBuilderTest.parse("<container version='1.0' />"); diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/zone/ZoneFilter.java b/config-provisioning/src/main/java/com/yahoo/config/provision/zone/ZoneFilter.java index 168d765ab06..058be998478 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/zone/ZoneFilter.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/zone/ZoneFilter.java @@ -19,9 +19,6 @@ public interface ZoneFilter { /** Zones which are upgraded by the controller. */ ZoneList controllerUpgraded(); - /** Zones which support direct routing through exclusive load balancers. */ - ZoneList directlyRouted(); - /** Zones where traffic is routed using given method */ ZoneList routingMethod(RoutingMethod method); diff --git a/config-provisioning/src/main/resources/configdefinitions/config.provisioning.node-repository.def b/config-provisioning/src/main/resources/configdefinitions/config.provisioning.node-repository.def index f6976273b7f..6181efc7184 100644 --- a/config-provisioning/src/main/resources/configdefinitions/config.provisioning.node-repository.def +++ b/config-provisioning/src/main/resources/configdefinitions/config.provisioning.node-repository.def @@ -4,7 +4,7 @@ namespace=config.provisioning # Default container image to use for nodes. containerImage string default="registry.example.com:9999/myorg/vespa" -# Default container image to use for tenant nodes. If this is unset (empty), it defaults to containerImage +# Default container image to use for tenant nodes. If this is unset (empty), it defaults to containerImage. tenantContainerImage string default="" # Whether to cache data read from ZooKeeper in-memory. diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/Subscriber.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/Subscriber.java index abe0ec5cbb0..70ff4456f6c 100644 --- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/Subscriber.java +++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/Subscriber.java @@ -43,8 +43,13 @@ public class Subscriber { public Optional<RawConfig> nextGeneration() { try { - if (subscriber.nextGeneration(0, true))// Proxy should never skip config due to not initializing - return Optional.of(handle.getRawConfig()); + // 'isInitializing' argument to nextGeneration() is true, config proxy should never skip config due to not initializing + if (subscriber.nextGeneration(0, true)) { + RawConfig rawConfig = handle.getRawConfig(); + if (rawConfig == null) + log.log(Level.SEVERE, "Config for " + config.getKey() + " is null"); + return Optional.ofNullable(rawConfig); + } } catch (Exception e) { // To avoid thread throwing exception and loop never running this again log.log(Level.WARNING, "Got exception: " + Exceptions.toMessageString(e)); } catch (Throwable e) { diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/package-info.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/package-info.java index 10e945ae202..b4c38c84482 100644 --- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/package-info.java +++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/package-info.java @@ -14,7 +14,7 @@ the config client library. </p> - <h3>Why Vespa needs a config proxy</h3> + <h2>Why Vespa needs a config proxy</h2> <p>It is possible for a client to subscribe to config from the config server directly. However, if all Vespa diff --git a/config/src/main/java/com/yahoo/config/subscription/ConfigSubscriber.java b/config/src/main/java/com/yahoo/config/subscription/ConfigSubscriber.java index f5ed79a1d44..07132c460f9 100644 --- a/config/src/main/java/com/yahoo/config/subscription/ConfigSubscriber.java +++ b/config/src/main/java/com/yahoo/config/subscription/ConfigSubscriber.java @@ -14,6 +14,7 @@ import java.util.List; import java.util.Map; import java.util.concurrent.CopyOnWriteArrayList; +import java.util.logging.Level; import java.util.logging.Logger; import static java.util.logging.Level.FINE; @@ -293,10 +294,13 @@ public class ConfigSubscriber implements AutoCloseable { // Keep on polling the subscriptions until we have a new generation across the board, or it times out for (ConfigHandle<? extends ConfigInstance> h : subscriptionHandles) { ConfigSubscription<? extends ConfigInstance> subscription = h.subscription(); + log.log(Level.FINEST, () -> "Calling nextConfig for " + subscription.getKey()); if ( ! subscription.nextConfig(timeLeftMillis)) { // This subscriber has no new state and we know it has exhausted all time + log.log(Level.FINEST, () -> "No new config for " + subscription.getKey()); return false; } + log.log(Level.FINEST, () -> "Got new generation or config for " + subscription.getKey()); throwIfExceptionSet(subscription); ConfigSubscription.ConfigState<? extends ConfigInstance> config = subscription.getConfigState(); if (currentGen == null) currentGen = config.getGeneration(); @@ -322,6 +326,7 @@ public class ConfigSubscriber implements AutoCloseable { if (reconfigDue) { // This indicates the clients will possibly reconfigure their services, so "reset" changed-logic in subscriptions. // Also if appropriate update the changed flag on the handler, which clients use. + log.log(Level.FINE, () -> "Reconfig will happen for generation " + generation); markSubsChangedSeen(currentGen); synchronized (monitor) { generation = currentGen; diff --git a/config/src/main/java/com/yahoo/config/subscription/impl/ConfigSubscription.java b/config/src/main/java/com/yahoo/config/subscription/impl/ConfigSubscription.java index 6a81c2279d1..a3265671d50 100644 --- a/config/src/main/java/com/yahoo/config/subscription/impl/ConfigSubscription.java +++ b/config/src/main/java/com/yahoo/config/subscription/impl/ConfigSubscription.java @@ -16,7 +16,9 @@ import com.yahoo.vespa.config.TimingValues; import com.yahoo.vespa.config.protocol.DefContent; import java.io.File; +import java.util.Objects; import java.util.concurrent.atomic.AtomicReference; +import java.util.logging.Level; import java.util.logging.Logger; import static com.yahoo.vespa.config.PayloadChecksum.Type.MD5; @@ -203,6 +205,18 @@ public abstract class ConfigSubscription<T extends ConfigInstance> { this.config.set(new ConfigState<>(true, generation, applyOnRestart, true, config, payloadChecksums)); } + void setConfigAndGeneration(Long generation, boolean applyOnRestart, T config, PayloadChecksums payloadChecksums) { + ConfigState<T> prev = this.config.get(); + boolean configChanged = !Objects.equals(prev.getConfig(), config); + String message = "Config has changed unexpectedly for " + key + ", generation " + generation; + if (configChanged) { + if (log.isLoggable(Level.FINE)) + message = message + ", config in state :" + prev.getConfig() + ", new config: " + config; + log.log(Level.WARNING, message); + } + this.config.set(new ConfigState<>(true, generation, applyOnRestart, configChanged, config, payloadChecksums)); + } + /** * Used by {@link FileConfigSubscription} and {@link ConfigSetSubscription} */ @@ -213,7 +227,7 @@ public abstract class ConfigSubscription<T extends ConfigInstance> { protected void setConfigIfChanged(T config) { ConfigState<T> prev = this.config.get(); - this.config.set(new ConfigState<>(true, prev.getGeneration(), prev.applyOnRestart(), !config.equals(prev.getConfig()), config, prev.payloadChecksums)); + this.config.set(new ConfigState<>(true, prev.getGeneration(), prev.applyOnRestart(), !Objects.equals(prev.getConfig(), config), config, prev.payloadChecksums)); } void setGeneration(Long generation) { diff --git a/config/src/main/java/com/yahoo/config/subscription/impl/GenericJRTConfigSubscription.java b/config/src/main/java/com/yahoo/config/subscription/impl/GenericJRTConfigSubscription.java index dbe13e90c9c..354489ea946 100644 --- a/config/src/main/java/com/yahoo/config/subscription/impl/GenericJRTConfigSubscription.java +++ b/config/src/main/java/com/yahoo/config/subscription/impl/GenericJRTConfigSubscription.java @@ -49,6 +49,17 @@ public class GenericJRTConfigSubscription extends JRTConfigSubscription<RawConfi } } + // Need to override this method, since we use RawConfig in this class, + @Override + protected void setNewConfigAndGeneration(JRTClientConfigRequest jrtReq) { + // Set generation first, as RawConfig contains generation and that + // will make configChanged in ConfigState always true otherwise + // (see equals usage in setConfigAndGeneration()) + setGeneration(jrtReq.getNewGeneration()); + RawConfig rawConfig = RawConfig.createFromResponseParameters(jrtReq); + setConfigAndGeneration(jrtReq.getNewGeneration(), jrtReq.responseIsApplyOnRestart(), rawConfig, jrtReq.getNewChecksums()); + } + // Override to propagate internal redeploy into the config value in addition to the config state @Override void setApplyOnRestart(boolean applyOnRestart) { diff --git a/config/src/main/java/com/yahoo/config/subscription/impl/JRTConfigSubscription.java b/config/src/main/java/com/yahoo/config/subscription/impl/JRTConfigSubscription.java index 27099790f5b..c6ea79ddbcd 100644 --- a/config/src/main/java/com/yahoo/config/subscription/impl/JRTConfigSubscription.java +++ b/config/src/main/java/com/yahoo/config/subscription/impl/JRTConfigSubscription.java @@ -73,7 +73,7 @@ public class JRTConfigSubscription<T extends ConfigInstance> extends ConfigSubsc if (jrtReq.hasUpdatedConfig()) { setNewConfig(jrtReq); } else { - setGeneration(jrtReq.getNewGeneration()); + setNewConfigAndGeneration(jrtReq); } } @@ -110,6 +110,18 @@ public class JRTConfigSubscription<T extends ConfigInstance> extends ConfigSubsc } } + protected void setNewConfigAndGeneration(JRTClientConfigRequest jrtReq) { + try { + T configInstance = toConfigInstance(jrtReq); + setConfigAndGeneration(jrtReq.getNewGeneration(), + jrtReq.responseIsApplyOnRestart(), + configInstance, + jrtReq.getNewChecksums()); + } catch (IllegalArgumentException e) { + throw new IllegalArgumentException("Bad config in response", e); + } + } + /** * This method should ideally throw new MissingConfig/Configuration exceptions and let the caller * catch them. However, this would make the code in JRT/File/RawSource uglier. diff --git a/config/src/main/java/com/yahoo/config/subscription/impl/MockConnection.java b/config/src/main/java/com/yahoo/config/subscription/impl/MockConnection.java index 640f3d0c27e..6446758a9ba 100644 --- a/config/src/main/java/com/yahoo/config/subscription/impl/MockConnection.java +++ b/config/src/main/java/com/yahoo/config/subscription/impl/MockConnection.java @@ -83,11 +83,13 @@ public class MockConnection implements ConnectionPool, Connection { static class OKResponseHandler extends AbstractResponseHandler { + long generation = 1; + protected void createResponse() { JRTServerConfigRequestV3 jrtReq = JRTServerConfigRequestV3.createFromRequest(request); Payload payload = Payload.from(ConfigPayload.empty()); - long generation = 1; jrtReq.addOkResponse(payload, generation, false, PayloadChecksums.fromPayload(payload)); + generation++; } } diff --git a/config/src/test/java/com/yahoo/config/subscription/GenericConfigSubscriberTest.java b/config/src/test/java/com/yahoo/config/subscription/GenericConfigSubscriberTest.java index 6174432676c..4616630557e 100644 --- a/config/src/test/java/com/yahoo/config/subscription/GenericConfigSubscriberTest.java +++ b/config/src/test/java/com/yahoo/config/subscription/GenericConfigSubscriberTest.java @@ -1,25 +1,23 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.config.subscription; -import java.util.HashMap; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; - import com.yahoo.config.subscription.impl.GenericConfigHandle; import com.yahoo.config.subscription.impl.GenericConfigSubscriber; import com.yahoo.config.subscription.impl.JRTConfigRequester; import com.yahoo.config.subscription.impl.JRTConfigRequesterTest; import com.yahoo.config.subscription.impl.MockConnection; import com.yahoo.vespa.config.ConfigKey; -import com.yahoo.vespa.config.JRTConnectionPool; +import com.yahoo.vespa.config.TimingValues; import com.yahoo.vespa.config.protocol.CompressionType; import org.junit.Test; -import static org.hamcrest.CoreMatchers.is; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; /** @@ -29,28 +27,45 @@ import static org.junit.Assert.assertTrue; * @author Ulf Lilleengen */ public class GenericConfigSubscriberTest { + private static final TimingValues tv = JRTConfigRequesterTest.getTestTimingValues(); @Test - public void testSubscribeGeneric() { + public void testSubscribeGeneric() throws InterruptedException { Map<ConfigSourceSet, JRTConfigRequester> requesters = new HashMap<>(); ConfigSourceSet sourceSet = new ConfigSourceSet("blabla"); - requesters.put(sourceSet, new JRTConfigRequester(new MockConnection(), JRTConfigRequesterTest.getTestTimingValues())); + requesters.put(sourceSet, new JRTConfigRequester(new MockConnection(), tv)); GenericConfigSubscriber sub = new GenericConfigSubscriber(requesters); final List<String> defContent = List.of("myVal int"); - GenericConfigHandle handle = sub.subscribe(new ConfigKey<>("simpletypes", "id", "config"), defContent, sourceSet, JRTConfigRequesterTest.getTestTimingValues()); + GenericConfigHandle handle = sub.subscribe(new ConfigKey<>("simpletypes", "id", "config"), + defContent, + sourceSet, + tv); assertTrue(sub.nextConfig(false)); assertTrue(handle.isChanged()); - assertThat(handle.getRawConfig().getPayload().withCompression(CompressionType.UNCOMPRESSED).toString(), is("{}")); // MockConnection returns empty string + // MockConnection returns empty string + assertEquals("{}", getConfig(handle)); + assertEquals(1L, handle.getRawConfig().getGeneration()); assertFalse(sub.nextConfig(false)); assertFalse(handle.isChanged()); + + // Wait some time, config should be the same, but generation should be higher + Thread.sleep(tv.getFixedDelay() * 2); + assertEquals("{}", getConfig(handle)); + assertTrue(handle.getRawConfig().getGeneration() > 1); + assertFalse(sub.nextConfig(false)); + assertFalse(handle.isChanged()); + } + + private String getConfig(GenericConfigHandle handle) { + return handle.getRawConfig().getPayload().withCompression(CompressionType.UNCOMPRESSED).toString(); } @Test public void testGenericRequesterPooling() { ConfigSourceSet source1 = new ConfigSourceSet("tcp/foo:78"); ConfigSourceSet source2 = new ConfigSourceSet("tcp/bar:79"); - JRTConfigRequester req1 = JRTConfigRequester.create(source1, JRTConfigRequesterTest.getTestTimingValues()); - JRTConfigRequester req2 = JRTConfigRequester.create(source2, JRTConfigRequesterTest.getTestTimingValues()); + JRTConfigRequester req1 = JRTConfigRequester.create(source1, tv); + JRTConfigRequester req2 = JRTConfigRequester.create(source2, tv); Map<ConfigSourceSet, JRTConfigRequester> requesters = new LinkedHashMap<>(); requesters.put(source1, req1); requesters.put(source2, req2); diff --git a/configdefinitions/src/vespa/configserver.def b/configdefinitions/src/vespa/configserver.def index 21a2e7064ac..05143bfef9f 100644 --- a/configdefinitions/src/vespa/configserver.def +++ b/configdefinitions/src/vespa/configserver.def @@ -38,6 +38,7 @@ cloud string default="default" environment string default="prod" region string default="default" system string default="main" +zoneDnsSuffixes[] string # RPC protocol maxgetconfigclients int default=1000000 diff --git a/configdefinitions/src/vespa/lb-services.def b/configdefinitions/src/vespa/lb-services.def index e5dcdc7692e..5f33d3120b3 100644 --- a/configdefinitions/src/vespa/lb-services.def +++ b/configdefinitions/src/vespa/lb-services.def @@ -6,17 +6,24 @@ namespace=cloud.config # Active rotation given as flag 'active' for a prod region in deployment.xml # Default true for now (since code in config-model to set it is not ready yet), should have no default value -tenants{}.applications{}.activeRotation bool default=true -tenants{}.applications{}.usePowerOfTwoChoicesLb bool default=true -tenants{}.applications{}.generateNonMtlsEndpoint bool default=true +tenants{}.applications{}.activeRotation bool default=true +tenants{}.applications{}.usePowerOfTwoChoicesLb bool default=true +tenants{}.applications{}.generateNonMtlsEndpoint bool default=true -tenants{}.applications{}.hosts{}.hostname string default="(unknownhostname)" -tenants{}.applications{}.hosts{}.services{}.type string default="(noservicetype)" -tenants{}.applications{}.hosts{}.services{}.clustertype string default="(unknownclustertype)" -tenants{}.applications{}.hosts{}.services{}.clustername string default="(unknownclustername)" -tenants{}.applications{}.hosts{}.services{}.configId string -tenants{}.applications{}.hosts{}.services{}.index int default=0 -tenants{}.applications{}.hosts{}.services{}.ports[].number int default=-1 -tenants{}.applications{}.hosts{}.services{}.ports[].tags string default="(notags)" -tenants{}.applications{}.hosts{}.services{}.servicealiases[] string -tenants{}.applications{}.hosts{}.services{}.endpointaliases[] string +tenants{}.applications{}.hosts{}.hostname string default="(unknownhostname)" +tenants{}.applications{}.hosts{}.services{}.type string default="(noservicetype)" +tenants{}.applications{}.hosts{}.services{}.clustertype string default="(unknownclustertype)" +tenants{}.applications{}.hosts{}.services{}.clustername string default="(unknownclustername)" +tenants{}.applications{}.hosts{}.services{}.configId string +tenants{}.applications{}.hosts{}.services{}.index int default=0 +tenants{}.applications{}.hosts{}.services{}.ports[].number int default=-1 +tenants{}.applications{}.hosts{}.services{}.ports[].tags string default="(notags)" +tenants{}.applications{}.hosts{}.services{}.servicealiases[] string +tenants{}.applications{}.hosts{}.services{}.endpointaliases[] string + + +tenants{}.applications{}.endpoints[].dnsName string +tenants{}.applications{}.endpoints[].scope enum {application, global, zone} +tenants{}.applications{}.endpoints[].routingMethod enum {shared, sharedLayer4} +tenants{}.applications{}.endpoints[].weight int default=1 +tenants{}.applications{}.endpoints[].hosts[] string
\ No newline at end of file diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java index e0629a2e5db..00dc1f4d065 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java @@ -587,11 +587,11 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye return orchestrator.getAllSuspendedApplications().contains(application); } - public HttpResponse filedistributionStatus(ApplicationId applicationId, Duration timeout) { + public HttpResponse fileDistributionStatus(ApplicationId applicationId, Duration timeout) { return fileDistributionStatus.status(getApplication(applicationId), timeout); } - public List<String> deleteUnusedFiledistributionReferences(File fileReferencesPath, + public List<String> deleteUnusedFileDistributionReferences(File fileReferencesPath, Duration keepFileReferencesDuration, int numberToAlwaysKeep) { log.log(Level.FINE, () -> "Keep unused file references for " + keepFileReferencesDuration); @@ -632,8 +632,16 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye return fileReferencesOnDisk .stream() .filter(fileReference -> ! fileReferencesInUse.contains(fileReference)) - .filter(fileReference -> isFileLastModifiedBefore(new File(fileReferencesPath, fileReference), instant)) - .sorted((a, b) -> lastModified(new File(fileReferencesPath, a)).isBefore(lastModified(new File(fileReferencesPath, b))) ? -1 : 1) + .filter(fileReference -> isLastFileAccessBefore(new File(fileReferencesPath, fileReference), instant)) + .sorted((a, b) -> { + if (a.equals(b)) + return 0; + else if (lastAccessed(new File(fileReferencesPath, a)) + .isBefore(lastAccessed(new File(fileReferencesPath, b)))) + return -1; + else + return 1; + }) .collect(Collectors.toList()); } @@ -688,15 +696,15 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye .collect(Collectors.toList()); } - private boolean isFileLastModifiedBefore(File fileReference, Instant instant) { - return lastModified(fileReference).isBefore(instant); + private boolean isLastFileAccessBefore(File fileReference, Instant instant) { + return lastAccessed(fileReference).isBefore(instant); } - private Instant lastModified(File fileReference) { + private Instant lastAccessed(File fileReference) { BasicFileAttributes fileAttributes; try { fileAttributes = readAttributes(fileReference.toPath(), BasicFileAttributes.class); - return fileAttributes.lastModifiedTime().toInstant(); + return fileAttributes.lastAccessTime().toInstant(); } catch (IOException e) { throw new UncheckedIOException(e); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java index c8c0e382ea6..57e49ef3e8d 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java @@ -10,8 +10,11 @@ import com.yahoo.config.provision.Deployment; import com.yahoo.config.provision.TransientException; import com.yahoo.container.handler.VipStatus; import com.yahoo.container.jdisc.state.StateMonitor; +import com.yahoo.vespa.config.server.application.ConfigConvergenceChecker; +import com.yahoo.vespa.config.server.maintenance.ConfigServerMaintenance; import com.yahoo.vespa.config.server.rpc.RpcServer; import com.yahoo.vespa.config.server.version.VersionState; +import com.yahoo.vespa.flags.FlagSource; import com.yahoo.yolean.Exceptions; import java.time.Duration; @@ -69,27 +72,32 @@ public class ConfigServerBootstrap extends AbstractComponent implements Runnable private final Duration sleepTimeWhenRedeployingFails; private final RedeployingApplicationsFails exitIfRedeployingApplicationsFails; private final ExecutorService rpcServerExecutor; + private final ConfigServerMaintenance configServerMaintenance; + @SuppressWarnings("unused") // Injected component @Inject public ConfigServerBootstrap(ApplicationRepository applicationRepository, RpcServer server, - VersionState versionState, StateMonitor stateMonitor, VipStatus vipStatus) { + VersionState versionState, StateMonitor stateMonitor, VipStatus vipStatus, + FlagSource flagSource, ConfigConvergenceChecker convergence) { this(applicationRepository, server, versionState, stateMonitor, vipStatus, BOOTSTRAP_IN_CONSTRUCTOR, EXIT_JVM, applicationRepository.configserverConfig().hostedVespa() ? VipStatusMode.VIP_STATUS_FILE - : VipStatusMode.VIP_STATUS_PROGRAMMATICALLY); + : VipStatusMode.VIP_STATUS_PROGRAMMATICALLY, + flagSource, convergence); } // For testing only ConfigServerBootstrap(ApplicationRepository applicationRepository, RpcServer server, VersionState versionState, - StateMonitor stateMonitor, VipStatus vipStatus, VipStatusMode vipStatusMode) { + StateMonitor stateMonitor, VipStatus vipStatus, VipStatusMode vipStatusMode, + FlagSource flagSource, ConfigConvergenceChecker convergence) { this(applicationRepository, server, versionState, stateMonitor, vipStatus, - FOR_TESTING_NO_BOOTSTRAP_OF_APPS, CONTINUE, vipStatusMode); + FOR_TESTING_NO_BOOTSTRAP_OF_APPS, CONTINUE, vipStatusMode, flagSource, convergence); } private ConfigServerBootstrap(ApplicationRepository applicationRepository, RpcServer server, VersionState versionState, StateMonitor stateMonitor, VipStatus vipStatus, Mode mode, RedeployingApplicationsFails exitIfRedeployingApplicationsFails, - VipStatusMode vipStatusMode) { + VipStatusMode vipStatusMode, FlagSource flagSource, ConfigConvergenceChecker convergence) { this.applicationRepository = applicationRepository; this.server = server; this.versionState = versionState; @@ -101,6 +109,12 @@ public class ConfigServerBootstrap extends AbstractComponent implements Runnable this.exitIfRedeployingApplicationsFails = exitIfRedeployingApplicationsFails; rpcServerExecutor = Executors.newSingleThreadExecutor(new DaemonThreadFactory("config server RPC server")); + configServerMaintenance = new ConfigServerMaintenance(configserverConfig, + applicationRepository, + applicationRepository.tenantRepository().getCurator(), + flagSource, + convergence); + configServerMaintenance.startBeforeBootstrap(); log.log(Level.FINE, () -> "Bootstrap mode: " + mode + ", VIP status mode: " + vipStatusMode); initializing(vipStatusMode); @@ -122,6 +136,7 @@ public class ConfigServerBootstrap extends AbstractComponent implements Runnable server.stop(); log.log(Level.FINE, "RPC server stopped"); rpcServerExecutor.shutdown(); + configServerMaintenance.shutdown(); } @Override @@ -156,11 +171,14 @@ public class ConfigServerBootstrap extends AbstractComponent implements Runnable applicationRepository.bootstrappingDone(); allowConfigRpcRequests(server); up(); + configServerMaintenance.startAfterBootstrap(); } - StateMonitor.Status status() { - return stateMonitor.status(); - } + StateMonitor.Status status() { return stateMonitor.status(); } + + VipStatus vipStatus() { return vipStatus; } + + public ConfigServerMaintenance configServerMaintenance() { return configServerMaintenance; } private void up() { vipStatus.setInRotation(true); @@ -193,7 +211,7 @@ public class ConfigServerBootstrap extends AbstractComponent implements Runnable } private void allowConfigRpcRequests(RpcServer rpcServer) { - log.log(Level.INFO, "Allowing RPC config requests"); + log.log(Level.FINE, "Allowing RPC config requests"); rpcServer.setUpGetConfigHandlers(); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelManager.java b/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelManager.java index e9be9c4e97f..c63ded1aea1 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelManager.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelManager.java @@ -121,7 +121,7 @@ public class SuperModelManager implements SuperModelProvider { public void markAsComplete() { // Invoked on component graph bootstrap (even before ConfigServerBootstrap), // there is no need to bump generation counter. - logger.log(Level.INFO, "Super model is complete"); + logger.log(Level.FINE, "Super model is complete"); SuperModel newSuperModel = getSuperModel().cloneAsComplete(); superModelConfigProvider = new SuperModelConfigProvider(newSuperModel, zone, flagSource); listeners.forEach(listener -> listener.notifyOfCompleteness(newSuperModel)); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java index a824433ae69..0b55c4bb53e 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java @@ -332,6 +332,7 @@ public class ModelContextImpl implements ModelContext { private final boolean allowDisableMtls; private final List<X509Certificate> operatorCertificates; private final List<String> tlsCiphersOverride; + private final List<String> zoneDnsSuffixes; public Properties(ApplicationId applicationId, ConfigserverConfig configserverConfig, @@ -370,6 +371,7 @@ public class ModelContextImpl implements ModelContext { this.operatorCertificates = operatorCertificates; this.tlsCiphersOverride = PermanentFlags.TLS_CIPHERS_OVERRIDE.bindTo(flagSource) .with(FetchVector.Dimension.APPLICATION_ID, applicationId.serializedForm()).value(); + this.zoneDnsSuffixes = configserverConfig.zoneDnsSuffixes(); } @Override public ModelContext.FeatureFlags featureFlags() { return featureFlags; } @@ -440,6 +442,11 @@ public class ModelContextImpl implements ModelContext { @Override public List<String> tlsCiphersOverride() { return tlsCiphersOverride; } + @Override + public List<String> zoneDnsSuffixes() { + return zoneDnsSuffixes; + } + public String flagValueForClusterType(StringFlag flag, Optional<ClusterSpec.Type> clusterType) { return clusterType.map(type -> flag.with(CLUSTER_TYPE, type.name())) .orElse(flag) diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java index bdb48cc0160..e581a1edc21 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java @@ -136,7 +136,7 @@ public class ApplicationHandler extends HttpHandler { } private HttpResponse filedistributionStatus(ApplicationId applicationId, HttpRequest request) { - return applicationRepository.filedistributionStatus(applicationId, getTimeoutFromRequest(request)); + return applicationRepository.fileDistributionStatus(applicationId, getTimeoutFromRequest(request)); } private HttpResponse logs(ApplicationId applicationId, HttpRequest request) { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ApplicationPackageMaintainer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ApplicationPackageMaintainer.java index 7147743a086..d4d4a7fa7d3 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ApplicationPackageMaintainer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ApplicationPackageMaintainer.java @@ -20,12 +20,6 @@ import com.yahoo.vespa.flags.FlagSource; import java.io.File; import java.time.Duration; -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.logging.Level; import java.util.logging.Logger; import static com.yahoo.vespa.config.server.filedistribution.FileDistributionUtil.fileReferenceExistsOnDisk; @@ -62,51 +56,38 @@ public class ApplicationPackageMaintainer extends ConfigServerMaintainer { protected double maintain() { if (getOtherConfigServersInCluster(configserverConfig).isEmpty()) return 1.0; // Nothing to do - final AtomicInteger attempts = new AtomicInteger(0); - final AtomicInteger failures = new AtomicInteger(0); + int attempts = 0; + int failures = 0; - List<CompletableFuture<Void>> futures = new ArrayList<>(); try (var fileDownloader = createFileDownloader()) { for (var applicationId : applicationRepository.listApplications()) { log.fine(() -> "Verifying application package for " + applicationId); Session session = applicationRepository.getActiveSession(applicationId); - if (session == null) continue; // App might be deleted after call to listApplications() + if (session == null) continue; // App might be deleted after call to listApplications() or not activated yet (bootstrap phase) FileReference applicationPackage = session.getApplicationPackageReference(); - if (applicationPackage == null) continue; - - if ( ! fileReferenceExistsOnDisk(downloadDirectory, applicationPackage)) { - long sessionId = session.getSessionId(); - log.fine(() -> "Downloading application package for " + applicationId + - " application package reference " + applicationPackage + - " (session " + sessionId + ")"); - - FileReferenceDownload download = new FileReferenceDownload(applicationPackage, - false, - this.getClass().getSimpleName()); - futures.add(CompletableFuture.supplyAsync(() -> fileDownloader.getFile(download)) - .thenAccept(file -> { - if (file.isPresent()) { - attempts.incrementAndGet(); - createLocalSessionIfMissing(applicationId, sessionId); - } else { - failures.incrementAndGet(); - log.warning("Failed to download application package for application " + - applicationId + " (session " + sessionId + ")"); - } - })); + long sessionId = session.getSessionId(); + log.fine(() -> "Verifying application package file reference " + applicationPackage + " for session " + sessionId); + + if (applicationPackage != null) { + attempts++; + if (! fileReferenceExistsOnDisk(downloadDirectory, applicationPackage)) { + log.fine(() -> "Downloading missing application package for application " + applicationId + " (session " + sessionId + ")"); + + FileReferenceDownload download = new FileReferenceDownload(applicationPackage, + false, + this.getClass().getSimpleName()); + if (fileDownloader.getFile(download).isEmpty()) { + failures++; + log.warning("Failed to download application package for application " + applicationId + " (session " + sessionId + ")"); + continue; + } + } + createLocalSessionIfMissing(applicationId, sessionId); } } } - log.fine(() -> "Attempts: " + attempts.get() + ", failures: " + failures.get()); - futures.forEach(future -> { - try { - future.get(); - } catch (InterruptedException | ExecutionException e) { - log.log(Level.WARNING, "Failed to get future", e); - } - }); - return asSuccessFactor(attempts.get(), failures.get()); + return asSuccessFactor(attempts, failures); } private FileDownloader createFileDownloader() { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ConfigServerMaintenance.java b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ConfigServerMaintenance.java index be19e53115f..a6516ac361b 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ConfigServerMaintenance.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ConfigServerMaintenance.java @@ -1,9 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.maintenance; -import com.google.inject.Inject; import com.yahoo.cloud.config.ConfigserverConfig; -import com.yahoo.component.AbstractComponent; import com.yahoo.concurrent.maintenance.Maintainer; import com.yahoo.vespa.config.server.ApplicationRepository; import com.yahoo.vespa.config.server.ConfigServerBootstrap; @@ -24,31 +22,48 @@ import java.util.concurrent.CopyOnWriteArrayList; * * @author hmusum */ -public class ConfigServerMaintenance extends AbstractComponent { +public class ConfigServerMaintenance { private final List<Maintainer> maintainers = new CopyOnWriteArrayList<>(); + private final ConfigserverConfig configserverConfig; + private final ApplicationRepository applicationRepository; + private final Curator curator; + private final FlagSource flagSource; + private final ConfigConvergenceChecker convergenceChecker; - @Inject - public ConfigServerMaintenance(ConfigServerBootstrap configServerBootstrap, - ConfigserverConfig configserverConfig, + public ConfigServerMaintenance(ConfigserverConfig configserverConfig, ApplicationRepository applicationRepository, Curator curator, FlagSource flagSource, - ConfigConvergenceChecker convergence) { - DefaultTimes defaults = new DefaultTimes(configserverConfig); - maintainers.add(new TenantsMaintainer(applicationRepository, curator, flagSource, defaults.defaultInterval, Clock.systemUTC())); - maintainers.add(new FileDistributionMaintainer(applicationRepository, curator, defaults.defaultInterval, flagSource)); - maintainers.add(new SessionsMaintainer(applicationRepository, curator, Duration.ofSeconds(30), flagSource)); + ConfigConvergenceChecker convergenceChecker) { + this.configserverConfig = configserverConfig; + this.applicationRepository = applicationRepository; + this.curator = curator; + this.flagSource = flagSource; + this.convergenceChecker = convergenceChecker; + } + + public void startBeforeBootstrap() { maintainers.add(new ApplicationPackageMaintainer(applicationRepository, curator, Duration.ofSeconds(30), flagSource)); - maintainers.add(new ReindexingMaintainer(applicationRepository, curator, flagSource, Duration.ofMinutes(3), convergence, Clock.systemUTC())); + maintainers.add(new TenantsMaintainer(applicationRepository, curator, flagSource, + new DefaultTimes(configserverConfig).defaultInterval, Clock.systemUTC())); } - @Override - public void deconstruct() { + public void startAfterBootstrap() { + maintainers.add(new FileDistributionMaintainer(applicationRepository, curator, + new DefaultTimes(configserverConfig).defaultInterval, flagSource)); + maintainers.add(new SessionsMaintainer(applicationRepository, curator, Duration.ofSeconds(30), flagSource)); + maintainers.add(new ReindexingMaintainer(applicationRepository, curator, flagSource, + Duration.ofMinutes(3), convergenceChecker, Clock.systemUTC())); + } + + public void shutdown() { maintainers.forEach(Maintainer::shutdown); maintainers.forEach(Maintainer::awaitShutdown); } + public List<Maintainer> maintainers() { return List.copyOf(maintainers); } + /* * Default values from config. If one of the values needs to be changed, add the value to * configserver-config.xml in the config server application directory and restart the config server diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/FileDistributionMaintainer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/FileDistributionMaintainer.java index 277e6acd6e6..f6aee416c9c 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/FileDistributionMaintainer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/FileDistributionMaintainer.java @@ -20,7 +20,7 @@ import java.time.Duration; */ public class FileDistributionMaintainer extends ConfigServerMaintainer { - private static final int numberToAlwaysKeep = 10; + private static final int numberToAlwaysKeep = 20; private final ApplicationRepository applicationRepository; private final File fileReferencesDir; @@ -39,7 +39,7 @@ public class FileDistributionMaintainer extends ConfigServerMaintainer { @Override protected double maintain() { - applicationRepository.deleteUnusedFiledistributionReferences(fileReferencesDir, maxUnusedFileReferenceAge, numberToAlwaysKeep); + applicationRepository.deleteUnusedFileDistributionReferences(fileReferencesDir, maxUnusedFileReferenceAge, numberToAlwaysKeep); return 1.0; } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/model/LbServicesProducer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/model/LbServicesProducer.java index abd04f62ac2..7ef65bda46d 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/model/LbServicesProducer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/model/LbServicesProducer.java @@ -3,6 +3,8 @@ package com.yahoo.vespa.config.server.model; import com.google.common.base.Joiner; import com.yahoo.cloud.config.LbServicesConfig; +import com.yahoo.config.model.api.ApplicationClusterEndpoint; +import com.yahoo.config.model.api.ApplicationClusterInfo; import com.yahoo.config.model.api.ApplicationInfo; import com.yahoo.config.model.api.HostInfo; import com.yahoo.config.model.api.ServiceInfo; @@ -71,15 +73,41 @@ public class LbServicesProducer implements LbServicesConfig.Producer { private LbServicesConfig.Tenants.Applications.Builder getAppConfig(ApplicationInfo app) { LbServicesConfig.Tenants.Applications.Builder ab = new LbServicesConfig.Tenants.Applications.Builder(); + + // TODO: read active rotation from ApplicationClusterInfo ab.activeRotation(getActiveRotation(app)); ab.usePowerOfTwoChoicesLb(true); ab.generateNonMtlsEndpoint(generateNonMtlsEndpoint(app)); + + // TODO: Remove when endpoints-config is read by all load balancers app.getModel().getHosts().stream() .sorted((a, b) -> a.getHostname().compareTo(b.getHostname())) .forEach(hostInfo -> ab.hosts(hostInfo.getHostname(), getHostsConfig(hostInfo))); + + Set<ApplicationClusterInfo> applicationClusterInfos = app.getModel().applicationClusterInfo(); + List<LbServicesConfig.Tenants.Applications.Endpoints.Builder> endpointBuilder = applicationClusterInfos.stream() + .map(ApplicationClusterInfo::endpoints) + .flatMap(endpoints -> getEndpointConfig(endpoints).stream()) + .collect(Collectors.toList()); + ab.endpoints(endpointBuilder); return ab; } + private List<LbServicesConfig.Tenants.Applications.Endpoints.Builder> getEndpointConfig(List<ApplicationClusterEndpoint> clusterEndpoints) { + return clusterEndpoints.stream() + .map(this::getEndpointConfig) + .collect(Collectors.toList()); + } + + private LbServicesConfig.Tenants.Applications.Endpoints.Builder getEndpointConfig(ApplicationClusterEndpoint clusterEndpoints) { + LbServicesConfig.Tenants.Applications.Endpoints.Builder builder = new LbServicesConfig.Tenants.Applications.Endpoints.Builder(); + return builder.dnsName(clusterEndpoints.dnsName().value()) + .scope(LbServicesConfig.Tenants.Applications.Endpoints.Scope.Enum.valueOf(clusterEndpoints.scope().name())) + .routingMethod(LbServicesConfig.Tenants.Applications.Endpoints.RoutingMethod.Enum.valueOf(clusterEndpoints.routingMethod().name())) + .weight(clusterEndpoints.weight()) + .hosts(clusterEndpoints.hostNames()); + } + private boolean getActiveRotation(ApplicationInfo app) { boolean activeRotation = false; for (HostInfo hostInfo : app.getModel().getHosts()) { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/ContainerEndpointSerializer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/ContainerEndpointSerializer.java index 1b8842ba746..55986e71b3d 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/ContainerEndpointSerializer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/ContainerEndpointSerializer.java @@ -1,11 +1,13 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.tenant; +import com.yahoo.config.model.api.ApplicationClusterEndpoint; import com.yahoo.config.model.api.ContainerEndpoint; import com.yahoo.slime.ArrayTraverser; import com.yahoo.slime.Cursor; import com.yahoo.slime.Inspector; import com.yahoo.slime.Slime; +import com.yahoo.slime.SlimeUtils; import java.util.ArrayList; import java.util.List; @@ -26,18 +28,26 @@ public class ContainerEndpointSerializer { // - CHANGING THE FORMAT OF A FIELD: Don't do it bro. private static final String clusterIdField = "clusterId"; + private static final String scopeField = "scope"; private static final String namesField = "names"; private ContainerEndpointSerializer() {} public static ContainerEndpoint endpointFromSlime(Inspector inspector) { final var clusterId = inspector.field(clusterIdField).asString(); + // Currently assigned endpoints that do not have scope should be interpreted as global endpoints + // TODO: Remove default assignment after 7.500 + final var scope = SlimeUtils.optionalString(inspector.field(scopeField)).orElse(ApplicationClusterEndpoint.Scope.global.name()); final var namesInspector = inspector.field(namesField); if (clusterId.isEmpty()) { throw new IllegalStateException("'clusterId' missing on serialized ContainerEndpoint"); } + if (scope.isEmpty()) { + throw new IllegalStateException("'scope' missing on serialized ContainerEndpoint"); + } + if (! namesInspector.valid()) { throw new IllegalStateException("'names' missing on serialized ContainerEndpoint"); } @@ -49,7 +59,7 @@ public class ContainerEndpointSerializer { names.add(containerName); }); - return new ContainerEndpoint(clusterId, names); + return new ContainerEndpoint(clusterId, ApplicationClusterEndpoint.Scope.valueOf(scope), names); } public static List<ContainerEndpoint> endpointListFromSlime(Slime slime) { @@ -70,6 +80,7 @@ public class ContainerEndpointSerializer { public static void endpointToSlime(Cursor cursor, ContainerEndpoint endpoint) { cursor.setString(clusterIdField, endpoint.clusterId()); + cursor.setString(scopeField, endpoint.scope().name()); final var namesInspector = cursor.setArray(namesField); endpoint.names().forEach(namesInspector::addString); diff --git a/configserver/src/main/resources/configserver-app/services.xml b/configserver/src/main/resources/configserver-app/services.xml index ca6ed4cff28..729b50419d7 100644 --- a/configserver/src/main/resources/configserver-app/services.xml +++ b/configserver/src/main/resources/configserver-app/services.xml @@ -35,7 +35,6 @@ <component id="com.yahoo.vespa.config.server.application.ConfigConvergenceChecker" bundle="configserver" /> <component id="com.yahoo.vespa.config.server.application.HttpProxy" bundle="configserver" /> <component id="com.yahoo.vespa.config.server.filedistribution.FileServer" bundle="configserver" /> - <component id="com.yahoo.vespa.config.server.maintenance.ConfigServerMaintenance" bundle="configserver" /> <component id="com.yahoo.vespa.config.server.rpc.RpcRequestHandlerProvider" bundle="configserver" /> <component id="com.yahoo.vespa.config.server.rpc.security.DummyNodeIdentifierProvider" bundle="configserver" /> <component id="com.yahoo.vespa.config.server.rpc.security.DefaultRpcAuthorizerProvider" bundle="configserver" /> diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java index 133570cc109..ce926016bd4 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java @@ -269,23 +269,23 @@ public class ApplicationRepositoryTest { } @Test - public void deleteUnusedFileReferences() throws IOException { + public void deleteUnusedFileReferences() throws IOException, InterruptedException { File fileReferencesDir = temporaryFolder.newFolder(); - Duration keepFileReferences = Duration.ofHours(48); + Duration keepFileReferencesDuration = Duration.ofSeconds(4); - // Add file reference that is not in use and should be deleted (older than 'keepFileReferences') + // Add file reference that is not in use and should be deleted (older than 'keepFileReferencesDuration') + File filereferenceDirOldest = createFilereferenceOnDisk(new File(fileReferencesDir, "foo")); + //Thread.sleep(Duration.ofSeconds(1).toMillis()); - Instant now = Instant.now(); - File filereferenceDirOldest = createFilereferenceOnDisk(new File(fileReferencesDir, "foo"), - now.minus(keepFileReferences.plus(Duration.ofHours(2)))); + // Add file references that are not in use and could be deleted + IntStream.range(0, 3).forEach(i -> { + createFilereferenceOnDisk(new File(fileReferencesDir, "bar" + i)); + try { Thread.sleep(Duration.ofSeconds(1).toMillis()); } catch (InterruptedException e) { /* ignore */ } + }); + Thread.sleep(keepFileReferencesDuration.toMillis()); - // Add file references that are not in use and some of them should be deleted (all are older than 'keepFileReferences') - IntStream.range(0, 6) - .forEach(i -> createFilereferenceOnDisk(new File(fileReferencesDir, "bar" + i), - now.minus(keepFileReferences.plus(Duration.ofHours(1).minus(Duration.ofMinutes(i)))))); - - // Add file reference that is not in use, but should not be deleted (newer than 'keepFileReferences') - File filereferenceDirNewest = createFilereferenceOnDisk(new File(fileReferencesDir, "baz"), now); + // Add file reference that is not in use, but should not be deleted (newer than 'keepFileReferencesDuration') + File filereferenceDirNewest = createFilereferenceOnDisk(new File(fileReferencesDir, "baz")); applicationRepository = new ApplicationRepository.Builder() .withTenantRepository(tenantRepository) @@ -298,22 +298,21 @@ public class ApplicationRepositoryTest { PrepareParams prepareParams = new PrepareParams.Builder().applicationId(applicationId()).ignoreValidationErrors(true).build(); deployApp(new File("src/test/apps/app"), prepareParams); - List<String> toBeDeleted = applicationRepository.deleteUnusedFiledistributionReferences(fileReferencesDir, - keepFileReferences, - 5); + List<String> toBeDeleted = applicationRepository.deleteUnusedFileDistributionReferences(fileReferencesDir, + keepFileReferencesDuration, + 2); Collections.sort(toBeDeleted); assertEquals(List.of("bar0", "foo"), toBeDeleted); - // bar0 and foo are the only ones that will be deleted (keeps 5 newest no matter how old they are) + // bar0 and foo are the only ones that will be deleted (keeps 2 newest no matter how old they are) assertFalse(filereferenceDirOldest.exists()); assertFalse(new File(fileReferencesDir, "bar0").exists()); assertTrue(filereferenceDirNewest.exists()); } - private File createFilereferenceOnDisk(File filereferenceDir, Instant lastModifiedTime) { + private File createFilereferenceOnDisk(File filereferenceDir) { assertTrue(filereferenceDir.mkdir()); File bar = new File(filereferenceDir, "file"); IOUtils.writeFile(bar, Utf8.toBytes("test")); - assertTrue(filereferenceDir.setLastModified(lastModifiedTime.toEpochMilli())); return filereferenceDir; } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/ConfigServerBootstrapTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/ConfigServerBootstrapTest.java index 69b9d67f84c..babc7b79b65 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/ConfigServerBootstrapTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/ConfigServerBootstrapTest.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.config.server; import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.component.Version; +import com.yahoo.concurrent.maintenance.Maintainer; import com.yahoo.config.model.provision.Host; import com.yahoo.config.model.provision.Hosts; import com.yahoo.config.model.provision.InMemoryProvisioner; @@ -18,11 +19,13 @@ import com.yahoo.container.jdisc.state.StateMonitor; import com.yahoo.docproc.jdisc.metric.NullMetric; import com.yahoo.path.Path; import com.yahoo.text.Utf8; +import com.yahoo.vespa.config.server.application.ConfigConvergenceChecker; import com.yahoo.vespa.config.server.deploy.DeployTester; import com.yahoo.vespa.config.server.rpc.RpcServer; import com.yahoo.vespa.config.server.version.VersionState; import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.curator.mock.MockCurator; +import com.yahoo.vespa.flags.InMemoryFlagSource; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; @@ -37,6 +40,7 @@ import java.util.Collections; import java.util.List; import java.util.Optional; import java.util.function.BooleanSupplier; +import java.util.stream.Collectors; import static com.yahoo.vespa.config.server.ConfigServerBootstrap.VipStatusMode.VIP_STATUS_FILE; import static com.yahoo.vespa.config.server.ConfigServerBootstrap.VipStatusMode.VIP_STATUS_PROGRAMMATICALLY; @@ -64,28 +68,31 @@ public class ConfigServerBootstrapTest { .configserverConfig(configserverConfig).hostProvisioner(provisioner).build(); tester.deployApp("src/test/apps/hosted/"); - VersionState versionState = createVersionState(); - assertTrue(versionState.isUpgraded()); - RpcServer rpcServer = createRpcServer(configserverConfig); // Take a host away so that there are too few for the application, to verify we can still bootstrap provisioner.allocations().values().iterator().next().remove(0); - StateMonitor stateMonitor = StateMonitor.createForTesting(); - VipStatus vipStatus = createVipStatus(stateMonitor); - ConfigServerBootstrap bootstrap = new ConfigServerBootstrap(tester.applicationRepository(), rpcServer, versionState, - stateMonitor, vipStatus, VIP_STATUS_PROGRAMMATICALLY); - assertFalse(vipStatus.isInRotation()); + ConfigServerBootstrap bootstrap = createBootstrap(tester, rpcServer, VIP_STATUS_PROGRAMMATICALLY); + assertEquals(List.of("ApplicationPackageMaintainer", "TenantsMaintainer"), + bootstrap.configServerMaintenance().maintainers().stream() + .map(Maintainer::name) + .sorted().collect(Collectors.toList())); + assertFalse(bootstrap.vipStatus().isInRotation()); + bootstrap.start(); waitUntil(rpcServer::isRunning, "failed waiting for Rpc server running"); assertTrue(rpcServer.isServingConfigRequests()); waitUntil(() -> bootstrap.status() == StateMonitor.Status.up, "failed waiting for status 'up'"); - waitUntil(vipStatus::isInRotation, "failed waiting for server to be in rotation"); + waitUntil(() -> bootstrap.vipStatus().isInRotation(), "failed waiting for server to be in rotation"); + assertEquals(List.of("ApplicationPackageMaintainer", "FileDistributionMaintainer", "ReindexingMaintainer", "SessionsMaintainer", "TenantsMaintainer"), + bootstrap.configServerMaintenance().maintainers().stream() + .map(Maintainer::name) + .sorted().collect(Collectors.toList())); bootstrap.deconstruct(); assertEquals(StateMonitor.Status.down, bootstrap.status()); assertFalse(rpcServer.isRunning()); assertTrue(rpcServer.isServingConfigRequests()); - assertFalse(vipStatus.isInRotation()); + assertFalse(bootstrap.vipStatus().isInRotation()); } // Just tests setup, the actual response of accessing /status.html depends on the status @@ -98,21 +105,15 @@ public class ConfigServerBootstrapTest { .configserverConfig(configserverConfig).hostProvisioner(provisioner).build(); tester.deployApp("src/test/apps/hosted/"); - VersionState versionState = createVersionState(); - assertTrue(versionState.isUpgraded()); - RpcServer rpcServer = createRpcServer(configserverConfig); - StateMonitor stateMonitor = StateMonitor.createForTesting(); - VipStatus vipStatus = createVipStatus(stateMonitor); - ConfigServerBootstrap bootstrap = new ConfigServerBootstrap(tester.applicationRepository(), rpcServer, versionState, - stateMonitor, vipStatus, VIP_STATUS_FILE); - assertTrue(vipStatus.isInRotation()); // default is in rotation when using status file + ConfigServerBootstrap bootstrap = createBootstrap(tester, rpcServer, VIP_STATUS_FILE); + assertTrue(bootstrap.vipStatus().isInRotation()); // default is in rotation when using status file bootstrap.start(); waitUntil(rpcServer::isRunning, "failed waiting for Rpc server running"); assertTrue(rpcServer.isServingConfigRequests()); waitUntil(() -> bootstrap.status() == StateMonitor.Status.up, "failed waiting for status 'up'"); - waitUntil(vipStatus::isInRotation, "failed waiting for server to be in rotation"); + waitUntil(() -> bootstrap.vipStatus().isInRotation(), "failed waiting for server to be in rotation"); bootstrap.deconstruct(); } @@ -123,9 +124,6 @@ public class ConfigServerBootstrapTest { .configserverConfig(configserverConfig).build(); tester.deployApp("src/test/apps/hosted/"); - VersionState versionState = createVersionState(); - assertTrue(versionState.isUpgraded()); - // Manipulate application package so that it will fail deployment when config server starts java.nio.file.Files.delete(Paths.get(configserverConfig.configServerDBDir()) .resolve("tenants/") @@ -133,19 +131,16 @@ public class ConfigServerBootstrapTest { .resolve("sessions/2/services.xml")); RpcServer rpcServer = createRpcServer(configserverConfig); - StateMonitor stateMonitor = StateMonitor.createForTesting(); - VipStatus vipStatus = createVipStatus(stateMonitor); - ConfigServerBootstrap bootstrap = new ConfigServerBootstrap(tester.applicationRepository(), rpcServer, versionState, - stateMonitor, vipStatus, VIP_STATUS_PROGRAMMATICALLY); - assertFalse(vipStatus.isInRotation()); + ConfigServerBootstrap bootstrap = createBootstrap(tester, rpcServer, VIP_STATUS_PROGRAMMATICALLY); + assertFalse(bootstrap.vipStatus().isInRotation()); // Call method directly, to be sure that it is finished redeploying all applications and we can check status bootstrap.start(); // App is invalid, bootstrapping was unsuccessful. Status should be 'initializing', // rpc server should not be running and it should be out of rotation - assertEquals(StateMonitor.Status.initializing, stateMonitor.status()); + assertEquals(StateMonitor.Status.initializing, bootstrap.status()); assertTrue(rpcServer.isRunning()); assertFalse(rpcServer.isServingConfigRequests()); - assertFalse(vipStatus.isInRotation()); + assertFalse(bootstrap.vipStatus().isInRotation()); bootstrap.deconstruct(); } @@ -168,24 +163,34 @@ public class ConfigServerBootstrapTest { tester.deployApp("src/test/apps/app/", vespaVersion); ApplicationId applicationId = tester.applicationId(); - VersionState versionState = createVersionState(); - assertTrue(versionState.isUpgraded()); - // Ugly hack, but I see no other way of doing it: // Manipulate application version in zookeeper so that it is an older version than the model we know, which is // the case when upgrading on non-hosted installations curator.set(Path.fromString("/config/v2/tenants/" + applicationId.tenant().value() + "/sessions/2/version"), Utf8.toBytes("1.2.2")); RpcServer rpcServer = createRpcServer(configserverConfig); - StateMonitor stateMonitor = StateMonitor.createForTesting(); - VipStatus vipStatus = createVipStatus(stateMonitor); - ConfigServerBootstrap bootstrap = new ConfigServerBootstrap(tester.applicationRepository(), rpcServer, versionState, - stateMonitor, vipStatus, VIP_STATUS_PROGRAMMATICALLY); + ConfigServerBootstrap bootstrap = createBootstrap(tester, rpcServer, VIP_STATUS_PROGRAMMATICALLY); bootstrap.start(); waitUntil(rpcServer::isRunning, "failed waiting for Rpc server running"); assertTrue(rpcServer.isServingConfigRequests()); waitUntil(() -> bootstrap.status() == StateMonitor.Status.up, "failed waiting for status 'up'"); - waitUntil(vipStatus::isInRotation, "failed waiting for server to be in rotation"); + waitUntil(() -> bootstrap.vipStatus().isInRotation(), "failed waiting for server to be in rotation"); + } + + private ConfigServerBootstrap createBootstrap(DeployTester tester, RpcServer rpcServer, ConfigServerBootstrap.VipStatusMode vipStatusProgrammatically) throws IOException { + VersionState versionState = createVersionState(); + assertTrue(versionState.isUpgraded()); + + StateMonitor stateMonitor = StateMonitor.createForTesting(); + VipStatus vipStatus = createVipStatus(stateMonitor); + return new ConfigServerBootstrap(tester.applicationRepository(), + rpcServer, + versionState, + stateMonitor, + vipStatus, + vipStatusProgrammatically, + new InMemoryFlagSource(), + new ConfigConvergenceChecker()); } private void waitUntil(BooleanSupplier booleanSupplier, String messageIfWaitingFails) throws InterruptedException { diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/ModelContextImplTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/ModelContextImplTest.java index c628a16d5e9..b0046a201ab 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/ModelContextImplTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/ModelContextImplTest.java @@ -5,6 +5,7 @@ import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.component.Version; import com.yahoo.concurrent.InThreadExecutorService; import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.config.model.api.ApplicationClusterEndpoint; import com.yahoo.config.model.api.ContainerEndpoint; import com.yahoo.config.model.api.HostProvisioner; import com.yahoo.config.model.api.ModelContext; @@ -41,7 +42,7 @@ public class ModelContextImplTest { @Test public void testModelContextTest() { - ContainerEndpoint endpoint = new ContainerEndpoint("foo", List.of("a", "b")); + ContainerEndpoint endpoint = new ContainerEndpoint("foo", ApplicationClusterEndpoint.Scope.global, List.of("a", "b")); Set<ContainerEndpoint> endpoints = Collections.singleton(endpoint); InMemoryFlagSource flagSource = new InMemoryFlagSource(); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/model/LbServicesProducerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/model/LbServicesProducerTest.java index d4f7b610360..2163d8738ab 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/model/LbServicesProducerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/model/LbServicesProducerTest.java @@ -2,8 +2,10 @@ package com.yahoo.vespa.config.server.model; import com.yahoo.cloud.config.LbServicesConfig; +import com.yahoo.cloud.config.LbServicesConfig.Tenants.Applications.Endpoints; import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.model.NullConfigModelRegistry; +import com.yahoo.config.model.api.ApplicationClusterEndpoint; import com.yahoo.config.model.api.ApplicationInfo; import com.yahoo.config.model.api.ContainerEndpoint; import com.yahoo.config.model.api.Model; @@ -35,16 +37,24 @@ import java.util.List; import java.util.Map; import java.util.Random; import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import static com.yahoo.cloud.config.LbServicesConfig.Tenants.Applications.Endpoints.RoutingMethod.Enum.sharedLayer4; +import static com.yahoo.cloud.config.LbServicesConfig.Tenants.Applications.Endpoints.Scope.Enum.application; +import static com.yahoo.cloud.config.LbServicesConfig.Tenants.Applications.Endpoints.Scope.Enum.global; +import static com.yahoo.cloud.config.LbServicesConfig.Tenants.Applications.Endpoints.Scope.Enum.zone; import static com.yahoo.config.model.api.container.ContainerServiceType.QRSERVER; import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.is; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; import static org.junit.Assume.assumeFalse; +import static org.junit.Assume.assumeTrue; /** * @author Ulf Lilleengen @@ -55,8 +65,11 @@ public class LbServicesProducerTest { private static final String rotation1 = "rotation-1"; private static final String rotation2 = "rotation-2"; private static final Set<ContainerEndpoint> endpoints = Set.of( - new ContainerEndpoint("mydisc", List.of("rotation-1", "rotation-2")) + new ContainerEndpoint("mydisc", ApplicationClusterEndpoint.Scope.global, List.of("rotation-1", "rotation-2")), + new ContainerEndpoint("mydisc", ApplicationClusterEndpoint.Scope.application, List.of("app-endpoint")) ); + private static final List<String> zoneDnsSuffixes = List.of(".endpoint1.suffix", ".endpoint2.suffix"); + private final InMemoryFlagSource flagSource = new InMemoryFlagSource(); private final boolean useGlobalServiceId; @@ -145,8 +158,8 @@ public class LbServicesProducerTest { .endpoints(endpoints) .properties(new TestProperties().setHostedVespa(true))); RegionName regionName = RegionName.from("us-east-1"); - - var services = getLbServicesConfig(new Zone(Environment.prod, regionName), testModel) + LbServicesConfig config = getLbServicesConfig(new Zone(Environment.prod, regionName), testModel); + LbServicesConfig.Tenants.Applications.Hosts.Services services = config .tenants("foo") .applications("foo:prod:" + regionName.value() + ":default") .hosts("foo.foo.yahoo.com") @@ -154,6 +167,25 @@ public class LbServicesProducerTest { assertThat(services.servicealiases(), contains("service1")); assertThat("Missing endpoints in list: " + services.endpointaliases(), services.endpointaliases(), containsInAnyOrder("foo1.bar1.com", "foo2.bar2.com", rotation1, rotation2)); + + List<Endpoints> endpointList = config.tenants("foo").applications("foo:prod:" + regionName.value() + ":default").endpoints(); + // Expect 4 zone endpoints (2 suffixes), 2 global endpoints and 1 application endpoint + assertEquals(7, endpointList.size()); + List<Endpoints> zoneEndpoints = endpointList.stream().filter(e -> e.scope() == zone).collect(Collectors.toList()); + assertEquals(4, zoneEndpoints.size()); + assertThat(zoneEndpoints.stream() + .filter(e -> e.routingMethod() == sharedLayer4) + .map(Endpoints::dnsName).collect(Collectors.toList()), + containsInAnyOrder("mydisc.foo.foo.endpoint1.suffix", "mydisc.foo.foo.endpoint2.suffix")); + + List<Endpoints> globalEndpoints = endpointList.stream().filter(e -> e.scope() == global).collect(Collectors.toList()); + assertEquals(2, globalEndpoints.size()); + assertThat(globalEndpoints.stream().map(Endpoints::dnsName).collect(Collectors.toList()), containsInAnyOrder("rotation-1", "rotation-2")); + + List<Endpoints> applicationEndpoints = endpointList.stream().filter(e -> e.scope() == application).collect(Collectors.toList()); + assertEquals(1, applicationEndpoints.size()); + assertThat(applicationEndpoints.stream().map(Endpoints::dnsName).collect(Collectors.toList()), containsInAnyOrder("app-endpoint")); + } @@ -184,8 +216,6 @@ public class LbServicesProducerTest { } private Map<TenantName, Set<ApplicationInfo>> createTestModel(DeployState.Builder deployStateBuilder) { - deployStateBuilder.properties(new TestProperties().setHostedVespa(true)); - Map<TenantName, Set<ApplicationInfo>> tMap = new LinkedHashMap<>(); TenantName foo = TenantName.from("foo"); TenantName bar = TenantName.from("bar"); @@ -207,6 +237,7 @@ public class LbServicesProducerTest { Set<ApplicationInfo> applicationInfoSet = new HashSet<>(); List<String> hostnames = new ArrayList<>(); appIds.forEach(appId -> { + deployStateBuilder.properties(getTestproperties(appId)); hostnames.add(appId.tenant() + "." + appId.application() + ".yahoo.com"); hostnames.add(appId.tenant().value() + "." + appId.application().value() + "2.yahoo.com"); try { @@ -275,4 +306,10 @@ public class LbServicesProducerTest { assertThat(ConfigPayload.fromInstance(expected).toString(true), is(ConfigPayload.fromInstance(actual).toString(true))); } + private TestProperties getTestproperties(ApplicationId applicationId) { + return new TestProperties() + .setHostedVespa(true) + .setZoneDnsSuffixes(zoneDnsSuffixes) + .setApplicationId(applicationId); + } } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/session/PrepareParamsTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/session/PrepareParamsTest.java index 366f936c318..2c97d0b9382 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/session/PrepareParamsTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/session/PrepareParamsTest.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.session; +import com.yahoo.config.model.api.ApplicationClusterEndpoint; import com.yahoo.config.model.api.ApplicationRoles; import com.yahoo.config.model.api.ContainerEndpoint; import com.yahoo.config.model.api.EndpointCertificateMetadata; @@ -87,10 +88,10 @@ public class PrepareParamsTest { @Test public void testCorrectParsingWithContainerEndpoints() throws IOException { - var endpoints = List.of(new ContainerEndpoint("qrs1", + var endpoints = List.of(new ContainerEndpoint("qrs1", ApplicationClusterEndpoint.Scope.global, List.of("c1.example.com", "c2.example.com")), - new ContainerEndpoint("qrs2", + new ContainerEndpoint("qrs2",ApplicationClusterEndpoint.Scope.global, List.of("c3.example.com", "c4.example.com"))); var param = "[\n" + diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java index 8c172a04d36..08e6a353fbb 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java @@ -6,6 +6,7 @@ import com.yahoo.component.Version; import com.yahoo.concurrent.InThreadExecutorService; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.application.api.FileRegistry; +import com.yahoo.config.model.api.ApplicationClusterEndpoint; import com.yahoo.config.model.api.ContainerEndpoint; import com.yahoo.config.model.api.EndpointCertificateSecrets; import com.yahoo.config.model.application.provider.BaseDeployLogger; @@ -264,9 +265,11 @@ public class SessionPreparerTest { prepare(new File("src/test/resources/deploy/hosted-app"), params); var expected = List.of(new ContainerEndpoint("foo", + ApplicationClusterEndpoint.Scope.global, List.of("foo.app1.tenant1.global.vespa.example.com", "rotation-042.vespa.global.routing")), new ContainerEndpoint("bar", + ApplicationClusterEndpoint.Scope.global, List.of("bar.app1.tenant1.global.vespa.example.com", "rotation-043.vespa.global.routing"))); assertEquals(expected, readContainerEndpoints(applicationId)); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/ContainerEndpointSerializerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/ContainerEndpointSerializerTest.java index 6f8c1759538..2d767cfded4 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/ContainerEndpointSerializerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/ContainerEndpointSerializerTest.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.tenant; +import com.yahoo.config.model.api.ApplicationClusterEndpoint; import com.yahoo.config.model.api.ContainerEndpoint; import com.yahoo.slime.Slime; import org.junit.Test; @@ -20,18 +21,36 @@ public class ContainerEndpointSerializerTest { final var entry = slime.setObject(); entry.setString("clusterId", "foobar"); + entry.setString("scope", "application"); final var entryNames = entry.setArray("names"); entryNames.addString("a"); entryNames.addString("b"); final var endpoint = ContainerEndpointSerializer.endpointFromSlime(slime.get()); assertEquals("foobar", endpoint.clusterId()); + assertEquals(ApplicationClusterEndpoint.Scope.application, endpoint.scope()); + assertEquals(List.of("a", "b"), endpoint.names()); + } + + @Test + public void readEndpointWithoutScope() { + final var slime = new Slime(); + final var entry = slime.setObject(); + + entry.setString("clusterId", "foobar"); + final var entryNames = entry.setArray("names"); + entryNames.addString("a"); + entryNames.addString("b"); + + final var endpoint = ContainerEndpointSerializer.endpointFromSlime(slime.get()); + assertEquals("foobar", endpoint.clusterId()); + assertEquals(ApplicationClusterEndpoint.Scope.global, endpoint.scope()); assertEquals(List.of("a", "b"), endpoint.names()); } @Test public void writeReadSingleEndpoint() { - final var endpoint = new ContainerEndpoint("foo", List.of("a", "b")); + final var endpoint = new ContainerEndpoint("foo", ApplicationClusterEndpoint.Scope.global, List.of("a", "b")); final var serialized = new Slime(); ContainerEndpointSerializer.endpointToSlime(serialized.setObject(), endpoint); final var deserialized = ContainerEndpointSerializer.endpointFromSlime(serialized.get()); @@ -41,7 +60,7 @@ public class ContainerEndpointSerializerTest { @Test public void writeReadEndpoints() { - final var endpoints = List.of(new ContainerEndpoint("foo", List.of("a", "b"))); + final var endpoints = List.of(new ContainerEndpoint("foo", ApplicationClusterEndpoint.Scope.global, List.of("a", "b"))); final var serialized = ContainerEndpointSerializer.endpointListToSlime(endpoints); final var deserialized = ContainerEndpointSerializer.endpointListFromSlime(serialized); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/ContainerEndpointsCacheTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/ContainerEndpointsCacheTest.java index 99360e7c0bf..6c78eb85ee6 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/ContainerEndpointsCacheTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/ContainerEndpointsCacheTest.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.tenant; +import com.yahoo.config.model.api.ApplicationClusterEndpoint; import com.yahoo.config.model.api.ContainerEndpoint; import com.yahoo.config.provision.ApplicationId; import com.yahoo.path.Path; @@ -17,7 +18,7 @@ public class ContainerEndpointsCacheTest { public void readWriteFromCache() { final var cache = new ContainerEndpointsCache(Path.createRoot(), new MockCurator()); final var endpoints = List.of( - new ContainerEndpoint("the-cluster-1", List.of("a", "b", "c")) + new ContainerEndpoint("the-cluster-1", ApplicationClusterEndpoint.Scope.global, List.of("a", "b", "c")) ); cache.write(ApplicationId.defaultId(), endpoints); diff --git a/container-core/src/main/java/com/yahoo/container/core/config/HandlersConfigurerDi.java b/container-core/src/main/java/com/yahoo/container/core/config/HandlersConfigurerDi.java index 8095e07cc9e..173695c5299 100644 --- a/container-core/src/main/java/com/yahoo/container/core/config/HandlersConfigurerDi.java +++ b/container-core/src/main/java/com/yahoo/container/core/config/HandlersConfigurerDi.java @@ -113,6 +113,7 @@ public class HandlersConfigurerDi { isInitializing); } + @SuppressWarnings("deprecation") private Injector createFallbackInjector(com.yahoo.container.Container vespaContainer, Injector discInjector) { return discInjector.createChildInjector(new AbstractModule() { @Override diff --git a/container-core/src/main/java/com/yahoo/container/di/CloudSubscriber.java b/container-core/src/main/java/com/yahoo/container/di/CloudSubscriber.java index f252287b403..0247cda8bbd 100644 --- a/container-core/src/main/java/com/yahoo/container/di/CloudSubscriber.java +++ b/container-core/src/main/java/com/yahoo/container/di/CloudSubscriber.java @@ -51,7 +51,14 @@ public class CloudSubscriber implements Subscriber { @Override public Map<ConfigKey<ConfigInstance>, ConfigInstance> config() { Map<ConfigKey<ConfigInstance>, ConfigInstance> ret = new HashMap<>(); - handles.forEach((k, v) -> ret.put(k, v.getConfig())); + handles.forEach((k, v) -> { + ConfigInstance config = v.getConfig(); + if (config == null) { + throw new IllegalArgumentException("Got a null config from the config system for key: " + k + + "\nConfig handle: " + v); + } + ret.put(k, config); + }); return ret; } @@ -60,11 +67,6 @@ public class CloudSubscriber implements Subscriber { if (handles.isEmpty()) throw new IllegalStateException("No config keys registered"); - // Catch and just log config exceptions due to missing config values for parameters that do - // not have a default value. These exceptions occur when the user has removed a component - // from services.xml, and the component takes a config that has parameters without a - // default value in the def-file. There is a new 'components' config underway, where the - // component is removed, so this old config generation will soon be replaced by a new one. boolean gotNextGen = false; while ( ! gotNextGen) { try { diff --git a/container-core/src/main/java/com/yahoo/container/di/componentgraph/core/ComponentNode.java b/container-core/src/main/java/com/yahoo/container/di/componentgraph/core/ComponentNode.java index c63ad991ab9..bf7366cc60f 100644 --- a/container-core/src/main/java/com/yahoo/container/di/componentgraph/core/ComponentNode.java +++ b/container-core/src/main/java/com/yahoo/container/di/componentgraph/core/ComponentNode.java @@ -142,8 +142,8 @@ public class ComponentNode extends Node { for (Object ob : arguments) { if (ob instanceof Node) { actualArguments.add(((Node) ob).component()); - } else if (ob instanceof ConfigKey) { - actualArguments.add(availableConfigs.get(ob)); + } else if (ob instanceof ConfigKey<?>) { + actualArguments.add(getConfigInstance((ConfigKey<?>)ob)); } else { actualArguments.add(ob); } @@ -214,8 +214,8 @@ public class ComponentNode extends Node { } List<ConfigInstance> ret = new ArrayList<>(); for (Object arg : arguments) { - if (arg instanceof ConfigKey) { - ret.add(availableConfigs.get(arg)); + if (arg instanceof ConfigKey<?>) { + ret.add(getConfigInstance((ConfigKey<?>)arg)); } } return ret; @@ -240,6 +240,15 @@ public class ComponentNode extends Node { this.availableConfigs = configs; } + private ConfigInstance getConfigInstance(ConfigKey<?> key) { + if (! availableConfigs.containsKey(key)) + throw new IllegalArgumentException("Config not found in the map of available configs: " + key); + else if (availableConfigs.get(key) == null) + throw new IllegalStateException("The map of available configs has a null config for: " + key); + + return availableConfigs.get(key); + } + @Override public Set<ConfigKey<ConfigInstance>> configKeys() { return configParameterClasses().stream().map(par -> new ConfigKey<>(par, configId)).collect(Collectors.toSet()); diff --git a/container-disc/src/main/java/com/yahoo/container/jdisc/SystemInfoProvider.java b/container-disc/src/main/java/com/yahoo/container/jdisc/SystemInfoProvider.java index a34b7506666..39c4b108aa6 100644 --- a/container-disc/src/main/java/com/yahoo/container/jdisc/SystemInfoProvider.java +++ b/container-disc/src/main/java/com/yahoo/container/jdisc/SystemInfoProvider.java @@ -1,12 +1,14 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.container.jdisc; +import ai.vespa.cloud.ApplicationId; import ai.vespa.cloud.Cluster; import ai.vespa.cloud.Environment; import ai.vespa.cloud.Node; import ai.vespa.cloud.SystemInfo; import ai.vespa.cloud.Zone; import com.google.inject.Inject; +import com.yahoo.cloud.config.ApplicationIdConfig; import com.yahoo.cloud.config.ClusterInfoConfig; import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.component.AbstractComponent; @@ -24,8 +26,14 @@ public class SystemInfoProvider extends AbstractComponent implements Provider<Sy private final SystemInfo instance; @Inject - public SystemInfoProvider(ConfigserverConfig csConfig, QrConfig qrConfig, ClusterInfoConfig ciConfig) { - this.instance = new SystemInfo(new Zone(Environment.valueOf(csConfig.environment()), csConfig.region()), + public SystemInfoProvider(ConfigserverConfig csConfig, + QrConfig qrConfig, + ClusterInfoConfig ciConfig, + ApplicationIdConfig applicationIdConfig) { + this.instance = new SystemInfo(new ApplicationId(applicationIdConfig.tenant(), + applicationIdConfig.application(), + applicationIdConfig.instance()), + new Zone(Environment.valueOf(csConfig.environment()), csConfig.region()), new Cluster(ciConfig.nodeCount(), ciConfig.nodeIndices()), new Node(qrConfig.nodeIndex())); } diff --git a/container-disc/src/main/sh/vespa-start-container-daemon.sh b/container-disc/src/main/sh/vespa-start-container-daemon.sh index 8c122d3170e..ded38e9f7c9 100755 --- a/container-disc/src/main/sh/vespa-start-container-daemon.sh +++ b/container-disc/src/main/sh/vespa-start-container-daemon.sh @@ -129,9 +129,14 @@ configure_memory() { available=`free -m | grep Mem | tr -s ' ' | cut -f2 -d' '` if hash cgget 2>/dev/null; then # TODO: Create vespa_cgget for this and remove dependency on libcgroup-tools - available_cgroup_bytes=$(cgget -nv -r memory.limit_in_bytes /) + available_cgroup_bytes=$(cgget -nv -r memory.limit_in_bytes / 2>&1) if [ $? -ne 0 ]; then - available_cgroup_bytes=$(vespa_cg2get memory.max) + if [[ "$available_cgroup_bytes" =~ "Cgroup is not mounted" ]]; then + available_cgroup_bytes=$(vespa_cg2get memory.max) + else + echo "$available_cgroup_bytes" >&2 + fi + # If command failed or returned value is 'max' assign a big value (default in CGroup v1) if ! [[ "$available_cgroup_bytes" =~ ^[0-9]+$ ]]; then available_cgroup_bytes=$(((1 << 63) -1)) diff --git a/container-messagebus/src/main/java/com/yahoo/container/jdisc/messagebus/NetworkMultiplexerProvider.java b/container-messagebus/src/main/java/com/yahoo/container/jdisc/messagebus/NetworkMultiplexerProvider.java index 48ffec4429e..fc7c82520a4 100644 --- a/container-messagebus/src/main/java/com/yahoo/container/jdisc/messagebus/NetworkMultiplexerProvider.java +++ b/container-messagebus/src/main/java/com/yahoo/container/jdisc/messagebus/NetworkMultiplexerProvider.java @@ -30,7 +30,7 @@ public class NetworkMultiplexerProvider { } public NetworkMultiplexerProvider(NetworkMultiplexerHolder net, ContainerMbusConfig mbusConfig, String identity) { - this.nets = () -> net.get(asParameters(mbusConfig, identity).setSlobrokConfigId(identity)); + this.nets = () -> net.get(asParameters(mbusConfig, identity)); } public static RPCNetworkParams asParameters(ContainerMbusConfig mbusConfig, SlobroksConfig slobroksConfig, String identity) { diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/MatchFeatureData.java b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/MatchFeatureData.java new file mode 100644 index 00000000000..7781a01b781 --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/MatchFeatureData.java @@ -0,0 +1,94 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.dispatch.rpc; + +import com.yahoo.collections.Hashlet; + +import com.yahoo.data.access.ArrayTraverser; +import com.yahoo.data.access.Inspector; +import com.yahoo.data.access.ObjectTraverser; +import com.yahoo.data.access.Type; +import com.yahoo.data.access.simple.Value; + +import java.util.ArrayList; +import java.util.AbstractMap.SimpleEntry; +import java.util.List; +import java.util.Map; + +/** + * MatchFeatureData helps pack match features for hits into + * inspectable HitValue objects, all sharing the same Hashlet + * for the field names. + * @author arnej + */ +class MatchFeatureData { + + private final Hashlet<String,Integer> hashlet; + + MatchFeatureData(List<String> keys) { + this.hashlet = new Hashlet<>(); + hashlet.reserve(keys.size()); + int i = 0; + for (String key : keys) { + hashlet.put(key, i++); + } + } + + static class HitValue extends Value { + private final Hashlet<String,Integer> hashlet; + private final byte[][] dataValues; + private final double[] doubleValues; + + public Type type() { return Type.OBJECT; } + public boolean valid() { return true; } + public int fieldCount() { return hashlet.size(); } + public void traverse(ObjectTraverser ot) { + for (int i = 0; i < hashlet.size(); i++) { + String fn = hashlet.key(i); + int offset = hashlet.value(i); + ot.field(fn, valueAt(offset)); + } + } + public Inspector field(String name) { + int offset = hashlet.getIndexOfKey(name); + if (offset < 0) { + return invalid(); + } + return valueAt(offset); + } + public Iterable<Map.Entry<String,Inspector>> fields() { + var list = new ArrayList<Map.Entry<String,Inspector>>(hashlet.size()); + for (int i = 0; i < hashlet.size(); i++) { + String fn = hashlet.key(i); + int offset = hashlet.value(i); + list.add(new SimpleEntry<String,Inspector>(fn, valueAt(offset))); + } + return list; + } + + // use from enclosing class only + private HitValue(Hashlet<String,Integer> hashlet) { + this.hashlet = hashlet; + this.dataValues = new byte[hashlet.size()][]; + this.doubleValues = new double[hashlet.size()]; + } + + void set(int index, byte[] data) { + dataValues[index] = data; + } + void set(int index, double value) { + doubleValues[index] = value; + } + + private Inspector valueAt(int index) { + if (dataValues[index] != null) { + return new Value.DataValue(dataValues[index]); + } + return new Value.DoubleValue(doubleValues[index]); + } + } + + HitValue addHit() { + return new HitValue(hashlet); + } + +} diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/ProtobufSerialization.java b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/ProtobufSerialization.java index ac41321f639..0e1df2b3d9a 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/ProtobufSerialization.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/ProtobufSerialization.java @@ -197,7 +197,7 @@ public class ProtobufSerialization { } static InvokerResult convertToResult(Query query, SearchProtocol.SearchReply protobuf, - DocumentDatabase documentDatabase, int partId, int distKey) + DocumentDatabase documentDatabase, int partId, int distKey) { InvokerResult result = new InvokerResult(query, protobuf.getHitsCount()); @@ -206,7 +206,8 @@ public class ProtobufSerialization { convertSearchReplyErrors(result.getResult(), protobuf.getErrorsList()); List<String> featureNames = protobuf.getMatchFeatureNamesList(); - + var haveMatchFeatures = ! featureNames.isEmpty(); + MatchFeatureData matchFeatures = haveMatchFeatures ? new MatchFeatureData(featureNames) : null; var haveGrouping = ! protobuf.getGroupingBlob().isEmpty(); if (haveGrouping) { BufferSerializer buf = new BufferSerializer(new GrowableByteBuffer(protobuf.getGroupingBlob().asReadOnlyByteBuffer())); @@ -221,27 +222,27 @@ public class ProtobufSerialization { hit.setQuery(query); result.getResult().hits().add(hit); } - for (var replyHit : protobuf.getHitsList()) { LeanHit hit = (replyHit.getSortData().isEmpty()) ? new LeanHit(replyHit.getGlobalId().toByteArray(), partId, distKey, replyHit.getRelevance()) : new LeanHit(replyHit.getGlobalId().toByteArray(), partId, distKey, replyHit.getRelevance(), replyHit.getSortData().toByteArray()); - if (! featureNames.isEmpty()) { - List<SearchProtocol.Feature> featureValues = replyHit.getMatchFeaturesList(); - var object = new Value.ObjectValue(); - var nameIter = featureNames.iterator(); - var valueIter = featureValues.iterator(); - while (nameIter.hasNext() && valueIter.hasNext()) { - String name = nameIter.next(); - SearchProtocol.Feature value = valueIter.next(); - ByteString tensorBlob = value.getTensor(); - if (tensorBlob.isEmpty()) { - object.put(name, value.getNumber()); - } else { - object.put(name, new Value.DataValue(tensorBlob.toByteArray())); + if (haveMatchFeatures) { + var hitFeatures = matchFeatures.addHit(); + var featureList = replyHit.getMatchFeaturesList(); + if (featureList.size() == featureNames.size()) { + int idx = 0; + for (SearchProtocol.Feature value : featureList) { + ByteString tensorBlob = value.getTensor(); + if (tensorBlob.isEmpty()) { + hitFeatures.set(idx++, value.getNumber()); + } else { + hitFeatures.set(idx++, tensorBlob.toByteArray()); + } } + hit.addMatchFeatures(hitFeatures); + } else { + result.getResult().hits().addError(ErrorMessage.createBackendCommunicationError("mismatch in match feature sizes")); } - hit.addMatchFeatures(object); } result.getLeanHits().add(hit); } diff --git a/container-search/src/main/java/com/yahoo/search/statistics/PeakQpsSearcher.java b/container-search/src/main/java/com/yahoo/search/statistics/PeakQpsSearcher.java index e4bce05b4f2..2823a7d74e1 100644 --- a/container-search/src/main/java/com/yahoo/search/statistics/PeakQpsSearcher.java +++ b/container-search/src/main/java/com/yahoo/search/statistics/PeakQpsSearcher.java @@ -14,13 +14,19 @@ import com.yahoo.statistics.Handle; import com.yahoo.statistics.Statistics; import com.yahoo.statistics.Value; -import java.util.*; +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.Deque; +import java.util.List; +import java.util.ListIterator; /** * Aggregate peak qps and expose through meta hits and/or log events. * * @author Steinar Knutsen + * @deprecated Will be removed on Vespa 8 */ +@Deprecated public class PeakQpsSearcher extends Searcher { private final ThreadLocalDirectory<Deque<QueryRatePerSecond>, Long> directory; diff --git a/container-search/src/main/java/com/yahoo/search/statistics/TimingSearcher.java b/container-search/src/main/java/com/yahoo/search/statistics/TimingSearcher.java index 8aa68d83d88..5d036b8fa20 100644 --- a/container-search/src/main/java/com/yahoo/search/statistics/TimingSearcher.java +++ b/container-search/src/main/java/com/yahoo/search/statistics/TimingSearcher.java @@ -19,8 +19,10 @@ import com.yahoo.statistics.Value; * measuring time consumption a search chain. * * @author Steinar Knutsen + * @deprecated Will be removed on Vespa 8 */ @Before("rawQuery") +@Deprecated public class TimingSearcher extends PingableSearcher { private Value measurements; diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/rpc/MatchFeatureDataTest.java b/container-search/src/test/java/com/yahoo/search/dispatch/rpc/MatchFeatureDataTest.java new file mode 100644 index 00000000000..6834a5edf9b --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/dispatch/rpc/MatchFeatureDataTest.java @@ -0,0 +1,102 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.search.dispatch.rpc; + +import com.yahoo.data.access.ArrayTraverser; +import com.yahoo.data.access.Inspector; +import com.yahoo.data.access.ObjectTraverser; +import com.yahoo.data.access.Type; +import org.junit.Test; + +import java.util.List; +import java.util.Map; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +/** + * @author arnej + */ +public class MatchFeatureDataTest { + + @Test + public void testHitValueAPI() { + List<String> names = List.of("foo", "bar", "baz", "qux", "quux"); + var mf = new MatchFeatureData(names); + var hit = mf.addHit(); + assertEquals(hit.type(), Type.OBJECT); + assertTrue(hit.valid()); + hit.set(0, 1.0); + byte[] somebytes = { 42, 0, 17 }; + hit.set(2, somebytes); + hit.set(4, 5.0); + hit.set(1, 2.0); + hit.set(3, 4.0); + assertEquals(0, hit.entryCount()); + assertEquals(5, hit.fieldCount()); + var f0 = hit.field("not"); + assertFalse(f0.valid()); + + var f1 = hit.field("foo"); + assertTrue(f1.valid()); + assertEquals(f1.type(), Type.DOUBLE); + assertEquals(f1.asDouble(), 1.0, 0.0); + + var f2 = hit.field("bar"); + assertTrue(f2.valid()); + assertEquals(f2.type(), Type.DOUBLE); + assertEquals(f2.asDouble(), 2.0, 0.0); + + var f3 = hit.field("baz"); + assertTrue(f3.valid()); + assertEquals(f3.type(), Type.DATA); + var gotbytes = f3.asData(); + assertEquals(3, gotbytes.length); + assertEquals(42, gotbytes[0]); + assertEquals(0, gotbytes[1]); + assertEquals(17, gotbytes[2]); + + var f5 = hit.field("quux"); + assertTrue(f5.valid()); + assertEquals(f5.type(), Type.DOUBLE); + assertEquals(f5.asDouble(), 5.0, 0.0); + + var fields = hit.fields().iterator(); + assertTrue(fields.hasNext()); + Map.Entry<String,Inspector> entry = fields.next(); + assertEquals("foo", entry.getKey()); + assertEquals(f1.type(), entry.getValue().type()); + assertEquals(f1.asDouble(), entry.getValue().asDouble(), 0.0); + + assertTrue(fields.hasNext()); + entry = fields.next(); + assertEquals("bar", entry.getKey()); + + assertTrue(fields.hasNext()); + entry = fields.next(); + assertEquals("baz", entry.getKey()); + assertEquals(f3.type(), entry.getValue().type()); + assertEquals(f3.asData(), entry.getValue().asData()); + + assertTrue(fields.hasNext()); + entry = fields.next(); + assertEquals("qux", entry.getKey()); + var f4 = entry.getValue(); + assertTrue(f4.valid()); + assertEquals(f4.type(), Type.DOUBLE); + assertEquals(f4.asDouble(), 4.0, 0.0); + + assertTrue(fields.hasNext()); + entry = fields.next(); + assertEquals("quux", entry.getKey()); + assertEquals(f5.type(), entry.getValue().type()); + assertEquals(f5.asDouble(), entry.getValue().asDouble(), 0.0); + + assertFalse(fields.hasNext()); + + assertEquals("{\"foo\":1.0,\"bar\":2.0,\"baz\":\"0x2A0011\",\"qux\":4.0,\"quux\":5.0}", + hit.toString()); + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/searchers/test/RateLimitingSearcherTestCase.java b/container-search/src/test/java/com/yahoo/search/searchers/test/RateLimitingSearcherTestCase.java index cb1a44b8afd..4ee57112960 100755 --- a/container-search/src/test/java/com/yahoo/search/searchers/test/RateLimitingSearcherTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/searchers/test/RateLimitingSearcherTestCase.java @@ -45,9 +45,9 @@ public class RateLimitingSearcherTestCase { ManualClock clock = new ManualClock(); MetricReceiver.MockReceiver metric = new MetricReceiver.MockReceiver(); - Chain<Searcher> chain = new Chain<Searcher>("test", new RateLimitingSearcher(new RateLimitingConfig(rateLimitingConfig), - new ClusterInfoConfig(clusterInfoConfig), - metric, clock), + Chain<Searcher> chain = new Chain<>("test", new RateLimitingSearcher(new RateLimitingConfig(rateLimitingConfig), + new ClusterInfoConfig(clusterInfoConfig), + metric, clock), new CostSettingSearcher()); assertEquals("'rate' request are available initially", 2, tryRequests(chain, "id1")); assertTrue("However, don't reject if we dryRun", executeWasAllowed(chain, "id1", true)); diff --git a/container-search/src/test/java/com/yahoo/search/statistics/PeakQpsTestCase.java b/container-search/src/test/java/com/yahoo/search/statistics/PeakQpsTestCase.java deleted file mode 100644 index 4bc8ee76165..00000000000 --- a/container-search/src/test/java/com/yahoo/search/statistics/PeakQpsTestCase.java +++ /dev/null @@ -1,164 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.statistics; - -import static org.junit.Assert.*; - -import java.util.Deque; -import java.util.List; - -import com.yahoo.statistics.Statistics; -import org.junit.Test; - -import com.yahoo.component.chain.Chain; -import com.yahoo.concurrent.LocalInstance; -import com.yahoo.concurrent.ThreadLocalDirectory; -import com.yahoo.search.Query; -import com.yahoo.search.Result; -import com.yahoo.search.Searcher; -import com.yahoo.search.result.Hit; -import com.yahoo.search.searchchain.Execution; -import com.yahoo.search.statistics.PeakQpsSearcher.QueryRatePerSecond; - -/** - * Check peak QPS aggregation has a chance of working. - * - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> - */ -public class PeakQpsTestCase { - - static class Producer implements Runnable { - private final ThreadLocalDirectory<Deque<QueryRatePerSecond>, Long> rates; - - Producer(ThreadLocalDirectory<Deque<QueryRatePerSecond>, Long> rates) { - this.rates = rates; - } - - @Override - public void run() { - LocalInstance<Deque<QueryRatePerSecond>, Long> rate = rates.getLocalInstance(); - rates.update(1L, rate); - rates.update(2L, rate); - rates.update(2L, rate); - rates.update(3L, rate); - rates.update(3L, rate); - rates.update(3L, rate); - rates.update(4L, rate); - rates.update(4L, rate); - rates.update(4L, rate); - rates.update(4L, rate); - } - } - - static class LaterProducer implements Runnable { - private final ThreadLocalDirectory<Deque<QueryRatePerSecond>, Long> rates; - - LaterProducer(ThreadLocalDirectory<Deque<QueryRatePerSecond>, Long> rates) { - this.rates = rates; - } - - @Override - public void run() { - LocalInstance<Deque<QueryRatePerSecond>, Long> rate = rates.getLocalInstance(); - rates.update(2L, rate); - rates.update(2L, rate); - rates.update(3L, rate); - rates.update(3L, rate); - rates.update(3L, rate); - rates.update(5L, rate); - rates.update(5L, rate); - rates.update(6L, rate); - rates.update(7L, rate); - } - } - - @Test - public void checkBasicDataAggregation() { - ThreadLocalDirectory<Deque<QueryRatePerSecond>, Long> directory = PeakQpsSearcher.createDirectory(); - final int threadCount = 20; - Thread[] threads = new Thread[threadCount]; - for (int i = 0; i < threadCount; ++i) { - Producer p = new Producer(directory); - threads[i] = new Thread(p); - threads[i].start(); - } - for (Thread t : threads) { - try { - t.join(); - } catch (InterruptedException e) { - // nop - } - } - List<Deque<QueryRatePerSecond>> measurements = directory.fetch(); - List<QueryRatePerSecond> results = PeakQpsSearcher.merge(measurements); - assertTrue(results.get(0).when == 1L); - assertTrue(results.get(0).howMany == threadCount); - assertTrue(results.get(1).when == 2L); - assertTrue(results.get(1).howMany == threadCount * 2); - assertTrue(results.get(2).when == 3L); - assertTrue(results.get(2).howMany == threadCount * 3); - assertTrue(results.get(3).when == 4L); - assertTrue(results.get(3).howMany == threadCount * 4); - } - - @Test - public void checkMixedDataAggregation() { - ThreadLocalDirectory<Deque<QueryRatePerSecond>, Long> directory = PeakQpsSearcher.createDirectory(); - final int firstThreads = 20; - final int secondThreads = 20; - final int threadCount = firstThreads + secondThreads; - Thread[] threads = new Thread[threadCount]; - for (int i = 0; i < threadCount; ++i) { - if (i < firstThreads) { - Producer p = new Producer(directory); - threads[i] = new Thread(p); - } else { - LaterProducer p = new LaterProducer(directory); - threads[i] = new Thread(p); - } - threads[i].start(); - - } - for (Thread t : threads) { - try { - t.join(); - } catch (InterruptedException e) { - // nop - } - } - List<Deque<QueryRatePerSecond>> measurements = directory.fetch(); - List<QueryRatePerSecond> results = PeakQpsSearcher.merge(measurements); - assertTrue(results.size() == 7); - assertTrue(results.get(0).when == 1L); - assertTrue(results.get(0).howMany == firstThreads); - assertTrue(results.get(1).when == 2L); - assertTrue(results.get(1).howMany == threadCount * 2); - assertTrue(results.get(2).when == 3L); - assertTrue(results.get(2).howMany == threadCount * 3); - assertTrue(results.get(3).when == 4L); - assertTrue(results.get(3).howMany == firstThreads * 4); - assertTrue(results.get(4).when == 5L); - assertTrue(results.get(4).howMany == secondThreads * 2); - assertTrue(results.get(5).when == 6L); - assertTrue(results.get(5).howMany == secondThreads); - assertTrue(results.get(6).when == 7L); - assertTrue(results.get(6).howMany == secondThreads); - } - - @Test - public void checkSearch() { - MeasureQpsConfig config = new MeasureQpsConfig( - new MeasureQpsConfig.Builder().outputmethod( - MeasureQpsConfig.Outputmethod.METAHIT).queryproperty( - "qpsprobe")); - Searcher s = new PeakQpsSearcher(config, Statistics.nullImplementation); - Chain<Searcher> c = new Chain<>(s); - Execution e = new Execution(c, Execution.Context.createContextStub()); - e.search(new Query("/?query=a")); - new Execution(c, Execution.Context.createContextStub()); - Result r = e.search(new Query("/?query=a&qpsprobe=true")); - final Hit hit = r.hits().get(0); - assertTrue(hit instanceof PeakQpsSearcher.QpsHit); - assertNotNull(hit.fields().get(PeakQpsSearcher.QpsHit.MEAN_QPS)); - assertNotNull(hit.fields().get(PeakQpsSearcher.QpsHit.PEAK_QPS)); - } -} diff --git a/container-search/src/test/java/com/yahoo/search/statistics/TimingSearcherTestCase.java b/container-search/src/test/java/com/yahoo/search/statistics/TimingSearcherTestCase.java deleted file mode 100644 index 673d38cc2b8..00000000000 --- a/container-search/src/test/java/com/yahoo/search/statistics/TimingSearcherTestCase.java +++ /dev/null @@ -1,89 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.statistics; - -import com.yahoo.component.ComponentId; -import com.yahoo.prelude.Ping; -import com.yahoo.search.Query; -import com.yahoo.search.Result; -import com.yahoo.search.result.Hit; -import com.yahoo.search.searchchain.Execution; -import com.yahoo.search.statistics.TimingSearcher.Parameters; -import com.yahoo.statistics.Statistics; -import com.yahoo.statistics.Value; -import org.junit.Test; - -import static org.junit.Assert.assertEquals; - -public class TimingSearcherTestCase { - - public static class MockValue extends Value { - public int putCount = 0; - - public MockValue() { - super("mock", Statistics.nullImplementation, new Value.Parameters()); - } - - @Override - public void put(double x) { - putCount += 1; - } - } - - @Test - public void testMeasurementSearchPath() { - Parameters p = new Parameters("timingtest", TimeTracker.Activity.SEARCH); - TimingSearcher ts = new TimingSearcher(new ComponentId("lblblbl"), p, Statistics.nullImplementation); - MockValue v = new MockValue(); - ts.setMeasurements(v); - Execution exec = new Execution(ts, Execution.Context.createContextStub()); - Result r = exec.search(new Query("/?query=a")); - Hit f = new Hit("blblbl"); - f.setFillable(); - r.hits().add(f); - exec.fill(r, "whatever"); - exec.fill(r, "lalala"); - exec.ping(new Ping()); - exec.ping(new Ping()); - exec.ping(new Ping()); - assertEquals(1, v.putCount); - } - - @Test - public void testMeasurementFillPath() { - Parameters p = new Parameters("timingtest", TimeTracker.Activity.FILL); - TimingSearcher ts = new TimingSearcher(new ComponentId("lblblbl"), p, Statistics.nullImplementation); - MockValue v = new MockValue(); - ts.setMeasurements(v); - Execution exec = new Execution(ts, Execution.Context.createContextStub()); - Result r = exec.search(new Query("/?query=a")); - Hit f = new Hit("blblbl"); - f.setFillable(); - r.hits().add(f); - exec.fill(r, "whatever"); - exec.fill(r, "lalala"); - exec.ping(new Ping()); - exec.ping(new Ping()); - exec.ping(new Ping()); - assertEquals(2, v.putCount); - } - - @Test - public void testMeasurementPingPath() { - Parameters p = new Parameters("timingtest", TimeTracker.Activity.PING); - TimingSearcher ts = new TimingSearcher(new ComponentId("lblblbl"), p, Statistics.nullImplementation); - MockValue v = new MockValue(); - ts.setMeasurements(v); - Execution exec = new Execution(ts, Execution.Context.createContextStub()); - Result r = exec.search(new Query("/?query=a")); - Hit f = new Hit("blblbl"); - f.setFillable(); - r.hits().add(f); - exec.fill(r, "whatever"); - exec.fill(r, "lalala"); - exec.ping(new Ping()); - exec.ping(new Ping()); - exec.ping(new Ping()); - assertEquals(3, v.putCount); - } - -} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/DeploymentData.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/DeploymentData.java index 1f1f8577a32..ad98197fa93 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/DeploymentData.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/DeploymentData.java @@ -64,13 +64,13 @@ public class DeploymentData { this.zone = requireNonNull(zone); this.applicationPackage = requireNonNull(applicationPackage); this.platform = requireNonNull(platform); - this.containerEndpoints = requireNonNull(containerEndpoints); + this.containerEndpoints = Set.copyOf(requireNonNull(containerEndpoints)); this.endpointCertificateMetadata = requireNonNull(endpointCertificateMetadata); this.dockerImageRepo = requireNonNull(dockerImageRepo); this.athenzDomain = athenzDomain; this.quota = quota; - this.tenantSecretStores = tenantSecretStores; - this.operatorCertificates = operatorCertificates; + this.tenantSecretStores = List.copyOf(requireNonNull(tenantSecretStores)); + this.operatorCertificates = List.copyOf(requireNonNull(operatorCertificates)); this.dryRun = dryRun; } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ContainerEndpoint.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ContainerEndpoint.java index 3e9169a83aa..bac34e73dc5 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ContainerEndpoint.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ContainerEndpoint.java @@ -12,10 +12,12 @@ import java.util.Objects; public class ContainerEndpoint { private final String clusterId; + private final String scope; private final List<String> names; - public ContainerEndpoint(String clusterId, List<String> names) { + public ContainerEndpoint(String clusterId, String scope, List<String> names) { this.clusterId = nonEmpty(clusterId, "message must be non-empty"); + this.scope = Objects.requireNonNull(scope, "scope must be non-null"); this.names = List.copyOf(Objects.requireNonNull(names, "names must be non-null")); } @@ -24,6 +26,11 @@ public class ContainerEndpoint { return clusterId; } + /** The scope of this endpoint */ + public String scope() { + return scope; + } + /** * All valid DNS names for this endpoint. This can contain both proper DNS names and synthetic identifiers used for * routing, such as a Host header value that is not necessarily a proper DNS name. @@ -37,18 +44,17 @@ public class ContainerEndpoint { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; ContainerEndpoint that = (ContainerEndpoint) o; - return clusterId.equals(that.clusterId) && - names.equals(that.names); + return clusterId.equals(that.clusterId) && scope.equals(that.scope) && names.equals(that.names); } @Override public int hashCode() { - return Objects.hash(clusterId, names); + return Objects.hash(clusterId, scope, names); } @Override public String toString() { - return "container endpoint for " + clusterId + " " + names; + return "container endpoint for " + clusterId + ": " + names + " [scope=" + scope + "]"; } private static String nonEmpty(String s, String message) { diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/MemoryNameService.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/MemoryNameService.java index 91e97fe73a5..18ff3f18137 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/MemoryNameService.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/MemoryNameService.java @@ -108,7 +108,7 @@ public class MemoryNameService implements NameService { @Override public void removeRecords(List<Record> records) { - this.records.removeAll(records); + records.forEach(this.records::remove); } /** diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/zone/ZoneRegistry.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/zone/ZoneRegistry.java index 852570b9ed4..893b7a1b1dc 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/zone/ZoneRegistry.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/zone/ZoneRegistry.java @@ -42,6 +42,9 @@ public interface ZoneRegistry { /** Returns the URI for the config server VIP in the given zone */ URI getConfigServerVipUri(ZoneId zoneId); + /** Returns the VIP hostname for the shared routing layer in given zone, if any */ + Optional<String> getVipHostname(ZoneId zoneId); + /** Returns the time to live for deployments in the given zone, or empty if this is infinite */ Optional<Duration> getDeploymentTimeToLive(ZoneId zoneId); diff --git a/controller-server/pom.xml b/controller-server/pom.xml index 3b2949f6964..e9fadee58c7 100644 --- a/controller-server/pom.xml +++ b/controller-server/pom.xml @@ -239,6 +239,17 @@ <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-compiler-plugin</artifactId> </plugin> + + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-surefire-plugin</artifactId> + <configuration> + <!-- Illegal reflective access by LogFileHandler via com.yahoo.io.NativeIO --> + <argLine> + --add-opens=java.base/java.io=ALL-UNNAMED + </argLine> + </configuration> + </plugin> </plugins> </build> </project> diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java index 6be62367407..16f12b3ac07 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java @@ -49,7 +49,6 @@ import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Objects; -import java.util.Optional; import java.util.Set; import java.util.TreeMap; import java.util.stream.Collectors; @@ -86,8 +85,8 @@ public class RoutingController { return rotationRepository; } - /** Returns endpoints for given deployment */ - public EndpointList endpointsOf(DeploymentId deployment) { + /** Read and return zone-scoped endpoints for given deployment */ + public EndpointList readEndpointsOf(DeploymentId deployment) { Set<Endpoint> endpoints = new LinkedHashSet<>(); boolean isSystemApplication = SystemApplication.matching(deployment.applicationId()).isPresent(); // Avoid reading application more than once per call to this @@ -104,46 +103,67 @@ public class RoutingController { return EndpointList.copyOf(endpoints); } - /** Returns global-scoped endpoints for given instance */ - public EndpointList endpointsOf(ApplicationId instance) { + /** Read application and return declared endpoints for given instance */ + public EndpointList readDeclaredEndpointsOf(ApplicationId instance) { if (SystemApplication.matching(instance).isPresent()) return EndpointList.EMPTY; - return endpointsOf(controller.applications().requireApplication(TenantAndApplicationId.from(instance)), - instance.instance()); + return readDeclaredEndpointsOf(TenantAndApplicationId.from(instance)).instance(instance.instance()); } - /** Returns global-scoped endpoints for given instance */ - public EndpointList endpointsOf(Application application, InstanceName instanceName) { + /** Read application and return declared endpoints for given application */ + public EndpointList readDeclaredEndpointsOf(TenantAndApplicationId application) { + return declaredEndpointsOf(controller.applications().requireApplication(application)); + } + + /** Returns endpoints declared in {@link DeploymentSpec} for given application */ + public EndpointList declaredEndpointsOf(Application application) { Set<Endpoint> endpoints = new LinkedHashSet<>(); - Instance instance = application.require(instanceName); DeploymentSpec deploymentSpec = application.deploymentSpec(); - Optional<DeploymentInstanceSpec> spec = deploymentSpec.instance(instanceName); - if (spec.isEmpty()) return EndpointList.EMPTY; - // Add endpoint declared with legacy syntax - spec.get().globalServiceId().ifPresent(clusterId -> { - List<DeploymentId> deployments = spec.get().zones().stream() - .filter(zone -> zone.concerns(Environment.prod)) - .map(zone -> new DeploymentId(instance.id(), ZoneId.from(Environment.prod, zone.region().get()))) - .collect(Collectors.toList()); - RoutingId routingId = RoutingId.of(instance.id(), EndpointId.defaultId()); - endpoints.addAll(computeGlobalEndpoints(routingId, ClusterSpec.Id.from(clusterId), deployments, deploymentSpec)); - }); - // Add endpoints declared with current syntax - spec.get().endpoints().forEach(declaredEndpoint -> { - RoutingId routingId = RoutingId.of(instance.id(), EndpointId.of(declaredEndpoint.endpointId())); - List<DeploymentId> deployments = declaredEndpoint.regions().stream() - .map(region -> new DeploymentId(instance.id(), - ZoneId.from(Environment.prod, region))) - .collect(Collectors.toList()); - endpoints.addAll(computeGlobalEndpoints(routingId, ClusterSpec.Id.from(declaredEndpoint.containerId()), deployments, deploymentSpec)); - }); + for (var spec : deploymentSpec.instances()) { + ApplicationId instance = application.id().instance(spec.name()); + // Add endpoint declared with legacy syntax + spec.globalServiceId().ifPresent(clusterId -> { + List<DeploymentId> deployments = spec.zones().stream() + .filter(zone -> zone.concerns(Environment.prod)) + .map(zone -> new DeploymentId(instance, ZoneId.from(Environment.prod, zone.region().get()))) + .collect(Collectors.toList()); + RoutingId routingId = RoutingId.of(instance, EndpointId.defaultId()); + endpoints.addAll(computeGlobalEndpoints(routingId, ClusterSpec.Id.from(clusterId), deployments, deploymentSpec)); + }); + // Add endpoints declared with current syntax + spec.endpoints().forEach(declaredEndpoint -> { + RoutingId routingId = RoutingId.of(instance, EndpointId.of(declaredEndpoint.endpointId())); + List<DeploymentId> deployments = declaredEndpoint.regions().stream() + .map(region -> new DeploymentId(instance, + ZoneId.from(Environment.prod, region))) + .collect(Collectors.toList()); + endpoints.addAll(computeGlobalEndpoints(routingId, ClusterSpec.Id.from(declaredEndpoint.containerId()), deployments, deploymentSpec)); + }); + } + // Add application endpoints + for (var declaredEndpoint : deploymentSpec.endpoints()) { + Map<DeploymentId, Integer> deployments = declaredEndpoint.targets().stream() + .collect(Collectors.toMap(t -> new DeploymentId(application.id().instance(t.instance()), + ZoneId.from(Environment.prod, t.region())), + t -> t.weight())); + List<RoutingMethod> availableRoutingMethods = routingMethodsOfAll(deployments.keySet(), deploymentSpec); + for (var routingMethod : availableRoutingMethods) { + endpoints.add(Endpoint.of(application.id()) + .targetApplication(EndpointId.of(declaredEndpoint.endpointId()), + ClusterSpec.Id.from(declaredEndpoint.containerId()), + deployments) + .routingMethod(routingMethod) + .on(Port.fromRoutingMethod(routingMethod)) + .in(controller.system())); + } + } return EndpointList.copyOf(endpoints); } - /** Returns all zone-scoped endpoints and corresponding cluster IDs for given deployments, grouped by their zone */ - public Map<ZoneId, List<Endpoint>> zoneEndpointsOf(Collection<DeploymentId> deployments) { + /** Read and return zone-scoped endpoints for given deployments, grouped by their zone */ + public Map<ZoneId, List<Endpoint>> readZoneEndpointsOf(Collection<DeploymentId> deployments) { var endpoints = new TreeMap<ZoneId, List<Endpoint>>(Comparator.comparing(ZoneId::value)); for (var deployment : deployments) { - EndpointList zoneEndpoints = endpointsOf(deployment).scope(Endpoint.Scope.zone).not().legacy(); + EndpointList zoneEndpoints = readEndpointsOf(deployment).scope(Endpoint.Scope.zone).not().legacy(); zoneEndpoints = directEndpoints(zoneEndpoints, deployment.applicationId()); if ( ! zoneEndpoints.isEmpty()) { endpoints.put(deployment.zoneId(), zoneEndpoints.asList()); @@ -183,7 +203,7 @@ public class RoutingController { /** Change status of all global endpoints for given deployment */ public void setGlobalRotationStatus(DeploymentId deployment, EndpointStatus status) { - endpointsOf(deployment.applicationId()).requiresRotation().primary().ifPresent(endpoint -> { + readDeclaredEndpointsOf(deployment.applicationId()).requiresRotation().primary().ifPresent(endpoint -> { try { controller.serviceRegistry().configServer().setGlobalRotationStatus(deployment, endpoint.upstreamIdOf(deployment), status); } catch (Exception e) { @@ -195,7 +215,7 @@ public class RoutingController { /** Get global endpoint status for given deployment */ public Map<Endpoint, EndpointStatus> globalRotationStatus(DeploymentId deployment) { var routingEndpoints = new LinkedHashMap<Endpoint, EndpointStatus>(); - endpointsOf(deployment.applicationId()).requiresRotation().primary().ifPresent(endpoint -> { + readDeclaredEndpointsOf(deployment.applicationId()).requiresRotation().primary().ifPresent(endpoint -> { var upstreamName = endpoint.upstreamIdOf(deployment); var status = controller.serviceRegistry().configServer().getGlobalRotationStatus(deployment, upstreamName); routingEndpoints.put(endpoint, status); @@ -218,17 +238,19 @@ public class RoutingController { return application; } - /** Returns the global endpoints for given deployment as container endpoints */ + /** Returns the global and application-level endpoints for given deployment, as container endpoints */ public Set<ContainerEndpoint> containerEndpointsOf(Application application, InstanceName instanceName, ZoneId zone) { Instance instance = application.require(instanceName); - boolean registerLegacyNames = legacyNamesAvailable(application.deploymentSpec(), instanceName); + boolean registerLegacyNames = requiresLegacyNames(application.deploymentSpec(), instanceName); Set<ContainerEndpoint> containerEndpoints = new HashSet<>(); - EndpointList endpoints = endpointsOf(application, instanceName); + DeploymentId deployment = new DeploymentId(instance.id(), zone); + EndpointList endpoints = declaredEndpointsOf(application).targets(deployment); + EndpointList globalEndpoints = endpoints.scope(Endpoint.Scope.global); // Add endpoints backed by a rotation, and register them in DNS if necessary for (var assignedRotation : instance.rotations()) { var names = new ArrayList<String>(); - EndpointList rotationEndpoints = endpoints.named(assignedRotation.endpointId()) - .requiresRotation(); + EndpointList rotationEndpoints = globalEndpoints.named(assignedRotation.endpointId()) + .requiresRotation(); // Skip rotations which do not apply to this zone. Legacy names always point to all zones if (!registerLegacyNames && !assignedRotation.regions().contains(zone.region())) { @@ -253,17 +275,41 @@ public class RoutingController { // Include rotation ID as a valid name of this container endpoint (required by global routing health checks) names.add(assignedRotation.rotationId().asString()); - containerEndpoints.add(new ContainerEndpoint(assignedRotation.clusterId().value(), names)); + containerEndpoints.add(new ContainerEndpoint(assignedRotation.clusterId().value(), + asString(Endpoint.Scope.global), + names)); } - // Add endpoints not backed by a rotation - DeploymentId deployment = new DeploymentId(instance.id(), zone); - endpoints.not().requiresRotation() - .targets(deployment) - .groupingBy(Endpoint::cluster) - .forEach((clusterId, clusterEndpoints) -> { - containerEndpoints.add(new ContainerEndpoint(clusterId.value(), - clusterEndpoints.mapToList(Endpoint::dnsName))); - }); + // Add endpoints not backed by a rotation (i.e. other routing methods so that the config server always knows + // about global names, even when not using rotations) + globalEndpoints.not().requiresRotation() + .groupingBy(Endpoint::cluster) + .forEach((clusterId, clusterEndpoints) -> { + containerEndpoints.add(new ContainerEndpoint(clusterId.value(), + asString(Endpoint.Scope.global), + clusterEndpoints.mapToList(Endpoint::dnsName))); + }); + // Add application endpoints + EndpointList applicationEndpoints = endpoints.scope(Endpoint.Scope.application); + for (var endpoint : applicationEndpoints.shared()) { // DNS for non-shared endpoints is handled by RoutingPolicies + Set<ZoneId> targetZones = endpoint.targets().stream() + .map(t -> t.deployment().zoneId()) + .collect(Collectors.toUnmodifiableSet()); + if (targetZones.size() != 1) throw new IllegalArgumentException("Endpoint '" + endpoint.name() + + "' must target a single zone, got " + + targetZones); + ZoneId targetZone = targetZones.iterator().next(); + String vipHostname = controller.zoneRegistry().getVipHostname(targetZone) + .orElseThrow(() -> new IllegalArgumentException("No VIP configured for zone " + targetZone)); + controller.nameServiceForwarder().createCname(RecordName.from(endpoint.dnsName()), + RecordData.fqdn(vipHostname), + Priority.normal); + } + applicationEndpoints.groupingBy(Endpoint::cluster) + .forEach((clusterId, clusterEndpoints) -> { + containerEndpoints.add(new ContainerEndpoint(clusterId.value(), + asString(Endpoint.Scope.application), + clusterEndpoints.mapToList(Endpoint::dnsName))); + }); return Collections.unmodifiableSet(containerEndpoints); } @@ -287,7 +333,7 @@ public class RoutingController { } /** Returns the routing methods that are available across all given deployments */ - private List<RoutingMethod> routingMethodsOfAll(List<DeploymentId> deployments, DeploymentSpec deploymentSpec) { + private List<RoutingMethod> routingMethodsOfAll(Collection<DeploymentId> deployments, DeploymentSpec deploymentSpec) { var deploymentsByMethod = new HashMap<RoutingMethod, Set<DeploymentId>>(); for (var deployment : deployments) { for (var method : controller.zoneRegistry().routingMethods(deployment.zoneId())) { @@ -306,7 +352,7 @@ public class RoutingController { } /** Returns whether traffic can be directly routed to all given deployments */ - private boolean canRouteDirectlyTo(List<DeploymentId> deployments, DeploymentSpec deploymentSpec) { + private boolean canRouteDirectlyTo(Collection<DeploymentId> deployments, DeploymentSpec deploymentSpec) { return deployments.stream().allMatch(deployment -> canRouteDirectlyTo(deployment, deploymentSpec)); } @@ -329,9 +375,8 @@ public class RoutingController { private List<Endpoint> computeGlobalEndpoints(RoutingId routingId, ClusterSpec.Id cluster, List<DeploymentId> deployments, DeploymentSpec deploymentSpec) { var endpoints = new ArrayList<Endpoint>(); var directMethods = 0; - var zones = deployments.stream().map(DeploymentId::zoneId).collect(Collectors.toList()); var availableRoutingMethods = routingMethodsOfAll(deployments, deploymentSpec); - boolean legacyNamesAvailable = legacyNamesAvailable(deploymentSpec, routingId.instance().instance()); + boolean legacyNamesAvailable = requiresLegacyNames(deploymentSpec, routingId.instance().instance()); for (var method : availableRoutingMethods) { if (method.isDirect() && ++directMethods > 1) { @@ -363,7 +408,7 @@ public class RoutingController { } /** Whether legacy global DNS names should be available for given application */ - private static boolean legacyNamesAvailable(DeploymentSpec deploymentSpec, InstanceName instanceName) { + private static boolean requiresLegacyNames(DeploymentSpec deploymentSpec, InstanceName instanceName) { return deploymentSpec.instance(instanceName) .flatMap(DeploymentInstanceSpec::globalServiceId) .isPresent(); @@ -387,5 +432,14 @@ public class RoutingController { return endpoints; } + private static String asString(Endpoint.Scope scope) { + switch (scope) { + case application: return "application"; + case global: return "global"; + case weighted: return "weighted"; + case zone: return "zone"; + } + throw new IllegalArgumentException("Unknown scope " + scope); + } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java index 3698c794e8f..35601dd94dd 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java @@ -36,6 +36,7 @@ public class Endpoint { private final EndpointId id; private final ClusterSpec.Id cluster; + private final Optional<InstanceName> instance; private final URI url; private final List<Target> targets; private final Scope scope; @@ -59,11 +60,11 @@ public class Endpoint { if (scope == Scope.zone && id != null) throw new IllegalArgumentException("Endpoint ID cannot be set for " + scope + " endpoints"); if (targets.size() != 1) throw new IllegalArgumentException("A single target must be given for " + scope + " endpoints"); } - if (scope != Scope.region && instanceName.isEmpty()) { + if (scope != Scope.application && instanceName.isEmpty()) { throw new IllegalArgumentException("Instance must be set for scope " + scope); } for (var target : targets) { - if (scope == Scope.region) { + if (scope == Scope.application) { TenantAndApplicationId owner = TenantAndApplicationId.from(target.deployment().applicationId()); if (!owner.equals(application)) { throw new IllegalArgumentException(id + " has target owned by " + owner + @@ -81,6 +82,7 @@ public class Endpoint { } this.id = id; this.cluster = cluster; + this.instance = instanceName; this.url = url; this.targets = List.copyOf(targets); this.scope = scope; @@ -124,6 +126,11 @@ public class Endpoint { return cluster; } + /** The specific instance this endpoint points to, if any */ + public Optional<InstanceName> instance() { + return instance; + } + /** Returns the URL used to access this */ public URI url() { return url; @@ -254,14 +261,14 @@ public class Endpoint { case zone: return "z"; case weighted: return "w"; case global: return "g"; - case region: return "r"; + case application: return "r"; } } switch (scope) { case zone: return ""; case weighted: return "w"; case global: return "global"; - case region: return "r"; + case application: return "r"; } throw new IllegalArgumentException("No scope symbol defined for " + scope + " in " + system); } @@ -353,7 +360,7 @@ public class Endpoint { * * Traffic is routed across instances according to weights specified in deployment.xml */ - region, + application, /** Endpoint points to one or more zones. Traffic is routed to the zone closest to the client */ global, @@ -370,7 +377,7 @@ public class Endpoint { /** Returns whether this scope may span multiple deployments */ public boolean multiDeployment() { - return this == region || this == global; + return this == application || this == global; } } @@ -532,7 +539,7 @@ public class Endpoint { this.targets = deployments.entrySet().stream() .map(kv -> new Target(kv.getKey(), kv.getValue())) .collect(Collectors.toUnmodifiableList()); - this.scope = Scope.region; + this.scope = Scope.application; return this; } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/EndpointList.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/EndpointList.java index 1ad315545e3..f9fd02fbf56 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/EndpointList.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/EndpointList.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.hosted.controller.application; import com.yahoo.collections.AbstractFilteringList; import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.InstanceName; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import java.util.Collection; @@ -41,6 +42,12 @@ public class EndpointList extends AbstractFilteringList<Endpoint, EndpointList> return matching(endpoint -> endpoint.cluster().equals(cluster)); } + /** Returns the subset of endpoints pointing to given instance */ + public EndpointList instance(InstanceName instance) { + return matching(endpoint -> endpoint.instance().isPresent() && + endpoint.instance().get().equals(instance)); + } + /** Returns the subset of endpoints which target all of the given deployments */ public EndpointList targets(List<DeploymentId> deployments) { return matching(endpoint -> endpoint.deployments().containsAll(deployments)); @@ -71,6 +78,11 @@ public class EndpointList extends AbstractFilteringList<Endpoint, EndpointList> return matching(endpoint -> endpoint.routingMethod().isDirect()); } + /** Returns the subset of endpoints that use shared routing */ + public EndpointList shared() { + return matching(endpoint -> endpoint.routingMethod().isShared()); + } + public static EndpointList copyOf(Collection<Endpoint> endpoints) { return new EndpointList(endpoints, false); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java index a34217d2226..3892ceeddf9 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java @@ -31,14 +31,12 @@ import java.util.Map; import java.util.Objects; import java.util.Optional; import java.util.OptionalLong; -import java.util.function.Supplier; import java.util.logging.Level; import java.util.logging.Logger; import java.util.stream.Collectors; import static java.util.Comparator.comparing; import static java.util.stream.Collectors.groupingBy; -import static java.util.stream.Collectors.partitioningBy; import static java.util.stream.Collectors.toList; /** @@ -129,28 +127,45 @@ public class DeploymentTrigger { /** * Finds and triggers jobs that can and should run but are currently not, and returns the number of triggered jobs. * - * Only one job is triggered each run for test jobs, since their environments have limited capacity. + * Only one job per type is triggered each run for test jobs, since their environments have limited capacity. */ public long triggerReadyJobs() { - return computeReadyJobs().stream() - .collect(partitioningBy(job -> job.jobType().environment().isTest())) - .entrySet().stream() - .flatMap(entry -> (entry.getKey() - // True for capacity constrained zones -- sort by priority and make a task for each job type. - ? entry.getValue().stream() - .sorted(comparing(Job::isRetry) - .thenComparing(Job::applicationUpgrade) - .reversed() - .thenComparing(Job::availableSince)) - .collect(groupingBy(Job::jobType)) - // False for production jobs -- keep step order and make a task for each application. - : entry.getValue().stream() - .collect(groupingBy(Job::applicationId))) - .values().stream() - .map(jobs -> (Supplier<Long>) jobs.stream() - .peek(this::trigger) - .limit(entry.getKey() ? 1 : Long.MAX_VALUE)::count)) - .parallel().map(Supplier::get).reduce(0L, Long::sum); + List<Job> readyJobs = computeReadyJobs(); + + var prodJobs = new ArrayList<Job>(); + var testJobs = new ArrayList<Job>(); + for (Job job : readyJobs) { + if (job.jobType.isTest()) testJobs.add(job); + else prodJobs.add(job); + } + + // Flat list of prod jobs, grouped by application id, retaining the step order + List<Job> sortedProdJobs = prodJobs.stream() + .collect(groupingBy(Job::applicationId)) + .values().stream() + .flatMap(List::stream) + .collect(Collectors.toUnmodifiableList()); + + // Map of test jobs, a list for each job type. Jobs in each list are sorted by priority. + Map<JobType, List<Job>> sortedTestJobsByType = testJobs.stream() + .sorted(comparing(Job::isRetry) + .thenComparing(Job::applicationUpgrade) + .reversed() + .thenComparing(Job::availableSince)) + .collect(groupingBy(Job::jobType)); + + // Trigger all prod jobs + sortedProdJobs.forEach(this::trigger); + long triggeredJobs = sortedProdJobs.size(); + + // Trigger max one test job per type + for (var jobs : sortedTestJobsByType.values()) { + if (jobs.size() > 0) { + trigger(jobs.get(0)); + triggeredJobs++; + } + } + return triggeredJobs; } /** Attempts to trigger the given job. */ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java index ee955fa8ff8..577dab69279 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java @@ -451,7 +451,7 @@ public class InternalStepRunner implements StepRunner { /** Returns true iff all calls to endpoint in the deployment give 100 consecutive 200 OK responses on /status.html. */ private boolean containersAreUp(ApplicationId id, ZoneId zoneId, DualLogger logger) { - var endpoints = controller.routing().zoneEndpointsOf(Set.of(new DeploymentId(id, zoneId))); + var endpoints = controller.routing().readZoneEndpointsOf(Set.of(new DeploymentId(id, zoneId))); if ( ! endpoints.containsKey(zoneId)) return false; @@ -477,7 +477,7 @@ public class InternalStepRunner implements StepRunner { } private boolean endpointsAvailable(ApplicationId id, ZoneId zone, DualLogger logger) { - var endpoints = controller.routing().zoneEndpointsOf(Set.of(new DeploymentId(id, zone))); + var endpoints = controller.routing().readZoneEndpointsOf(Set.of(new DeploymentId(id, zone))); if ( ! endpoints.containsKey(zone)) { logger.log("Endpoints not yet ready."); return false; @@ -586,7 +586,7 @@ public class InternalStepRunner implements StepRunner { deployments.add(new DeploymentId(id.application(), zoneId)); logger.log("Attempting to find endpoints ..."); - var endpoints = controller.routing().zoneEndpointsOf(deployments); + var endpoints = controller.routing().readZoneEndpointsOf(deployments); if ( ! endpoints.containsKey(zoneId)) { logger.log(WARNING, "Endpoints for the deployment to test vanished again, while it was still active!"); return Optional.of(error); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java index 23c69719d72..6ba8ad5bf36 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java @@ -21,9 +21,9 @@ import com.yahoo.vespa.hosted.controller.api.integration.deployment.TestReport; import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterCloud; import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterId; import com.yahoo.vespa.hosted.controller.application.ApplicationList; -import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; +import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackageDiff; import com.yahoo.vespa.hosted.controller.persistence.BufferedLogStore; import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; @@ -49,7 +49,6 @@ import java.util.function.UnaryOperator; import java.util.logging.Level; import java.util.stream.Stream; -import static com.google.common.collect.ImmutableList.copyOf; import static com.yahoo.vespa.hosted.controller.deployment.Step.copyVespaLogs; import static com.yahoo.vespa.hosted.controller.deployment.Step.deactivateTester; import static com.yahoo.vespa.hosted.controller.deployment.Step.endStagingSetup; @@ -212,10 +211,10 @@ public class JobController { /** Returns a list of all instances of applications which have registered. */ public List<ApplicationId> instances() { - return copyOf(controller.applications().readable().stream() - .flatMap(application -> application.instances().values().stream()) - .map(Instance::id) - .iterator()); + return controller.applications().readable().stream() + .flatMap(application -> application.instances().values().stream()) + .map(Instance::id) + .collect(toUnmodifiableList()); } /** Returns all job types which have been run for the given application. */ @@ -298,21 +297,21 @@ public class JobController { /** Returns a list of all active runs for the given application. */ public List<Run> active(TenantAndApplicationId id) { - return copyOf(controller.applications().requireApplication(id).instances().keySet().stream() - .flatMap(name -> Stream.of(JobType.values()) - .map(type -> last(id.instance(name), type)) - .flatMap(Optional::stream) - .filter(run -> ! run.hasEnded())) - .iterator()); + return controller.applications().requireApplication(id).instances().keySet().stream() + .flatMap(name -> Stream.of(JobType.values()) + .map(type -> last(id.instance(name), type)) + .flatMap(Optional::stream) + .filter(run -> !run.hasEnded())) + .collect(toUnmodifiableList()); } /** Returns a list of all active runs for the given instance. */ public List<Run> active(ApplicationId id) { - return copyOf(Stream.of(JobType.values()) - .map(type -> last(id, type)) - .flatMap(Optional::stream) - .filter(run -> ! run.hasEnded()) - .iterator()); + return Stream.of(JobType.values()) + .map(type -> last(id, type)) + .flatMap(Optional::stream) + .filter(run -> !run.hasEnded()) + .collect(toUnmodifiableList()); } /** Returns the job status of the given job, possibly empty. */ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/NameServiceForwarder.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/NameServiceForwarder.java index 2c59211e50c..aecb1e7a2c1 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/NameServiceForwarder.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/NameServiceForwarder.java @@ -73,6 +73,11 @@ public class NameServiceForwarder { forward(new RemoveRecords(type, data), priority); } + /** Remove all records of given type, name and data */ + public void removeRecords(Record.Type type, RecordName name, RecordData data, NameServiceQueue.Priority priority) { + forward(new RemoveRecords(type, name, data), priority); + } + protected void forward(NameServiceRequest request, NameServiceQueue.Priority priority) { try (Lock lock = db.lockNameServiceQueue()) { NameServiceQueue queue = db.readNameServiceQueue(); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/RemoveRecords.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/RemoveRecords.java index 6bc67af5b98..f940d53fab3 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/RemoveRecords.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/RemoveRecords.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.dns; +import com.yahoo.vespa.hosted.controller.api.integration.dns.AliasTarget; import com.yahoo.vespa.hosted.controller.api.integration.dns.NameService; import com.yahoo.vespa.hosted.controller.api.integration.dns.Record; import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordData; @@ -12,7 +13,11 @@ import java.util.Objects; import java.util.Optional; /** - * Permanently removes all matching records by type and name or data. + * Permanently removes all matching records by type and matching either: + * + * - name and data + * - only name + * - only data * * @author mpolden */ @@ -30,13 +35,17 @@ public class RemoveRecords implements NameServiceRequest { this(type, Optional.empty(), Optional.of(data)); } + public RemoveRecords(Record.Type type, RecordName name, RecordData data) { + this(type, Optional.of(name), Optional.of(data)); + } + /** DO NOT USE. Public for serialization purposes */ public RemoveRecords(Record.Type type, Optional<RecordName> name, Optional<RecordData> data) { this.type = Objects.requireNonNull(type, "type must be non-null"); this.name = Objects.requireNonNull(name, "name must be non-null"); this.data = Objects.requireNonNull(data, "data must be non-null"); - if (name.isPresent() == data.isPresent()) { - throw new IllegalArgumentException("exactly one of name or data must be non-empty"); + if (name.isEmpty() && data.isEmpty()) { + throw new IllegalArgumentException("at least one of name and data must be non-empty"); } } @@ -55,8 +64,23 @@ public class RemoveRecords implements NameServiceRequest { @Override public void dispatchTo(NameService nameService) { List<Record> records = new ArrayList<>(); - name.ifPresent(n -> records.addAll(nameService.findRecords(type, n))); - data.ifPresent(d -> records.addAll(nameService.findRecords(type, d))); + if (name.isPresent() && data.isPresent()) { + nameService.findRecords(type, name.get()) + .stream() + .filter(record -> { + // Records to remove must match both name and data fields + String dataValue = record.data().asString(); + // If we're comparing an ALIAS record we have to unpack it to access the target name + if (record.type() == Record.Type.ALIAS) { + dataValue = AliasTarget.unpack(record.data()).name().value(); + } + return fqdn(dataValue).equals(fqdn(data.get().asString())); + }) + .forEach(records::add); + } else { + name.ifPresent(n -> records.addAll(nameService.findRecords(type, n))); + data.ifPresent(d -> records.addAll(nameService.findRecords(type, d))); + } nameService.removeRecords(records); } @@ -82,4 +106,8 @@ public class RemoveRecords implements NameServiceRequest { return Objects.hash(type, name, data); } + private static String fqdn(String name) { + return name.endsWith(".") ? name : name + "."; + } + } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CloudEventTracker.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CloudEventTracker.java index 34f41ce2784..98e9fc7c159 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CloudEventTracker.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CloudEventTracker.java @@ -55,7 +55,7 @@ public class CloudEventTracker extends ControllerMaintainer { private void deprovisionAffectedHosts(String region, CloudEvent event) { for (var zone : zonesByCloudNativeRegion.get(region)) { for (var node : nodeRepository.list(zone.getId(), NodeFilter.all())) { - if (!affects(node, event)) continue; + if (!deprovision(node, event)) continue; log.info("Retiring and deprovisioning " + node.hostname().value() + " in " + zone.getId() + ": Affected by maintenance event " + event.instanceEventId); nodeRepository.retire(zone.getId(), node.hostname().value(), true, true); @@ -63,8 +63,9 @@ public class CloudEventTracker extends ControllerMaintainer { } } - private static boolean affects(Node node, CloudEvent event) { + private static boolean deprovision(Node node, CloudEvent event) { if (!node.type().isHost()) return false; // Non-hosts are never affected + if (node.wantToRetire() && node.wantToDeprovision()) return false; // Already deprovisioning return event.affectedInstances.stream() .anyMatch(instance -> node.hostname().value().contains(instance)); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/UserManagementMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/UserManagementMaintainer.java index 870e3af678f..5f6f917bc75 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/UserManagementMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/UserManagementMaintainer.java @@ -36,11 +36,8 @@ public class UserManagementMaintainer extends ControllerMaintainer { @Override protected double maintain() { findLeftoverRoles().forEach(role -> { - /* - Log discrepancy now - TODO: userManagement.deleteRole(role); - */ - logger.warning(String.format("Found unexpected role %s - Please investigate", role.toString())); + logger.warning(String.format("Found unexpected %s - Deleting", role.toString())); + userManagement.deleteRole(role); }); return 1.0; } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java index 652f85a7508..e10dcfd3b3b 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java @@ -29,7 +29,7 @@ import com.yahoo.vespa.hosted.controller.deployment.Step; import com.yahoo.vespa.hosted.controller.dns.NameServiceQueue; import com.yahoo.vespa.hosted.controller.api.integration.vcmr.VespaChangeRequest; import com.yahoo.vespa.hosted.controller.notification.Notification; -import com.yahoo.vespa.hosted.controller.routing.GlobalRouting; +import com.yahoo.vespa.hosted.controller.routing.RoutingStatus; import com.yahoo.vespa.hosted.controller.routing.RoutingPolicy; import com.yahoo.vespa.hosted.controller.routing.RoutingPolicyId; import com.yahoo.vespa.hosted.controller.routing.ZoneRoutingPolicy; @@ -535,7 +535,7 @@ public class CuratorDb { public ZoneRoutingPolicy readZoneRoutingPolicy(ZoneId zone) { return readSlime(zoneRoutingPolicyPath(zone)).map(data -> zoneRoutingPolicySerializer.fromSlime(zone, data)) - .orElse(new ZoneRoutingPolicy(zone, GlobalRouting.DEFAULT_STATUS)); + .orElse(new ZoneRoutingPolicy(zone, RoutingStatus.DEFAULT)); } // -------------- Application endpoint certificates ---------------------------- diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializer.java index d707b769f29..04d1a4c7433 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializer.java @@ -11,10 +11,9 @@ import com.yahoo.slime.Inspector; import com.yahoo.slime.Slime; import com.yahoo.slime.SlimeUtils; import com.yahoo.vespa.hosted.controller.application.EndpointId; -import com.yahoo.vespa.hosted.controller.routing.GlobalRouting; +import com.yahoo.vespa.hosted.controller.routing.RoutingStatus; import com.yahoo.vespa.hosted.controller.routing.RoutingPolicy; import com.yahoo.vespa.hosted.controller.routing.RoutingPolicyId; -import com.yahoo.vespa.hosted.controller.routing.Status; import java.time.Instant; import java.util.Collections; @@ -66,7 +65,7 @@ public class RoutingPolicySerializer { var applicationEndpointsArray = policyObject.setArray(applicationEndpointsField); policy.applicationEndpoints().forEach(endpointId -> applicationEndpointsArray.addString(endpointId.id())); policyObject.setBool(loadBalancerActiveField, policy.status().isActive()); - globalRoutingToSlime(policy.status().globalRouting(), policyObject.setObject(globalRoutingField)); + globalRoutingToSlime(policy.status().routingStatus(), policyObject.setObject(globalRoutingField)); }); return slime; } @@ -88,23 +87,23 @@ public class RoutingPolicySerializer { SlimeUtils.optionalString(inspect.field(dnsZoneField)), instanceEndpoints, applicationEndpoints, - new Status(inspect.field(loadBalancerActiveField).asBool(), - globalRoutingFromSlime(inspect.field(globalRoutingField))))); + new RoutingPolicy.Status(inspect.field(loadBalancerActiveField).asBool(), + globalRoutingFromSlime(inspect.field(globalRoutingField))))); }); return Collections.unmodifiableMap(policies); } - public void globalRoutingToSlime(GlobalRouting globalRouting, Cursor object) { - object.setString(statusField, globalRouting.status().name()); - object.setString(agentField, globalRouting.agent().name()); - object.setLong(changedAtField, globalRouting.changedAt().toEpochMilli()); + public void globalRoutingToSlime(RoutingStatus routingStatus, Cursor object) { + object.setString(statusField, routingStatus.value().name()); + object.setString(agentField, routingStatus.agent().name()); + object.setLong(changedAtField, routingStatus.changedAt().toEpochMilli()); } - public GlobalRouting globalRoutingFromSlime(Inspector object) { - var status = GlobalRouting.Status.valueOf(object.field(statusField).asString()); - var agent = GlobalRouting.Agent.valueOf(object.field(agentField).asString()); + public RoutingStatus globalRoutingFromSlime(Inspector object) { + var status = RoutingStatus.Value.valueOf(object.field(statusField).asString()); + var agent = RoutingStatus.Agent.valueOf(object.field(agentField).asString()); var changedAt = SlimeUtils.optionalInstant(object.field(changedAtField)).orElse(Instant.EPOCH); - return new GlobalRouting(status, agent, changedAt); + return new RoutingStatus(status, agent, changedAt); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ZoneRoutingPolicySerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ZoneRoutingPolicySerializer.java index 957c9166d28..5932c54650b 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ZoneRoutingPolicySerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ZoneRoutingPolicySerializer.java @@ -37,7 +37,7 @@ public class ZoneRoutingPolicySerializer { public Slime toSlime(ZoneRoutingPolicy policy) { var slime = new Slime(); var root = slime.setObject(); - routingPolicySerializer.globalRoutingToSlime(policy.globalRouting(), root.setObject(GLOBAL_ROUTING_FIELD)); + routingPolicySerializer.globalRoutingToSlime(policy.routingStatus(), root.setObject(GLOBAL_ROUTING_FIELD)); return slime; } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index 2845dd53b24..ef129dd76f7 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -100,7 +100,7 @@ import com.yahoo.vespa.hosted.controller.persistence.SupportAccessSerializer; import com.yahoo.vespa.hosted.controller.rotation.RotationId; import com.yahoo.vespa.hosted.controller.rotation.RotationState; import com.yahoo.vespa.hosted.controller.rotation.RotationStatus; -import com.yahoo.vespa.hosted.controller.routing.GlobalRouting; +import com.yahoo.vespa.hosted.controller.routing.RoutingStatus; import com.yahoo.vespa.hosted.controller.security.AccessControlRequests; import com.yahoo.vespa.hosted.controller.security.Credentials; import com.yahoo.vespa.hosted.controller.support.access.SupportAccess; @@ -137,7 +137,6 @@ import java.util.Base64; import java.util.Collection; import java.util.Comparator; import java.util.HashSet; -import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Optional; @@ -1164,8 +1163,8 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { })); } - // Global endpoints - globalEndpointsToSlime(object, instance); + // Rotation ID + addRotationId(object, instance); // Deployments sorted according to deployment spec List<Deployment> deployments = deploymentSpec.instance(instance.name()) @@ -1195,23 +1194,8 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { } } - // TODO(mpolden): Remove once legacy dashboard and integration tests stop expecting these fields - private void globalEndpointsToSlime(Cursor object, Instance instance) { - var globalEndpointUrls = new LinkedHashSet<String>(); - - // Add global endpoints backed by rotations - controller.routing().endpointsOf(instance.id()) - .requiresRotation() - .not().legacy() // Hide legacy names - .asList().stream() - .map(Endpoint::url) - .map(URI::toString) - .forEach(globalEndpointUrls::add); - - - var globalRotationsArray = object.setArray("globalRotations"); - globalEndpointUrls.forEach(globalRotationsArray::addString); - + // TODO(mpolden): Remove once MultiRegionTest stops expecting this field + private void addRotationId(Cursor object, Instance instance) { // Legacy field. Identifies the first assigned rotation, if any. instance.rotations().stream() .map(AssignedRotation::rotationId) @@ -1267,8 +1251,8 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { application.majorVersion().ifPresent(majorVersion -> object.setLong("majorVersion", majorVersion)); - // Global endpoint - globalEndpointsToSlime(object, instance); + // Rotation ID + addRotationId(object, instance); // Deployments sorted according to deployment spec List<Deployment> deployments = @@ -1387,7 +1371,7 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { // Add zone endpoints boolean legacyEndpoints = request.getBooleanProperty("includeLegacyEndpoints"); var endpointArray = response.setArray("endpoints"); - EndpointList zoneEndpoints = controller.routing().endpointsOf(deploymentId) + EndpointList zoneEndpoints = controller.routing().readEndpointsOf(deploymentId) .scope(Endpoint.Scope.zone); if (!legacyEndpoints) { zoneEndpoints = zoneEndpoints.not().legacy(); @@ -1395,13 +1379,13 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { for (var endpoint : controller.routing().directEndpoints(zoneEndpoints, deploymentId.applicationId())) { toSlime(endpoint, endpointArray.addObject()); } - // Add global endpoints - EndpointList globalEndpoints = controller.routing().endpointsOf(application, deploymentId.applicationId().instance()) - .targets(deploymentId); + // Add declared endpoints + EndpointList declaredEndpoints = controller.routing().declaredEndpointsOf(application) + .targets(deploymentId); if (!legacyEndpoints) { - globalEndpoints = globalEndpoints.not().legacy(); + declaredEndpoints = declaredEndpoints.not().legacy(); } - for (var endpoint : controller.routing().directEndpoints(globalEndpoints, deploymentId.applicationId())) { + for (var endpoint : controller.routing().directEndpoints(declaredEndpoints, deploymentId.applicationId())) { toSlime(endpoint, endpointArray.addObject()); } @@ -1561,16 +1545,16 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { /** Set the global endpoint status for given deployment. This only applies to global endpoints backed by a cloud service */ private void setGlobalEndpointStatus(DeploymentId deployment, boolean inService, HttpRequest request) { - var agent = isOperator(request) ? GlobalRouting.Agent.operator : GlobalRouting.Agent.tenant; - var status = inService ? GlobalRouting.Status.in : GlobalRouting.Status.out; - controller.routing().policies().setGlobalRoutingStatus(deployment, status, agent); + var agent = isOperator(request) ? RoutingStatus.Agent.operator : RoutingStatus.Agent.tenant; + var status = inService ? RoutingStatus.Value.in : RoutingStatus.Value.out; + controller.routing().policies().setRoutingStatus(deployment, status, agent); } /** Set the global rotation status for given deployment. This only applies to global endpoints backed by a rotation */ private void setGlobalRotationStatus(DeploymentId deployment, boolean inService, HttpRequest request) { var requestData = toSlime(request.getData()).get(); var reason = mandatory("reason", requestData).asString(); - var agent = isOperator(request) ? GlobalRouting.Agent.operator : GlobalRouting.Agent.tenant; + var agent = isOperator(request) ? RoutingStatus.Agent.operator : RoutingStatus.Agent.tenant; long timestamp = controller.clock().instant().getEpochSecond(); var status = inService ? EndpointStatus.Status.in : EndpointStatus.Status.out; var endpointStatus = new EndpointStatus(status, reason, agent.name(), timestamp); @@ -2077,7 +2061,7 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { return new SlimeJsonResponse(testConfigSerializer.configSlime(id, type, false, - controller.routing().zoneEndpointsOf(deployments), + controller.routing().readZoneEndpointsOf(deployments), controller.applications().reachableContentClustersByZone(deployments))); } @@ -2694,7 +2678,8 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { private static String endpointScopeString(Endpoint.Scope scope) { switch (scope) { - case weighted: return "region"; + case weighted: return "weighted"; + case application: return "application"; case global: return "global"; case zone: return "zone"; } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiHandler.java index 34fcda3bff8..45abf7f2946 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiHandler.java @@ -21,10 +21,13 @@ import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.Instance; import com.yahoo.vespa.hosted.controller.api.application.v4.model.EndpointStatus; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; +import com.yahoo.vespa.hosted.controller.api.role.Role; +import com.yahoo.vespa.hosted.controller.api.role.RoleDefinition; +import com.yahoo.vespa.hosted.controller.api.role.SecurityContext; import com.yahoo.vespa.hosted.controller.application.Endpoint; import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; import com.yahoo.vespa.hosted.controller.auditlog.AuditLoggingRequestHandler; -import com.yahoo.vespa.hosted.controller.routing.GlobalRouting; +import com.yahoo.vespa.hosted.controller.routing.RoutingStatus; import com.yahoo.yolean.Exceptions; import java.net.URI; @@ -38,8 +41,8 @@ import java.util.stream.Collectors; import java.util.stream.Stream; /** - * This implements the /routing/v1 API, which provides operator with global routing control at both zone- and - * deployment-level. + * This implements the /routing/v1 API, which provides operators and tenants routing control at both zone- (operator + * only) and deployment-level. * * @author mpolden */ @@ -58,8 +61,8 @@ public class RoutingApiHandler extends AuditLoggingRequestHandler { var path = new Path(request.getUri()); switch (request.getMethod()) { case GET: return get(path, request); - case POST: return post(path); - case DELETE: return delete(path); + case POST: return post(path, request); + case DELETE: return delete(path, request); default: return ErrorResponse.methodNotAllowed("Method '" + request.getMethod() + "' is not supported"); } } catch (IllegalArgumentException e) { @@ -70,14 +73,14 @@ public class RoutingApiHandler extends AuditLoggingRequestHandler { } } - private HttpResponse delete(Path path) { - if (path.matches("/routing/v1/inactive/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}")) return setDeploymentStatus(path, true); + private HttpResponse delete(Path path, HttpRequest request) { + if (path.matches("/routing/v1/inactive/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}")) return setDeploymentStatus(path, true, request); if (path.matches("/routing/v1/inactive/environment/{environment}/region/{region}")) return setZoneStatus(path, true); return ErrorResponse.notFoundError("Nothing at " + path); } - private HttpResponse post(Path path) { - if (path.matches("/routing/v1/inactive/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}")) return setDeploymentStatus(path, false); + private HttpResponse post(Path path, HttpRequest request) { + if (path.matches("/routing/v1/inactive/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}")) return setDeploymentStatus(path, false, request); if (path.matches("/routing/v1/inactive/environment/{environment}/region/{region}")) return setZoneStatus(path, false); return ErrorResponse.notFoundError("Nothing at " + path); } @@ -96,7 +99,7 @@ public class RoutingApiHandler extends AuditLoggingRequestHandler { private HttpResponse endpoints(Path path) { var instanceId = instanceFrom(path); - var endpoints = controller.routing().endpointsOf(instanceId) + var endpoints = controller.routing().readDeclaredEndpointsOf(instanceId) .sortedBy(Comparator.comparing(Endpoint::name)) .asList(); @@ -209,9 +212,9 @@ public class RoutingApiHandler extends AuditLoggingRequestHandler { private HttpResponse setZoneStatus(Path path, boolean in) { var zone = zoneFrom(path); - if (controller.zoneRegistry().zones().directlyRouted().ids().contains(zone)) { - var status = in ? GlobalRouting.Status.in : GlobalRouting.Status.out; - controller.routing().policies().setGlobalRoutingStatus(zone, status); + if (exclusiveRoutingIn(zone)) { + var status = in ? RoutingStatus.Value.in : RoutingStatus.Value.out; + controller.routing().policies().setRoutingStatus(zone, status); } else { controller.serviceRegistry().configServer().setGlobalRotationStatus(zone, in); } @@ -228,35 +231,36 @@ public class RoutingApiHandler extends AuditLoggingRequestHandler { } private void toSlime(ZoneId zone, Cursor zoneObject) { - if (controller.zoneRegistry().zones().directlyRouted().ids().contains(zone)) { + if (exclusiveRoutingIn(zone)) { var zonePolicy = controller.routing().policies().get(zone); - zoneStatusToSlime(zoneObject, zonePolicy.zone(), zonePolicy.globalRouting(), RoutingMethod.exclusive); + zoneStatusToSlime(zoneObject, zonePolicy.zone(), zonePolicy.routingStatus(), RoutingMethod.exclusive); } else { // Rotation status per zone only exposes in/out status, no agent or time of change. var in = controller.serviceRegistry().configServer().getGlobalRotationStatus(zone); - var globalRouting = new GlobalRouting(in ? GlobalRouting.Status.in : GlobalRouting.Status.out, - GlobalRouting.Agent.operator, Instant.EPOCH); + var globalRouting = new RoutingStatus(in ? RoutingStatus.Value.in : RoutingStatus.Value.out, + RoutingStatus.Agent.operator, Instant.EPOCH); zoneStatusToSlime(zoneObject, zone, globalRouting, RoutingMethod.shared); } } - private HttpResponse setDeploymentStatus(Path path, boolean in) { + private HttpResponse setDeploymentStatus(Path path, boolean in, HttpRequest request) { var deployment = deploymentFrom(path); var instance = controller.applications().requireInstance(deployment.applicationId()); - var status = in ? GlobalRouting.Status.in : GlobalRouting.Status.out; - var agent = GlobalRouting.Agent.operator; // Always operator as this is an operator API + var status = in ? RoutingStatus.Value.in : RoutingStatus.Value.out; + var agent = isOperator(request) ? RoutingStatus.Agent.operator : RoutingStatus.Agent.tenant; requireDeployment(deployment, instance); - // Set rotation status, if rotations can route to this zone - if (rotationCanRouteTo(deployment.zoneId())) { - var endpointStatus = new EndpointStatus(in ? EndpointStatus.Status.in : EndpointStatus.Status.out, "", + if (sharedRoutingIn(deployment.zoneId())) { + // Set rotation status + var endpointStatus = new EndpointStatus(in ? EndpointStatus.Status.in : EndpointStatus.Status.out, + "", agent.name(), controller.clock().instant().getEpochSecond()); controller.routing().setGlobalRotationStatus(deployment, endpointStatus); + } else { + // Set policy status + controller.routing().policies().setRoutingStatus(deployment, status, agent); } - - // Set policy status - controller.routing().policies().setGlobalRoutingStatus(deployment, status, agent); return new MessageResponse("Set global routing status for " + deployment + " to " + (in ? "IN" : "OUT")); } @@ -297,61 +301,63 @@ public class RoutingApiHandler extends AuditLoggingRequestHandler { } - private Optional<GlobalRouting> sharedGlobalRoutingStatus(DeploymentId deploymentId) { - if (rotationCanRouteTo(deploymentId.zoneId())) { + private Optional<RoutingStatus> sharedGlobalRoutingStatus(DeploymentId deploymentId) { + if (sharedRoutingIn(deploymentId.zoneId())) { var rotationStatus = controller.routing().globalRotationStatus(deploymentId); // Status is equal across all global endpoints, as the status is per deployment, not per endpoint. var endpointStatus = rotationStatus.values().stream().findFirst(); if (endpointStatus.isPresent()) { var changedAt = Instant.ofEpochSecond(endpointStatus.get().getEpoch()); - GlobalRouting.Agent agent; + RoutingStatus.Agent agent; try { - agent = GlobalRouting.Agent.valueOf(endpointStatus.get().getAgent()); + agent = RoutingStatus.Agent.valueOf(endpointStatus.get().getAgent()); } catch (IllegalArgumentException e) { - agent = GlobalRouting.Agent.unknown; + agent = RoutingStatus.Agent.unknown; } var status = endpointStatus.get().getStatus() == EndpointStatus.Status.in - ? GlobalRouting.Status.in - : GlobalRouting.Status.out; - return Optional.of(new GlobalRouting(status, agent, changedAt)); + ? RoutingStatus.Value.in + : RoutingStatus.Value.out; + return Optional.of(new RoutingStatus(status, agent, changedAt)); } } return Optional.empty(); } - private List<GlobalRouting> directGlobalRoutingStatus(DeploymentId deploymentId) { + private List<RoutingStatus> directGlobalRoutingStatus(DeploymentId deploymentId) { return controller.routing().policies().get(deploymentId).values().stream() .filter(p -> ! p.instanceEndpoints().isEmpty()) // This policy does not apply to a global endpoint - .filter(p -> controller.zoneRegistry().routingMethods(p.id().zone()).contains(RoutingMethod.exclusive)) - .map(p -> p.status().globalRouting()) + .filter(p -> exclusiveRoutingIn(p.id().zone())) + .map(p -> p.status().routingStatus()) .collect(Collectors.toList()); } - /** Returns whether a rotation can route traffic to given zone */ - private boolean rotationCanRouteTo(ZoneId zone) { - // A system may support multiple routing methods, i.e. it has both exclusively routed zones and zones using - // shared routing. When changing or reading routing status in the context of a specific deployment, rotation - // status should only be considered if the zone supports shared routing. + /** Returns whether given zone uses exclusive routing */ + private boolean exclusiveRoutingIn(ZoneId zone) { + return controller.zoneRegistry().routingMethods(zone).contains(RoutingMethod.exclusive); + } + + /** Returns whether given zone uses shared routing */ + private boolean sharedRoutingIn(ZoneId zone) { return controller.zoneRegistry().routingMethods(zone).stream().anyMatch(RoutingMethod::isShared); } - private static void zoneStatusToSlime(Cursor object, ZoneId zone, GlobalRouting globalRouting, RoutingMethod method) { + private static void zoneStatusToSlime(Cursor object, ZoneId zone, RoutingStatus routingStatus, RoutingMethod method) { object.setString("routingMethod", asString(method)); object.setString("environment", zone.environment().value()); object.setString("region", zone.region().value()); - object.setString("status", asString(globalRouting.status())); - object.setString("agent", asString(globalRouting.agent())); - object.setLong("changedAt", globalRouting.changedAt().toEpochMilli()); + object.setString("status", asString(routingStatus.value())); + object.setString("agent", asString(routingStatus.agent())); + object.setLong("changedAt", routingStatus.changedAt().toEpochMilli()); } - private static void deploymentStatusToSlime(Cursor object, DeploymentId deployment, GlobalRouting globalRouting, RoutingMethod method) { + private static void deploymentStatusToSlime(Cursor object, DeploymentId deployment, RoutingStatus routingStatus, RoutingMethod method) { object.setString("routingMethod", asString(method)); object.setString("instance", deployment.applicationId().serializedForm()); object.setString("environment", deployment.zoneId().environment().value()); object.setString("region", deployment.zoneId().region().value()); - object.setString("status", asString(globalRouting.status())); - object.setString("agent", asString(globalRouting.agent())); - object.setLong("changedAt", globalRouting.changedAt().toEpochMilli()); + object.setString("status", asString(routingStatus.value())); + object.setString("agent", asString(routingStatus.agent())); + object.setLong("changedAt", routingStatus.changedAt().toEpochMilli()); } private static void endpointToSlime(Cursor object, Endpoint endpoint) { @@ -397,19 +403,29 @@ public class RoutingApiHandler extends AuditLoggingRequestHandler { return deployment; } + private static boolean isOperator(HttpRequest request) { + SecurityContext securityContext = Optional.ofNullable(request.getJDiscRequest().context().get(SecurityContext.ATTRIBUTE_NAME)) + .filter(SecurityContext.class::isInstance) + .map(SecurityContext.class::cast) + .orElseThrow(() -> new IllegalArgumentException("Attribute '" + SecurityContext.ATTRIBUTE_NAME + "' was not set on request")); + return securityContext.roles().stream() + .map(Role::definition) + .anyMatch(definition -> definition == RoleDefinition.hostedOperator); + } + private static boolean isRecursive(HttpRequest request) { return "true".equals(request.getProperty("recursive")); } - private static String asString(GlobalRouting.Status status) { - switch (status) { + private static String asString(RoutingStatus.Value value) { + switch (value) { case in: return "in"; case out: return "out"; default: return "unknown"; } } - private static String asString(GlobalRouting.Agent agent) { + private static String asString(RoutingStatus.Agent agent) { switch (agent) { case operator: return "operator"; case system: return "system"; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingId.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingId.java index 343fa5417ce..67eafe6235d 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingId.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingId.java @@ -3,22 +3,30 @@ package com.yahoo.vespa.hosted.controller.routing; import com.yahoo.config.provision.ApplicationId; import com.yahoo.vespa.hosted.controller.application.EndpointId; +import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; import java.util.Objects; /** - * Unique identifier for a global routing table entry (instance x endpoint ID). + * Unique identifier for a instance routing table entry (instance x endpoint ID). * * @author mpolden */ public class RoutingId { + private final TenantAndApplicationId application; private final ApplicationId instance; private final EndpointId endpointId; - public RoutingId(ApplicationId instance, EndpointId endpointId) { - this.instance = Objects.requireNonNull(instance, "instance must be non-null"); + private RoutingId(ApplicationId instance, EndpointId endpointId) { + this.instance = Objects.requireNonNull(instance, "application must be non-null"); this.endpointId = Objects.requireNonNull(endpointId, "endpointId must be non-null"); + + application = TenantAndApplicationId.from(instance); + } + + public TenantAndApplicationId application() { + return application; } public ApplicationId instance() { @@ -33,14 +41,13 @@ public class RoutingId { public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; - RoutingId that = (RoutingId) o; - return instance.equals(that.instance) && - endpointId.equals(that.endpointId); + RoutingId routingId = (RoutingId) o; + return application.equals(routingId.application) && instance.equals(routingId.instance) && endpointId.equals(routingId.endpointId); } @Override public int hashCode() { - return Objects.hash(instance, endpointId); + return Objects.hash(application, instance, endpointId); } @Override diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java index e0c0df5234e..2a39ed08014 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java @@ -3,10 +3,12 @@ package com.yahoo.vespa.hosted.controller.routing; import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.zone.RoutingMethod; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.curator.Lock; +import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.integration.configserver.LoadBalancer; @@ -18,6 +20,8 @@ import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordName; import com.yahoo.vespa.hosted.controller.api.integration.dns.WeightedAliasTarget; import com.yahoo.vespa.hosted.controller.application.Endpoint; import com.yahoo.vespa.hosted.controller.application.EndpointId; +import com.yahoo.vespa.hosted.controller.application.EndpointList; +import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; import com.yahoo.vespa.hosted.controller.dns.NameServiceForwarder; import com.yahoo.vespa.hosted.controller.dns.NameServiceQueue.Priority; import com.yahoo.vespa.hosted.controller.dns.NameServiceRequest; @@ -26,17 +30,19 @@ import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; +import java.util.HashMap; import java.util.HashSet; import java.util.LinkedHashMap; import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; /** - * Updates routing policies and their associated DNS records based on an deployment's load balancers. + * Updates routing policies and their associated DNS records based on a deployment's load balancers. * * @author mortent * @author mpolden @@ -75,26 +81,31 @@ public class RoutingPolicies { } /** - * Refresh routing policies for application in given zone. This is idempotent and changes will only be performed if - * load balancers for given application have changed. + * Refresh routing policies for instance in given zone. This is idempotent and changes will only be performed if + * load balancers for given instance have changed. */ - public void refresh(ApplicationId application, DeploymentSpec deploymentSpec, ZoneId zone) { - var allocation = new LoadBalancerAllocation(application, zone, controller.serviceRegistry().configServer() - .getLoadBalancers(application, zone), - deploymentSpec); - var inactiveZones = inactiveZones(application, deploymentSpec); + public void refresh(ApplicationId instance, DeploymentSpec deploymentSpec, ZoneId zone) { + LoadBalancerAllocation allocation = new LoadBalancerAllocation(instance, zone, controller.serviceRegistry().configServer() + .getLoadBalancers(instance, zone), + deploymentSpec); + Set<ZoneId> inactiveZones = inactiveZones(instance, deploymentSpec); try (var lock = db.lockRoutingPolicies()) { removeGlobalDnsUnreferencedBy(allocation, lock); + removeApplicationDnsUnreferencedBy(allocation, lock); + storePoliciesOf(allocation, lock); removePoliciesUnreferencedBy(allocation, lock); - updateGlobalDnsOf(get(allocation.deployment.applicationId()).values(), inactiveZones, lock); + + Collection<RoutingPolicy> policies = get(allocation.deployment.applicationId()).values(); + updateGlobalDnsOf(policies, inactiveZones, lock); + updateApplicationDnsOf(policies, inactiveZones, lock); } } /** Set the status of all global endpoints in given zone */ - public void setGlobalRoutingStatus(ZoneId zone, GlobalRouting.Status status) { + public void setRoutingStatus(ZoneId zone, RoutingStatus.Value value) { try (var lock = db.lockRoutingPolicies()) { - db.writeZoneRoutingPolicy(new ZoneRoutingPolicy(zone, GlobalRouting.status(status, GlobalRouting.Agent.operator, + db.writeZoneRoutingPolicy(new ZoneRoutingPolicy(zone, RoutingStatus.create(value, RoutingStatus.Agent.operator, controller.clock().instant()))); Map<ApplicationId, Map<RoutingPolicyId, RoutingPolicy>> allPolicies = db.readRoutingPolicies(); for (var applicationPolicies : allPolicies.values()) { @@ -104,27 +115,28 @@ public class RoutingPolicies { } /** Set the status of all global endpoints for given deployment */ - public void setGlobalRoutingStatus(DeploymentId deployment, GlobalRouting.Status status, GlobalRouting.Agent agent) { + public void setRoutingStatus(DeploymentId deployment, RoutingStatus.Value value, RoutingStatus.Agent agent) { try (var lock = db.lockRoutingPolicies()) { var policies = get(deployment.applicationId()); var newPolicies = new LinkedHashMap<>(policies); for (var policy : policies.values()) { - if (!policy.id().zone().equals(deployment.zoneId())) continue; // Wrong zone - var newPolicy = policy.with(policy.status().with(GlobalRouting.status(status, agent, + if (!policy.appliesTo(deployment)) continue; + var newPolicy = policy.with(policy.status().with(RoutingStatus.create(value, agent, controller.clock().instant()))); newPolicies.put(policy.id(), newPolicy); } db.writeRoutingPolicies(deployment.applicationId(), newPolicies); updateGlobalDnsOf(newPolicies.values(), Set.of(), lock); + updateApplicationDnsOf(newPolicies.values(), Set.of(), lock); } } /** Update global DNS records for given policies */ private void updateGlobalDnsOf(Collection<RoutingPolicy> routingPolicies, Set<ZoneId> inactiveZones, @SuppressWarnings("unused") Lock lock) { - Map<RoutingId, List<RoutingPolicy>> routingTable = routingTableFrom(routingPolicies); + Map<RoutingId, List<RoutingPolicy>> routingTable = instanceRoutingTable(routingPolicies); for (Map.Entry<RoutingId, List<RoutingPolicy>> routeEntry : routingTable.entrySet()) { RoutingId routingId = routeEntry.getKey(); - controller.routing().endpointsOf(routingId.instance()) + controller.routing().readDeclaredEndpointsOf(routingId.instance()) .named(routingId.endpointId()) .not().requiresRotation() .forEach(endpoint -> updateGlobalDnsOf(endpoint, inactiveZones, routeEntry.getValue())); @@ -167,7 +179,6 @@ public class RoutingPolicies { Priority.normal)); } - /** Compute region endpoints and their targets from given policies */ private Collection<RegionEndpoint> computeRegionEndpoints(List<RoutingPolicy> policies, Set<ZoneId> inactiveZones) { Map<Endpoint, RegionEndpoint> endpoints = new LinkedHashMap<>(); @@ -178,8 +189,8 @@ public class RoutingPolicies { Endpoint regionEndpoint = policy.regionEndpointIn(controller.system(), routingMethod); var zonePolicy = db.readZoneRoutingPolicy(policy.id().zone()); long weight = 1; - if (isConfiguredOut(policy, zonePolicy, inactiveZones)) { - weight = 0; // A record with 0 weight will not received traffic. If all records within a group have 0 + if (isConfiguredOut(zonePolicy, policy, inactiveZones)) { + weight = 0; // A record with 0 weight will not receive traffic. If all records within a group have 0 // weight, traffic is routed to all records with equal probability. } var weightedTarget = new WeightedAliasTarget(policy.canonicalName(), policy.dnsZone().get(), @@ -193,6 +204,48 @@ public class RoutingPolicies { return endpoints.values(); } + + private void updateApplicationDnsOf(Collection<RoutingPolicy> routingPolicies, Set<ZoneId> inactiveZones, @SuppressWarnings("unused") Lock lock) { + // In the context of single deployment (which this is) there is only one routing policy per routing ID. I.e. + // there is no scenario where more than one deployment within an instance can be a member the same + // application-level endpoint. However, to allow this in the future the routing table remains + // Map<RoutingId, List<RoutingPolicy>> instead of Map<RoutingId, RoutingPolicy>. + Map<RoutingId, List<RoutingPolicy>> routingTable = applicationRoutingTable(routingPolicies); + if (routingTable.isEmpty()) return; + + Application application = controller.applications().requireApplication(routingTable.keySet().iterator().next().application()); + Map<DeploymentId, Map<EndpointId, Integer>> targetWeights = targetWeights(application); + Map<String, Set<AliasTarget>> targetsByEndpoint = new LinkedHashMap<>(); + for (Map.Entry<RoutingId, List<RoutingPolicy>> routeEntry : routingTable.entrySet()) { + RoutingId routingId = routeEntry.getKey(); + EndpointList endpoints = controller.routing().declaredEndpointsOf(application) + .scope(Endpoint.Scope.application) + .named(routingId.endpointId()); + if (endpoints.isEmpty()) continue; + if (endpoints.size() > 1) { + throw new IllegalArgumentException("Expected at most 1 endpoint with ID '" + routingId.endpointId() + + ", got " + endpoints.size()); + } + Endpoint endpoint = endpoints.asList().get(0); + for (var policy : routeEntry.getValue()) { + for (var target : endpoint.targets()) { + if (!policy.appliesTo(target.deployment())) continue; + int weight = target.weight(); + if (isConfiguredOut(policy, inactiveZones) && removableFromApplicationEndpoint(policy, application, targetWeights)) { + weight = 0; + } + WeightedAliasTarget weightedAliasTarget = new WeightedAliasTarget(policy.canonicalName(), policy.dnsZone().get(), + target.deployment().zoneId(), weight); + targetsByEndpoint.computeIfAbsent(endpoint.dnsName(), (k) -> new LinkedHashSet<>()) + .add(weightedAliasTarget); + } + } + } + targetsByEndpoint.forEach((applicationEndpoint, targets) -> { + controller.nameServiceForwarder().createAlias(RecordName.from(applicationEndpoint), targets, Priority.normal); + }); + } + /** Store routing policies for given load balancers */ private void storePoliciesOf(LoadBalancerAllocation allocation, @SuppressWarnings("unused") Lock lock) { var policies = new LinkedHashMap<>(get(allocation.deployment.applicationId())); @@ -201,12 +254,12 @@ public class RoutingPolicies { var policyId = new RoutingPolicyId(loadBalancer.application(), loadBalancer.cluster(), allocation.deployment.zoneId()); var existingPolicy = policies.get(policyId); var newPolicy = new RoutingPolicy(policyId, loadBalancer.hostname().get(), loadBalancer.dnsZone(), - allocation.endpointIdsOf(loadBalancer), - Set.of(), - new Status(isActive(loadBalancer), GlobalRouting.DEFAULT_STATUS)); + allocation.instanceEndpointsOf(loadBalancer), + allocation.applicationEndpointsOf(loadBalancer), + new RoutingPolicy.Status(isActive(loadBalancer), RoutingStatus.DEFAULT)); // Preserve global routing status for existing policy if (existingPolicy != null) { - newPolicy = newPolicy.with(newPolicy.status().with(existingPolicy.status().globalRouting())); + newPolicy = newPolicy.with(newPolicy.status().with(existingPolicy.status().routingStatus())); } updateZoneDnsOf(newPolicy); policies.put(newPolicy.id(), newPolicy); @@ -230,8 +283,7 @@ public class RoutingPolicies { var activeIds = allocation.asPolicyIds(); for (var policy : policies.values()) { // Leave active load balancers and irrelevant zones alone - if (activeIds.contains(policy.id()) || - !policy.id().zone().equals(allocation.deployment.zoneId())) continue; + if (activeIds.contains(policy.id()) || !policy.appliesTo(allocation.deployment)) continue; for (var endpoint : policy.zoneEndpointsIn(controller.system(), RoutingMethod.exclusive, controller.zoneRegistry())) { var dnsName = endpoint.dnsName(); nameServiceForwarderIn(allocation.deployment.zoneId()).removeRecords(Record.Type.CNAME, @@ -243,39 +295,122 @@ public class RoutingPolicies { db.writeRoutingPolicies(allocation.deployment.applicationId(), newPolicies); } - /** Remove unreferenced global endpoints from DNS */ + /** Remove unreferenced instance endpoints from DNS */ private void removeGlobalDnsUnreferencedBy(LoadBalancerAllocation allocation, @SuppressWarnings("unused") Lock lock) { - var zonePolicies = get(allocation.deployment).values(); - var removalCandidates = new HashSet<>(routingTableFrom(zonePolicies).keySet()); - var activeRoutingIds = routingIdsFrom(allocation); + Collection<RoutingPolicy> zonePolicies = get(allocation.deployment).values(); + Set<RoutingId> removalCandidates = new HashSet<>(instanceRoutingTable(zonePolicies).keySet()); + Set<RoutingId> activeRoutingIds = instanceRoutingIds(allocation); removalCandidates.removeAll(activeRoutingIds); for (var id : removalCandidates) { - var endpoints = controller.routing().endpointsOf(id.instance()) - .not().requiresRotation() - .named(id.endpointId()); - var forwarder = nameServiceForwarderIn(allocation.deployment.zoneId()); + EndpointList endpoints = controller.routing().readDeclaredEndpointsOf(id.instance()) + .not().requiresRotation() + .named(id.endpointId()); + NameServiceForwarder forwarder = nameServiceForwarderIn(allocation.deployment.zoneId()); + // This removes all ALIAS records having this DNS name. There is no attempt to delete only the entry for the + // affected zone. Instead, the correct set of records is (re)created by updateGlobalDnsOf endpoints.forEach(endpoint -> forwarder.removeRecords(Record.Type.ALIAS, RecordName.from(endpoint.dnsName()), Priority.normal)); } } + /** Remove unreferenced application endpoints in given allocation from DNS */ + private void removeApplicationDnsUnreferencedBy(LoadBalancerAllocation allocation, @SuppressWarnings("unused") Lock lock) { + Collection<RoutingPolicy> zonePolicies = get(allocation.deployment).values(); + Map<RoutingId, List<RoutingPolicy>> routingTable = applicationRoutingTable(zonePolicies); + Set<RoutingId> removalCandidates = new HashSet<>(routingTable.keySet()); + Set<RoutingId> activeRoutingIds = applicationRoutingIds(allocation); + removalCandidates.removeAll(activeRoutingIds); + for (var id : removalCandidates) { + TenantAndApplicationId application = TenantAndApplicationId.from(id.instance()); + EndpointList endpoints = controller.routing() + .readDeclaredEndpointsOf(application) + .named(id.endpointId()); + List<RoutingPolicy> policies = routingTable.get(id); + for (var policy : policies) { + if (!policy.appliesTo(allocation.deployment)) continue; + NameServiceForwarder forwarder = nameServiceForwarderIn(policy.id().zone()); + endpoints.forEach(endpoint -> forwarder.removeRecords(Record.Type.ALIAS, + RecordName.from(endpoint.dnsName()), + RecordData.fqdn(policy.canonicalName().value()), + Priority.normal)); + } + } + } + + /** Returns whether we disable given policy from its application endpoints, taking weights and status of other instances into account */ + private boolean removableFromApplicationEndpoint(RoutingPolicy policy, Application application, Map<DeploymentId, Map<EndpointId, Integer>> targetWeights) { + List<RoutingPolicy> relatedPolicies = application.productionInstances().keySet().stream() + .filter(instanceName -> !policy.id().owner().instance().equals(instanceName)) + .map(instanceName -> application.id().instance(instanceName)) + .flatMap(instance -> get(instance).values().stream()) + .filter(relatedPolicy -> relatedPolicy.id().zone().equals(policy.id().zone()) && + relatedPolicy.id().cluster().equals(policy.id().cluster())) + .collect(Collectors.toUnmodifiableList()); + for (var endpointId : policy.applicationEndpoints()) { + boolean anyIn = relatedPolicies.stream() + .anyMatch(rp -> rp.applicationEndpoints().contains(endpointId) && + rp.status().routingStatus().value() == RoutingStatus.Value.in && + targetWeights.get(rp.id().deployment()) + .get(endpointId) > 0); + if (!anyIn) { + return false; + } + } + return true; + } + + /** Returns target weights of application endpoints in given application, grouped by deployment */ + private Map<DeploymentId, Map<EndpointId, Integer>> targetWeights(Application application) { + Map<DeploymentId, Map<EndpointId, Integer>> weights = new HashMap<>(); + for (var endpoint : application.deploymentSpec().endpoints()) { + for (var target : endpoint.targets()) { + weights.computeIfAbsent(new DeploymentId(application.id().instance(target.instance()), + ZoneId.from(Environment.prod, target.region())), + (k) -> new HashMap<>()) + .put(EndpointId.of(endpoint.endpointId()), target.weight()); + } + } + return weights; + } + + private Set<RoutingId> instanceRoutingIds(LoadBalancerAllocation allocation) { + return routingIdsFrom(allocation, false); + } + + private Set<RoutingId> applicationRoutingIds(LoadBalancerAllocation allocation) { + return routingIdsFrom(allocation, true); + } + /** Compute routing IDs from given load balancers */ - private static Set<RoutingId> routingIdsFrom(LoadBalancerAllocation allocation) { + private static Set<RoutingId> routingIdsFrom(LoadBalancerAllocation allocation, boolean applicationLevel) { Set<RoutingId> routingIds = new LinkedHashSet<>(); for (var loadBalancer : allocation.loadBalancers) { - for (var endpointId : allocation.endpointIdsOf(loadBalancer)) { - routingIds.add(new RoutingId(loadBalancer.application(), endpointId)); + Set<EndpointId> endpoints = applicationLevel + ? allocation.applicationEndpointsOf(loadBalancer) + : allocation.instanceEndpointsOf(loadBalancer); + for (var endpointId : endpoints) { + routingIds.add(RoutingId.of(loadBalancer.application(), endpointId)); } } return Collections.unmodifiableSet(routingIds); } - /** Compute a routing table from given policies */ - private static Map<RoutingId, List<RoutingPolicy>> routingTableFrom(Collection<RoutingPolicy> routingPolicies) { - var routingTable = new LinkedHashMap<RoutingId, List<RoutingPolicy>>(); + /** Compute a routing table for instance-level endpoints from given policies */ + private static Map<RoutingId, List<RoutingPolicy>> instanceRoutingTable(Collection<RoutingPolicy> routingPolicies) { + return routingTable(routingPolicies, false); + } + + /** Compute a routing table for application-level endpoints from given policies */ + private static Map<RoutingId, List<RoutingPolicy>> applicationRoutingTable(Collection<RoutingPolicy> routingPolicies) { + return routingTable(routingPolicies, true); + } + + private static Map<RoutingId, List<RoutingPolicy>> routingTable(Collection<RoutingPolicy> routingPolicies, boolean applicationLevel) { + Map<RoutingId, List<RoutingPolicy>> routingTable = new LinkedHashMap<>(); for (var policy : routingPolicies) { - for (var endpoint : policy.instanceEndpoints()) { - var id = new RoutingId(policy.id().owner(), endpoint); + Set<EndpointId> endpoints = applicationLevel ? policy.applicationEndpoints() : policy.instanceEndpoints(); + for (var endpoint : endpoints) { + RoutingId id = RoutingId.of(policy.id().owner(), endpoint); routingTable.computeIfAbsent(id, k -> new ArrayList<>()) .add(policy); } @@ -283,14 +418,23 @@ public class RoutingPolicies { return Collections.unmodifiableMap(routingTable); } - /** Returns whether the global routing status of given policy is configured to be {@link GlobalRouting.Status#out} */ - private static boolean isConfiguredOut(RoutingPolicy policy, ZoneRoutingPolicy zonePolicy, Set<ZoneId> inactiveZones) { - // A deployment is can be configured out at any of the following levels: - // - zone level (ZoneRoutingPolicy) + /** Returns whether the endpoints of given policy are globally configured {@link RoutingStatus.Value#out} */ + private static boolean isConfiguredOut(ZoneRoutingPolicy zonePolicy, RoutingPolicy policy, Set<ZoneId> inactiveZones) { + return isConfiguredOut(policy, Optional.of(zonePolicy), inactiveZones); + } + + /** Returns whether the endpoints of given policy are configured {@link RoutingStatus.Value#out} */ + private static boolean isConfiguredOut(RoutingPolicy policy, Set<ZoneId> inactiveZones) { + return isConfiguredOut(policy, Optional.empty(), inactiveZones); + } + + private static boolean isConfiguredOut(RoutingPolicy policy, Optional<ZoneRoutingPolicy> zonePolicy, Set<ZoneId> inactiveZones) { + // A deployment can be configured out from endpoints at any of the following levels: + // - zone level (ZoneRoutingPolicy, only applies to global endpoints) // - deployment level (RoutingPolicy) // - application package level (deployment.xml) - return zonePolicy.globalRouting().status() == GlobalRouting.Status.out || - policy.status().globalRouting().status() == GlobalRouting.Status.out || + return (zonePolicy.isPresent() && zonePolicy.get().routingStatus().value() == RoutingStatus.Value.out) || + policy.status().routingStatus().value() == RoutingStatus.Value.out || inactiveZones.contains(policy.id().zone()); } @@ -363,8 +507,8 @@ public class RoutingPolicies { .collect(Collectors.toUnmodifiableSet()); } - /** Compute all endpoint IDs for given load balancer */ - private Set<EndpointId> endpointIdsOf(LoadBalancer loadBalancer) { + /** Returns all instance endpoint IDs served by given load balancer */ + private Set<EndpointId> instanceEndpointsOf(LoadBalancer loadBalancer) { if (!deployment.zoneId().environment().isProduction()) { // Only production deployments have configurable endpoints return Set.of(); } @@ -384,6 +528,21 @@ public class RoutingPolicies { .collect(Collectors.toUnmodifiableSet()); } + /** Returns all application endpoint IDs served by given load balancer */ + private Set<EndpointId> applicationEndpointsOf(LoadBalancer loadBalancer) { + if (!deployment.zoneId().environment().isProduction()) { // Only production deployments have configurable endpoints + return Set.of(); + } + return deploymentSpec.endpoints().stream() + .filter(endpoint -> endpoint.containerId().equals(loadBalancer.cluster().value())) + .filter(endpoint -> endpoint.targets().stream() + .anyMatch(target -> target.region().equals(deployment.zoneId().region()) && + target.instance().equals(deployment.applicationId().instance()))) + .map(com.yahoo.config.application.api.Endpoint::endpointId) + .map(EndpointId::of) + .collect(Collectors.toUnmodifiableSet()); + } + } /** Returns zones where global routing is declared inactive for instance through deploymentSpec */ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicy.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicy.java index 5653b51f6c9..be8e49cf661 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicy.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicy.java @@ -75,6 +75,12 @@ public class RoutingPolicy { return status; } + /** Returns whether this policy applies to given deployment */ + public boolean appliesTo(DeploymentId deployment) { + return id.owner().equals(deployment.applicationId()) && + id.zone().equals(deployment.zoneId()); + } + /** Returns a copy of this with status set to given status */ public RoutingPolicy with(Status status) { return new RoutingPolicy(id, canonicalName, dnsZone, instanceEndpoints, applicationEndpoints, status); @@ -136,4 +142,47 @@ public class RoutingPolicy { .routingMethod(routingMethod); } + /** The status of a routing policy */ + public static class Status { + + private final boolean active; + private final RoutingStatus routingStatus; + + /** DO NOT USE. Public for serialization purposes */ + public Status(boolean active, RoutingStatus routingStatus) { + this.active = active; + this.routingStatus = Objects.requireNonNull(routingStatus, "globalRouting must be non-null"); + } + + /** Returns whether this is considered active according to the load balancer status */ + public boolean isActive() { + return active; + } + + /** Return status of routing */ + public RoutingStatus routingStatus() { + return routingStatus; + } + + /** Returns a copy of this with routing status changed */ + public Status with(RoutingStatus routingStatus) { + return new Status(active, routingStatus); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Status status = (Status) o; + return active == status.active && + routingStatus.equals(status.routingStatus); + } + + @Override + public int hashCode() { + return Objects.hash(active, routingStatus); + } + + } + } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicyId.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicyId.java index b7daa9fb6f7..e9cbdbd9b75 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicyId.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicyId.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.hosted.controller.routing; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import java.util.Objects; @@ -34,6 +35,11 @@ public class RoutingPolicyId { return zone; } + /** The deployment this applies to */ + public DeploymentId deployment() { + return new DeploymentId(owner, zone); + } + /** The cluster this applies to */ public ClusterSpec.Id cluster() { return cluster; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/GlobalRouting.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingStatus.java index 1b97397b7c2..58f0005d488 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/GlobalRouting.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingStatus.java @@ -5,34 +5,35 @@ import java.time.Instant; import java.util.Objects; /** - * Represents the global routing status of a {@link RoutingPolicy} or {@link ZoneRoutingPolicy}. This contains the - * time global routing status was last changed and who changed it. + * Represents the routing status of a {@link RoutingPolicy} or {@link ZoneRoutingPolicy}. + * + * This describes which agent last changed the routing status and at which time. * * This is immutable. * * @author mpolden */ -public class GlobalRouting { +public class RoutingStatus { - public static final GlobalRouting DEFAULT_STATUS = new GlobalRouting(Status.in, Agent.system, Instant.EPOCH); + public static final RoutingStatus DEFAULT = new RoutingStatus(Value.in, Agent.system, Instant.EPOCH); - private final Status status; + private final Value value; private final Agent agent; private final Instant changedAt; /** DO NOT USE. Public for serialization purposes */ - public GlobalRouting(Status status, Agent agent, Instant changedAt) { - this.status = Objects.requireNonNull(status, "status must be non-null"); + public RoutingStatus(Value value, Agent agent, Instant changedAt) { + this.value = Objects.requireNonNull(value, "value must be non-null"); this.agent = Objects.requireNonNull(agent, "agent must be non-null"); this.changedAt = Objects.requireNonNull(changedAt, "changedAt must be non-null"); } /** - * The wanted status of this. The system will try to set this status, but there are constraints that may lead to - * the effective status not matching this. See {@link RoutingPolicies}. + * The wanted value of this. The system will try to set this value, but there are constraints that may lead to + * the effective value not matching this. See {@link RoutingPolicies}. */ - public Status status() { - return status; + public Value value() { + return value; } /** The agent who last changed this */ @@ -49,28 +50,28 @@ public class GlobalRouting { public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; - GlobalRouting that = (GlobalRouting) o; - return status == that.status && + RoutingStatus that = (RoutingStatus) o; + return value == that.value && agent == that.agent && changedAt.equals(that.changedAt); } @Override public int hashCode() { - return Objects.hash(status, agent, changedAt); + return Objects.hash(value, agent, changedAt); } @Override public String toString() { - return "status " + status + ", changed by " + agent + " @ " + changedAt; + return "status " + value + ", changed by " + agent + " @ " + changedAt; } - public static GlobalRouting status(Status status, Agent agent, Instant instant) { - return new GlobalRouting(status, agent, instant); + public static RoutingStatus create(Value value, Agent agent, Instant instant) { + return new RoutingStatus(value, agent, instant); } // Used in serialization. Do not change. - public enum Status { + public enum Value { /** Status is determined by health checks **/ in, @@ -83,7 +84,7 @@ public class GlobalRouting { operator, tenant, system, - unknown, // For compatibility old values from /routing/v1 on config server, which may contain a specific user name. + unknown, // For compatibility old values from /routing/v1 on config server, which may contain a specific username. } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/Status.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/Status.java deleted file mode 100644 index c0918a7cbb4..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/Status.java +++ /dev/null @@ -1,53 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.routing; - -import java.util.Objects; - -/** - * Represents the status of a routing policy. - * - * This is immutable. - * - * @author mpolden - */ -public class Status { - - private final boolean active; - private final GlobalRouting globalRouting; - - /** DO NOT USE. Public for serialization purposes */ - public Status(boolean active, GlobalRouting globalRouting) { - this.active = active; - this.globalRouting = Objects.requireNonNull(globalRouting, "globalRouting must be non-null"); - } - - /** Returns whether this is considered active according to the load balancer status */ - public boolean isActive() { - return active; - } - - /** Return status of global routing */ - public GlobalRouting globalRouting() { - return globalRouting; - } - - /** Returns a copy of this with global routing changed */ - public Status with(GlobalRouting globalRouting) { - return new Status(active, globalRouting); - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - Status status = (Status) o; - return active == status.active && - globalRouting.equals(status.globalRouting); - } - - @Override - public int hashCode() { - return Objects.hash(active, globalRouting); - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/ZoneRoutingPolicy.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/ZoneRoutingPolicy.java index 8d7d17e9736..60605df1002 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/ZoneRoutingPolicy.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/ZoneRoutingPolicy.java @@ -6,7 +6,8 @@ import com.yahoo.config.provision.zone.ZoneId; import java.util.Objects; /** - * Represents the DNS routing policy for a zone. This takes precedence over of an individual {@link RoutingPolicy}. + * Represents the DNS routing policy for a zone. This takes precedence over of a deployment-specific + * {@link RoutingPolicy}. * * This is immutable. * @@ -15,11 +16,11 @@ import java.util.Objects; public class ZoneRoutingPolicy { private final ZoneId zone; - private final GlobalRouting globalRouting; + private final RoutingStatus routingStatus; - public ZoneRoutingPolicy(ZoneId zone, GlobalRouting globalRouting) { + public ZoneRoutingPolicy(ZoneId zone, RoutingStatus routingStatus) { this.zone = Objects.requireNonNull(zone, "zone must be non-null"); - this.globalRouting = Objects.requireNonNull(globalRouting, "globalRouting must be non-null"); + this.routingStatus = Objects.requireNonNull(routingStatus, "globalRouting must be non-null"); } /** The zone this applies to */ @@ -27,9 +28,9 @@ public class ZoneRoutingPolicy { return zone; } - /** The status of global routing */ - public GlobalRouting globalRouting() { - return globalRouting; + /** Routing status of this policy */ + public RoutingStatus routingStatus() { + return routingStatus; } @Override @@ -38,12 +39,12 @@ public class ZoneRoutingPolicy { if (o == null || getClass() != o.getClass()) return false; ZoneRoutingPolicy that = (ZoneRoutingPolicy) o; return zone.equals(that.zone) && - globalRouting.equals(that.globalRouting); + routingStatus.equals(that.routingStatus); } @Override public int hashCode() { - return Objects.hash(zone, globalRouting); + return Objects.hash(zone, routingStatus); } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java index e2bc1fa305d..9472801ef2c 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java @@ -6,12 +6,14 @@ import com.yahoo.component.Version; import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.application.api.ValidationId; import com.yahoo.config.application.api.ValidationOverrides; +import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.AthenzDomain; import com.yahoo.config.provision.AthenzService; import com.yahoo.config.provision.CloudName; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.HostName; +import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.zone.RoutingMethod; @@ -20,16 +22,17 @@ import com.yahoo.path.Path; import com.yahoo.vespa.hosted.controller.api.application.v4.model.EndpointStatus; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateMetadata; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.ContainerEndpoint; import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; import com.yahoo.vespa.hosted.controller.api.integration.dns.LatencyAliasTarget; import com.yahoo.vespa.hosted.controller.api.integration.dns.Record; import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordData; import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordName; import com.yahoo.vespa.hosted.controller.api.integration.dns.WeightedAliasTarget; -import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics; import com.yahoo.vespa.hosted.controller.application.Endpoint; +import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; import com.yahoo.vespa.hosted.controller.deployment.DeploymentContext; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; @@ -44,6 +47,7 @@ import java.time.Duration; import java.time.Instant; import java.util.Collection; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.function.Function; @@ -230,35 +234,58 @@ public class ControllerTest { @Test public void testDnsUpdatesForGlobalEndpoint() { - var context = tester.newDeploymentContext("tenant1", "app1", "default"); + var betaContext = tester.newDeploymentContext("tenant1", "app1", "beta"); + var defaultContext = tester.newDeploymentContext("tenant1", "app1", "default"); ApplicationPackage applicationPackage = new ApplicationPackageBuilder() + .instances("beta,default") .endpoint("default", "foo") .region("us-west-1") .region("us-central-1") // Two deployments should result in each DNS alias being registered once .build(); - context.submit(applicationPackage).deploy(); - - Collection<Deployment> deployments = context.instance().deployments().values(); - assertFalse(deployments.isEmpty()); - for (Deployment deployment : deployments) { - assertEquals("Rotation names are passed to config server in " + deployment.zone(), - Set.of("rotation-id-01", - "app1--tenant1.global.vespa.oath.cloud"), - tester.configServer().containerEndpoints().get(context.deploymentIdIn(deployment.zone()))); + betaContext.submit(applicationPackage).deploy(); + + { // Expected rotation names are passed to beta instance deployments + Collection<Deployment> betaDeployments = betaContext.instance().deployments().values(); + assertFalse(betaDeployments.isEmpty()); + for (Deployment deployment : betaDeployments) { + assertEquals("Rotation names are passed to config server in " + deployment.zone(), + Set.of("rotation-id-01", + "beta--app1--tenant1.global.vespa.oath.cloud"), + tester.configServer().containerEndpointNames(betaContext.deploymentIdIn(deployment.zone()))); + } + betaContext.flushDnsUpdates(); } - context.flushDnsUpdates(); - - assertEquals(1, tester.controllerTester().nameService().records().size()); - var record = tester.controllerTester().findCname("app1--tenant1.global.vespa.oath.cloud"); - assertTrue(record.isPresent()); - assertEquals("app1--tenant1.global.vespa.oath.cloud", record.get().name().asString()); - assertEquals("rotation-fqdn-01.", record.get().data().asString()); + { // Expected rotation names are passed to default instance deployments + Collection<Deployment> defaultDeployments = defaultContext.instance().deployments().values(); + assertFalse(defaultDeployments.isEmpty()); + for (Deployment deployment : defaultDeployments) { + assertEquals("Rotation names are passed to config server in " + deployment.zone(), + Set.of("rotation-id-02", + "app1--tenant1.global.vespa.oath.cloud"), + tester.configServer().containerEndpointNames(defaultContext.deploymentIdIn(deployment.zone()))); + } + defaultContext.flushDnsUpdates(); + } - List<String> globalDnsNames = tester.controller().routing().endpointsOf(context.instanceId()) - .scope(Endpoint.Scope.global) - .mapToList(Endpoint::dnsName); - assertEquals(List.of("app1--tenant1.global.vespa.oath.cloud"), globalDnsNames); + Map<String, String> rotationCnames = Map.of("beta--app1--tenant1.global.vespa.oath.cloud", "rotation-fqdn-01.", + "app1--tenant1.global.vespa.oath.cloud", "rotation-fqdn-02."); + rotationCnames.forEach((cname, data) -> { + var record = tester.controllerTester().findCname(cname); + assertTrue(record.isPresent()); + assertEquals(cname, record.get().name().asString()); + assertEquals(data, record.get().data().asString()); + }); + + Map<ApplicationId, List<String>> globalDnsNamesByInstance = Map.of(betaContext.instanceId(), List.of("beta--app1--tenant1.global.vespa.oath.cloud"), + defaultContext.instanceId(), List.of("app1--tenant1.global.vespa.oath.cloud")); + + globalDnsNamesByInstance.forEach((instance, dnsNames) -> { + List<String> actualDnsNames = tester.controller().routing().readDeclaredEndpointsOf(instance) + .scope(Endpoint.Scope.global) + .mapToList(Endpoint::dnsName); + assertEquals("Global DNS names for " + instance, dnsNames, actualDnsNames); + }); } @Test @@ -279,7 +306,7 @@ public class ControllerTest { "app1--tenant1.global.vespa.oath.cloud", "app1.tenant1.global.vespa.yahooapis.com", "app1--tenant1.global.vespa.yahooapis.com"), - tester.configServer().containerEndpoints().get(context.deploymentIdIn(deployment.zone()))); + tester.configServer().containerEndpointNames(context.deploymentIdIn(deployment.zone()))); } context.flushDnsUpdates(); assertEquals(3, tester.controllerTester().nameService().records().size()); @@ -299,7 +326,7 @@ public class ControllerTest { assertEquals("app1.tenant1.global.vespa.yahooapis.com", record.get().name().asString()); assertEquals("rotation-fqdn-01.", record.get().data().asString()); - List<String> globalDnsNames = tester.controller().routing().endpointsOf(context.instanceId()) + List<String> globalDnsNames = tester.controller().routing().readDeclaredEndpointsOf(context.instanceId()) .scope(Endpoint.Scope.global) .mapToList(Endpoint::dnsName); assertEquals(List.of("app1--tenant1.global.vespa.oath.cloud", @@ -334,7 +361,7 @@ public class ControllerTest { for (Deployment deployment : deployments) { assertEquals("Rotation names are passed to config server in " + deployment.zone(), ZoneId.from("prod.us-west-1").equals(deployment.zone()) ? west : notWest, - tester.configServer().containerEndpoints().get(context.deploymentIdIn(deployment.zone()))); + tester.configServer().containerEndpointNames(context.deploymentIdIn(deployment.zone()))); } context.flushDnsUpdates(); @@ -381,7 +408,7 @@ public class ControllerTest { assertEquals( "Zone " + zone + " is a member of global endpoint", Set.of("rotation-id-01", "app1--tenant1.global.vespa.oath.cloud"), - tester.configServer().containerEndpoints().get(context.deploymentIdIn(zone)) + tester.configServer().containerEndpointNames(context.deploymentIdIn(zone)) ); } @@ -399,13 +426,13 @@ public class ControllerTest { assertEquals( "Zone " + zone + " is a member of global endpoint", Set.of("rotation-id-01", "app1--tenant1.global.vespa.oath.cloud"), - tester.configServer().containerEndpoints().get(context.deploymentIdIn(zone)) + tester.configServer().containerEndpointNames(context.deploymentIdIn(zone)) ); } assertEquals( "Zone " + east + " is a member of global endpoint", Set.of("rotation-id-02", "east--app1--tenant1.global.vespa.oath.cloud"), - tester.configServer().containerEndpoints().get(context.deploymentIdIn(east)) + tester.configServer().containerEndpointNames(context.deploymentIdIn(east)) ); // Application is deployed with default endpoint pointing to 3/3 zones @@ -424,7 +451,7 @@ public class ControllerTest { ? Set.of("rotation-id-01", "app1--tenant1.global.vespa.oath.cloud", "rotation-id-02", "east--app1--tenant1.global.vespa.oath.cloud") : Set.of("rotation-id-01", "app1--tenant1.global.vespa.oath.cloud"), - tester.configServer().containerEndpoints().get(context.deploymentIdIn(zone)) + tester.configServer().containerEndpointNames(context.deploymentIdIn(zone)) ); } @@ -592,6 +619,59 @@ public class ControllerTest { } @Test + public void testDnsUpdatesForApplicationEndpoint() { + var context = tester.newDeploymentContext("tenant1", "app1", "beta"); + ApplicationPackage applicationPackage = new ApplicationPackageBuilder() + .instances("beta,main") + .region("us-west-1") + .region("us-east-3") + .applicationEndpoint("a", "qrs", "us-west-1", + Map.of(InstanceName.from("beta"), 2, + InstanceName.from("main"), 8)) + .applicationEndpoint("b", "qrs", "us-west-1", + Map.of(InstanceName.from("beta"), 1, + InstanceName.from("main"), 1)) + .applicationEndpoint("c", "qrs", "us-east-3", + Map.of(InstanceName.from("beta"), 4, + InstanceName.from("main"), 6)) + .build(); + context.submit(applicationPackage).deploy(); + + // Endpoint names are passed to each deployment + DeploymentId usWest = context.deploymentIdIn(ZoneId.from("prod", "us-west-1")); + DeploymentId usEast = context.deploymentIdIn(ZoneId.from("prod", "us-east-3")); + Map<DeploymentId, List<String>> deploymentEndpoints = Map.of(usWest, List.of("a--app1--tenant1.us-west-1-r.vespa.oath.cloud", "b--app1--tenant1.us-west-1-r.vespa.oath.cloud"), + usEast, List.of("c--app1--tenant1.us-east-3-r.vespa.oath.cloud")); + deploymentEndpoints.forEach((zone, endpointNames) -> { + assertEquals("Endpoint names are passed to config server in " + zone, + Set.of(new ContainerEndpoint("qrs", "application", + endpointNames)), + tester.configServer().containerEndpoints().get(zone)); + }); + context.flushDnsUpdates(); + + // DNS records are created for each endpoint + Set<Record> records = tester.controllerTester().nameService().records(); + assertEquals(Set.of(new Record(Record.Type.CNAME, + RecordName.from("a--app1--tenant1.us-west-1-r.vespa.oath.cloud"), + RecordData.from("vip.prod.us-west-1.")), + new Record(Record.Type.CNAME, + RecordName.from("b--app1--tenant1.us-west-1-r.vespa.oath.cloud"), + RecordData.from("vip.prod.us-west-1.")), + new Record(Record.Type.CNAME, + RecordName.from("c--app1--tenant1.us-east-3-r.vespa.oath.cloud"), + RecordData.from("vip.prod.us-east-3."))), + records); + List<String> endpointDnsNames = tester.controller().routing().declaredEndpointsOf(context.application()) + .scope(Endpoint.Scope.application) + .mapToList(Endpoint::dnsName); + assertEquals(List.of("a--app1--tenant1.us-west-1-r.vespa.oath.cloud", + "b--app1--tenant1.us-west-1-r.vespa.oath.cloud", + "c--app1--tenant1.us-east-3-r.vespa.oath.cloud"), + endpointDnsNames); + } + + @Test public void testDevDeployment() { ApplicationPackage applicationPackage = new ApplicationPackageBuilder().build(); @@ -610,7 +690,7 @@ public class ControllerTest { assertEquals("DeploymentSpec is not stored", DeploymentSpec.empty, context.application().deploymentSpec()); // Verify zone supports shared layer 4 and shared routing methods - Set<RoutingMethod> routingMethods = tester.controller().routing().endpointsOf(context.deploymentIdIn(zone)) + Set<RoutingMethod> routingMethods = tester.controller().routing().readEndpointsOf(context.deploymentIdIn(zone)) .asList() .stream() .map(Endpoint::routingMethod) @@ -856,12 +936,12 @@ public class ControllerTest { Set.of("application.tenant.global.vespa.oath.cloud", "foo.application.tenant.global.vespa.oath.cloud", "us.application.tenant.global.vespa.oath.cloud"), - tester.configServer().containerEndpoints().get(context.deploymentIdIn(zone))); + tester.configServer().containerEndpointNames(context.deploymentIdIn(zone))); } assertEquals("Expected container endpoints in " + zone3, Set.of("application.tenant.global.vespa.oath.cloud", "foo.application.tenant.global.vespa.oath.cloud"), - tester.configServer().containerEndpoints().get(context.deploymentIdIn(zone3))); + tester.configServer().containerEndpointNames(context.deploymentIdIn(zone3))); } @Test @@ -875,7 +955,7 @@ public class ControllerTest { tester.controllerTester().zoneRegistry() .setRoutingMethod(ZoneApiMock.from(zone1), RoutingMethod.shared, RoutingMethod.sharedLayer4) .setRoutingMethod(ZoneApiMock.from(zone2), RoutingMethod.shared, RoutingMethod.sharedLayer4); - Supplier<Set<RoutingMethod>> routingMethods = () -> tester.controller().routing().endpointsOf(context.deploymentIdIn(zone1)) + Supplier<Set<RoutingMethod>> routingMethods = () -> tester.controller().routing().readEndpointsOf(context.deploymentIdIn(zone1)) .asList() .stream() .map(Endpoint::routingMethod) @@ -899,9 +979,9 @@ public class ControllerTest { assertEquals(Set.of("rotation-id-01", "application.tenant.global.vespa.oath.cloud", "application--tenant.global.vespa.oath.cloud"), - tester.configServer().containerEndpoints().get(context.deploymentIdIn(zone))); + tester.configServer().containerEndpointNames(context.deploymentIdIn(zone))); } - List<String> zoneDnsNames = tester.controller().routing().endpointsOf(context.deploymentIdIn(zone1)) + List<String> zoneDnsNames = tester.controller().routing().readEndpointsOf(context.deploymentIdIn(zone1)) .scope(Endpoint.Scope.zone) .mapToList(Endpoint::dnsName); assertEquals(List.of("application--tenant.us-west-1.vespa.oath.cloud", diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificatesTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificatesTest.java index 8a0b97f20db..c6a9c12027f 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificatesTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificatesTest.java @@ -6,6 +6,7 @@ import com.yahoo.config.application.api.xml.DeploymentSpecXmlReader; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.SystemName; +import com.yahoo.config.provision.zone.RoutingMethod; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.security.KeyAlgorithm; import com.yahoo.security.KeyUtils; @@ -101,7 +102,7 @@ public class EndpointCertificatesTest { @Before public void setUp() { tester.zoneRegistry().exclusiveRoutingIn(tester.zoneRegistry().zones().all().zones()); - testZone = tester.zoneRegistry().zones().directlyRouted().in(Environment.prod).zones().stream().findFirst().orElseThrow().getId(); + testZone = tester.zoneRegistry().zones().routingMethod(RoutingMethod.exclusive).in(Environment.prod).zones().stream().findFirst().orElseThrow().getId(); clock.setInstant(Instant.EPOCH); testCertificate = makeTestCert(expectedSans); testCertificate2 = makeTestCert(expectedCombinedSans); @@ -109,7 +110,7 @@ public class EndpointCertificatesTest { @Test public void provisions_new_certificate_in_dev() { - ZoneId testZone = tester.zoneRegistry().zones().directlyRouted().in(Environment.dev).zones().stream().findFirst().orElseThrow().getId(); + ZoneId testZone = tester.zoneRegistry().zones().routingMethod(RoutingMethod.exclusive).in(Environment.dev).zones().stream().findFirst().orElseThrow().getId(); Optional<EndpointCertificateMetadata> endpointCertificateMetadata = endpointCertificates.getMetadata(testInstance, testZone, Optional.empty()); assertTrue(endpointCertificateMetadata.isPresent()); assertTrue(endpointCertificateMetadata.get().keyName().matches("vespa.tls.default.default.*-key")); @@ -183,7 +184,7 @@ public class EndpointCertificatesTest { @Test public void reprovisions_certificate_with_added_sans_when_deploying_to_new_zone() { - ZoneId testZone = tester.zoneRegistry().zones().directlyRouted().in(Environment.prod).zones().stream().skip(1).findFirst().orElseThrow().getId(); + ZoneId testZone = tester.zoneRegistry().zones().routingMethod(RoutingMethod.exclusive).in(Environment.prod).zones().stream().skip(1).findFirst().orElseThrow().getId(); mockCuratorDb.writeEndpointCertificateMetadata(testInstance.id(), new EndpointCertificateMetadata(testKeyName, testCertName, -1, 0, "original-request-uuid", expectedSans, "mockCa", Optional.empty(), Optional.empty())); secretStore.setSecret("vespa.tls.default.default.default-key", KeyUtils.toPem(testKeyPair.getPrivate()), -1); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java index 64821756105..91a12d3b465 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java @@ -5,6 +5,7 @@ import com.yahoo.component.Version; import com.yahoo.config.application.api.ValidationId; import com.yahoo.config.provision.AthenzDomain; import com.yahoo.config.provision.AthenzService; +import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.RegionName; import com.yahoo.security.SignatureAlgorithm; import com.yahoo.security.X509CertificateBuilder; @@ -26,8 +27,10 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Date; import java.util.List; +import java.util.Map; import java.util.OptionalInt; import java.util.StringJoiner; +import java.util.TreeMap; import java.util.zip.Deflater; import java.util.zip.ZipEntry; import java.util.zip.ZipOutputStream; @@ -48,6 +51,7 @@ public class ApplicationPackageBuilder { "<notifications>\n <email ", "/>\n</notifications>\n").setEmptyValue(""); private final StringBuilder endpointsBody = new StringBuilder(); + private final StringBuilder applicationEndpointsBody = new StringBuilder(); private final List<X509Certificate> trustedCertificates = new ArrayList<>(); private OptionalInt majorVersion = OptionalInt.empty(); @@ -86,9 +90,9 @@ public class ApplicationPackageBuilder { return this; } - public ApplicationPackageBuilder endpoint(String endpointId, String containerId, String... regions) { + public ApplicationPackageBuilder endpoint(String id, String containerId, String... regions) { endpointsBody.append(" <endpoint"); - endpointsBody.append(" id='").append(endpointId).append("'"); + endpointsBody.append(" id='").append(id).append("'"); endpointsBody.append(" container-id='").append(containerId).append("'"); endpointsBody.append(">\n"); for (var region : regions) { @@ -98,6 +102,23 @@ public class ApplicationPackageBuilder { return this; } + public ApplicationPackageBuilder applicationEndpoint(String id, String containerId, String region, + Map<InstanceName, Integer> instanceWeights) { + if (instanceWeights.isEmpty()) throw new IllegalArgumentException("At least one instance must be given"); + applicationEndpointsBody.append(" <endpoint"); + applicationEndpointsBody.append(" id='").append(id).append("'"); + applicationEndpointsBody.append(" container-id='").append(containerId).append("'"); + applicationEndpointsBody.append(" region='").append(region).append("'"); + applicationEndpointsBody.append(">\n"); + for (var kv : new TreeMap<>(instanceWeights).entrySet()) { + applicationEndpointsBody.append(" <instance weight='").append(kv.getValue().toString()).append("'>") + .append(kv.getKey().value()) + .append("</instance>\n"); + } + applicationEndpointsBody.append(" </endpoint>\n"); + return this; + } + public ApplicationPackageBuilder systemTest() { explicitSystemTest = true; return this; @@ -248,10 +269,17 @@ public class ApplicationPackageBuilder { xml.append(">\n"); xml.append(prodBody); xml.append(" </prod>\n"); - xml.append(" <endpoints>\n"); - xml.append(endpointsBody); - xml.append(" </endpoints>\n"); + if (endpointsBody.length() > 0 ) { + xml.append(" <endpoints>\n"); + xml.append(endpointsBody); + xml.append(" </endpoints>\n"); + } xml.append(" </instance>\n"); + if (applicationEndpointsBody.length() > 0) { + xml.append(" <endpoints>\n"); + xml.append(applicationEndpointsBody); + xml.append(" </endpoints>\n"); + } xml.append("</deployment>\n"); return xml.toString().getBytes(UTF_8); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java index ba6ab4b9152..b9c1e9f3e72 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java @@ -35,10 +35,9 @@ import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; import com.yahoo.vespa.hosted.controller.integration.ConfigServerMock; import com.yahoo.vespa.hosted.controller.maintenance.JobRunner; import com.yahoo.vespa.hosted.controller.maintenance.NameServiceDispatcher; -import com.yahoo.vespa.hosted.controller.routing.GlobalRouting; +import com.yahoo.vespa.hosted.controller.routing.RoutingStatus; import com.yahoo.vespa.hosted.controller.routing.RoutingPolicy; import com.yahoo.vespa.hosted.controller.routing.RoutingPolicyId; -import com.yahoo.vespa.hosted.controller.routing.Status; import javax.security.auth.x500.X500Principal; import java.math.BigInteger; @@ -46,6 +45,7 @@ import java.security.KeyPair; import java.security.cert.X509Certificate; import java.time.Duration; import java.time.Instant; +import java.util.HashMap; import java.util.HashSet; import java.util.LinkedHashMap; import java.util.List; @@ -165,13 +165,16 @@ public class DeploymentContext { /** Completely deploy the latest change */ public DeploymentContext deploy() { - assertTrue("Application package submitted", application().latestVersion().isPresent()); - assertFalse("Submission is not already deployed", application().instances().values().stream() - .anyMatch(instance -> instance.deployments().values().stream() + Application application = application(); + assertTrue("Application package submitted", application.latestVersion().isPresent()); + assertFalse("Submission is not already deployed", application.instances().values().stream() + .anyMatch(instance -> instance.deployments().values().stream() .anyMatch(deployment -> deployment.applicationVersion().equals(lastSubmission)))); - assertEquals(application().latestVersion(), instance().change().application()); - completeRollout(); - assertFalse(instance().change().hasTargets()); + assertEquals(application.latestVersion(), instance().change().application()); + completeRollout(application.deploymentSpec().instances().size() > 1); + for (var instance : application().instances().values()) { + assertFalse(instance.change().hasTargets()); + } return this; } @@ -241,7 +244,7 @@ public class DeploymentContext { Optional.empty(), Set.of(EndpointId.of("default")), Set.of(), - new Status(false, GlobalRouting.DEFAULT_STATUS))); + new RoutingPolicy.Status(false, RoutingStatus.DEFAULT))); tester.controller().curator().writeRoutingPolicies(instanceId, policies); return this; } @@ -304,19 +307,28 @@ public class DeploymentContext { return Optional.ofNullable(lastSubmission); } - /** Runs and returns all remaining jobs for the application, at most once, and asserts the current change is rolled out. */ public DeploymentContext completeRollout() { + return completeRollout(false); + } + + /** Runs and returns all remaining jobs for the application, at most once, and asserts the current change is rolled out. */ + public DeploymentContext completeRollout(boolean multiInstance) { triggerJobs(); - Set<JobType> jobs = new HashSet<>(); + Map<ApplicationId, Set<JobType>> jobsByInstance = new HashMap<>(); List<Run> activeRuns; while ( ! (activeRuns = this.jobs.active(applicationId)).isEmpty()) - for (Run run : activeRuns) + for (Run run : activeRuns) { + Set<JobType> jobs = jobsByInstance.computeIfAbsent(run.id().application(), k -> new HashSet<>()); if (jobs.add(run.id().type())) { - runJob(run.id().type()); + runJob(run.id().type(), run.id().application()); + if (multiInstance) { + tester.outstandingChangeDeployer().run(); + } triggerJobs(); - } - else + } else { throw new AssertionError("Job '" + run.id() + "' was run twice"); + } + } assertFalse("Change should have no targets, but was " + instance().change(), instance().change().hasTargets()); return this; @@ -338,9 +350,14 @@ public class DeploymentContext { return runJob(JobType.from(tester.controller().system(), zone).get(), applicationPackage, null); } - /** Pulls the ready job trigger, and then runs the whole of the given job, successfully. */ + /** Pulls the ready job trigger, and then runs the whole of the given job in the instance of this, successfully. */ public DeploymentContext runJob(JobType type) { - var job = jobId(type); + return runJob(type, instanceId); + } + + /** Pulls the ready job trigger, and then runs the whole of job for the given instance, successfully. */ + public DeploymentContext runJob(JobType type, ApplicationId instance) { + var job = new JobId(instance, type); triggerJobs(); doDeploy(job); if (job.type().isDeployment()) { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java index 0f32d01ffe3..01e1301c8cf 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java @@ -143,9 +143,13 @@ public class DeploymentTester { /** Triggers jobs until nothing more triggers, and returns the number of triggered jobs. */ public int triggerJobs() { - int triggered = 0; - while (triggered != (triggered += deploymentTrigger().triggerReadyJobs())); - return triggered; + int triggered; + int triggeredTotal = 0; + do { + triggered = (int)deploymentTrigger().triggerReadyJobs(); + triggeredTotal += triggered; + } while (triggered > 0); + return triggeredTotal; } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java index 353756b3a4f..1f5fa243838 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java @@ -782,10 +782,9 @@ public class DeploymentTriggerTest { .instances("instance1,instance2") .region("us-east-3") .build(); - var app = tester.newDeploymentContext("tenant1", "application1", "instance1").submit(applicationPackage); // TODO jonmv: support instances in deployment context> - var otherInstance = tester.newDeploymentContext("tenant1", "application1", "instance2"); - app.runJob(systemTest).runJob(stagingTest).runJob(productionUsEast3); - otherInstance.runJob(productionUsEast3); + var app = tester.newDeploymentContext("tenant1", "application1", "instance1") + .submit(applicationPackage) + .completeRollout(); assertEquals(2, app.application().instances().size()); assertEquals(2, app.application().productionDeployments().values().stream() .mapToInt(Collection::size) diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ZipBuilderTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ZipBuilderTest.java index fd2de2e1c1d..f48658af7e3 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ZipBuilderTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ZipBuilderTest.java @@ -59,4 +59,5 @@ public class ZipBuilderTest { } return contents; } + }
\ No newline at end of file diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java index 37f9b66d5fe..a3674fa27bd 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java @@ -40,8 +40,8 @@ import com.yahoo.vespa.hosted.controller.api.integration.deployment.TestReport; import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterCloud; import com.yahoo.vespa.hosted.controller.api.integration.noderepository.RestartFilter; import com.yahoo.vespa.hosted.controller.api.integration.secrets.TenantSecretStore; -import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.SystemApplication; +import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; import com.yahoo.vespa.serviceview.bindings.ApplicationView; import com.yahoo.vespa.serviceview.bindings.ClusterView; import com.yahoo.vespa.serviceview.bindings.ServiceView; @@ -90,7 +90,7 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer private final Map<ZoneId, Set<LoadBalancer>> loadBalancers = new HashMap<>(); private final Set<Environment> deferLoadBalancerProvisioning = new HashSet<>(); private final Map<DeploymentId, List<Log>> warnings = new HashMap<>(); - private final Map<DeploymentId, Set<String>> containerEndpoints = new HashMap<>(); + private final Map<DeploymentId, Set<ContainerEndpoint>> containerEndpoints = new HashMap<>(); private final Map<DeploymentId, List<ClusterMetrics>> clusterMetrics = new HashMap<>(); private final Map<DeploymentId, TestReport> testReport = new HashMap<>(); private List<ProtonMetrics> protonMetrics; @@ -280,10 +280,17 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer warnings.put(deployment, List.copyOf(logs)); } - public Map<DeploymentId, Set<String>> containerEndpoints() { + public Map<DeploymentId, Set<ContainerEndpoint>> containerEndpoints() { return Collections.unmodifiableMap(containerEndpoints); } + public Set<String> containerEndpointNames(DeploymentId deployment) { + return containerEndpoints.getOrDefault(deployment, Set.of()).stream() + .map(ContainerEndpoint::names) + .flatMap(Collection::stream) + .collect(Collectors.toUnmodifiableSet()); + } + public void setMetrics(DeploymentId deployment, ClusterMetrics clusterMetrics) { setMetrics(deployment, List.of(clusterMetrics)); } @@ -386,13 +393,7 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer if (nodeRepository().list(id.zoneId(), NodeFilter.all().applications(id.applicationId())).isEmpty()) provision(id.zoneId(), id.applicationId(), cluster); - this.containerEndpoints.put( - id, - deployment.containerEndpoints().stream() - .map(ContainerEndpoint::names) - .flatMap(Collection::stream) - .collect(Collectors.toSet()) - ); + this.containerEndpoints.put(id, deployment.containerEndpoints()); if (!deferLoadBalancerProvisioning.contains(id.zoneId().environment())) { putLoadBalancers(id.zoneId(), List.of(new LoadBalancer(UUID.randomUUID().toString(), diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneFilterMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneFilterMock.java index 4621e12f05f..c17b2fcb36a 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneFilterMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneFilterMock.java @@ -56,11 +56,6 @@ public class ZoneFilterMock implements ZoneList { } @Override - public ZoneList directlyRouted() { - return routingMethod(RoutingMethod.exclusive); - } - - @Override public ZoneList routingMethod(RoutingMethod method) { return filter(zone -> zoneRoutingMethods.getOrDefault(zone, List.of()).contains(method)); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneRegistryMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneRegistryMock.java index 2ef98ec6d2a..23ab91aaf8c 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneRegistryMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneRegistryMock.java @@ -230,6 +230,14 @@ public class ZoneRegistryMock extends AbstractComponent implements ZoneRegistry } @Override + public Optional<String> getVipHostname(ZoneId zoneId) { + if (routingMethods(zoneId).stream().anyMatch(RoutingMethod::isShared)) { + return Optional.of("vip." + zoneId.value()); + } + return Optional.empty(); + } + + @Override public Optional<Duration> getDeploymentTimeToLive(ZoneId zoneId) { return Optional.ofNullable(deploymentTimeToLive.get(zoneId)); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CloudEventTrackerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CloudEventTrackerTest.java index f0aef75bb11..8daedc05e96 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CloudEventTrackerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CloudEventTrackerTest.java @@ -31,7 +31,6 @@ public class CloudEventTrackerTest { private final ZoneApiMock zone1 = createZone("prod.zone1", "region-1", "aws"); private final ZoneApiMock zone2 = createZone("prod.zone2", "region-2", "aws"); - /** * Test scenario: Consider three zones, two of which are supported * diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java index e65f6e087c5..7619cf71f1a 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java @@ -280,7 +280,7 @@ public class MetricsReporterTest { context.submit(applicationPackage).deploy(); reporter.maintain(); - assertEquals("Deployment queues name services requests", 15, metrics.getMetric(MetricsReporter.NAME_SERVICE_REQUESTS_QUEUED).intValue()); + assertEquals("Deployment queues name services requests", 6, metrics.getMetric(MetricsReporter.NAME_SERVICE_REQUESTS_QUEUED).intValue()); context.flushDnsUpdates(); reporter.maintain(); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializerTest.java index f4475038b17..2e36b8969ba 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializerTest.java @@ -7,10 +7,9 @@ import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.application.EndpointId; -import com.yahoo.vespa.hosted.controller.routing.GlobalRouting; +import com.yahoo.vespa.hosted.controller.routing.RoutingStatus; import com.yahoo.vespa.hosted.controller.routing.RoutingPolicy; import com.yahoo.vespa.hosted.controller.routing.RoutingPolicyId; -import com.yahoo.vespa.hosted.controller.routing.Status; import org.junit.Test; import java.time.Instant; @@ -43,16 +42,16 @@ public class RoutingPolicySerializerTest { Optional.of("zone1"), instanceEndpoints, applicationEndpoints, - new Status(true, GlobalRouting.DEFAULT_STATUS)), + new RoutingPolicy.Status(true, RoutingStatus.DEFAULT)), id2, new RoutingPolicy(id2, HostName.from("long-and-ugly-name-2"), Optional.empty(), instanceEndpoints, Set.of(), - new Status(false, - new GlobalRouting(GlobalRouting.Status.out, - GlobalRouting.Agent.tenant, - Instant.ofEpochSecond(123))))); + new RoutingPolicy.Status(false, + new RoutingStatus(RoutingStatus.Value.out, + RoutingStatus.Agent.tenant, + Instant.ofEpochSecond(123))))); var serialized = serializer.fromSlime(owner, serializer.toSlime(policies)); assertEquals(policies.size(), serialized.size()); for (Iterator<RoutingPolicy> it1 = policies.values().iterator(), it2 = serialized.values().iterator(); it1.hasNext();) { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ZoneRoutingPolicySerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ZoneRoutingPolicySerializerTest.java index 4cc6bd93d01..234de233571 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ZoneRoutingPolicySerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ZoneRoutingPolicySerializerTest.java @@ -2,7 +2,7 @@ package com.yahoo.vespa.hosted.controller.persistence; import com.yahoo.config.provision.zone.ZoneId; -import com.yahoo.vespa.hosted.controller.routing.GlobalRouting; +import com.yahoo.vespa.hosted.controller.routing.RoutingStatus; import com.yahoo.vespa.hosted.controller.routing.ZoneRoutingPolicy; import org.junit.Test; @@ -20,8 +20,8 @@ public class ZoneRoutingPolicySerializerTest { var serializer = new ZoneRoutingPolicySerializer(new RoutingPolicySerializer()); var zone = ZoneId.from("prod", "us-north-1"); var policy = new ZoneRoutingPolicy(zone, - GlobalRouting.status(GlobalRouting.Status.out, GlobalRouting.Agent.operator, - Instant.ofEpochMilli(123))); + RoutingStatus.create(RoutingStatus.Value.out, RoutingStatus.Agent.operator, + Instant.ofEpochMilli(123))); var serialized = serializer.fromSlime(zone, serializer.toSlime(policy)); assertEquals(policy, serialized); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java index e204a8f820d..ae6232ae419 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java @@ -10,6 +10,7 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.AthenzService; import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.zone.RoutingMethod; @@ -67,7 +68,7 @@ import com.yahoo.vespa.hosted.controller.notification.Notification; import com.yahoo.vespa.hosted.controller.notification.NotificationSource; import com.yahoo.vespa.hosted.controller.restapi.ContainerTester; import com.yahoo.vespa.hosted.controller.restapi.ControllerContainerTest; -import com.yahoo.vespa.hosted.controller.routing.GlobalRouting; +import com.yahoo.vespa.hosted.controller.routing.RoutingStatus; import com.yahoo.vespa.hosted.controller.security.AthenzCredentials; import com.yahoo.vespa.hosted.controller.security.AthenzTenantSpec; import com.yahoo.vespa.hosted.controller.support.access.SupportAccessGrant; @@ -140,6 +141,7 @@ public class ApplicationApiTest extends ControllerContainerTest { .region("us-east-3") .region("us-west-1") .blockChange(false, true, "mon-fri", "0-8", "UTC") + .applicationEndpoint("a0", "foo", "us-central-1", Map.of(InstanceName.from("instance1"), 1)) .build(); private static final AthenzDomain ATHENZ_TENANT_DOMAIN = new AthenzDomain("domain1"); @@ -957,14 +959,14 @@ public class ApplicationApiTest extends ControllerContainerTest { new File("global-rotation-put.json")); // Status of routing policy is changed - assertGlobalRouting(app.deploymentIdIn(westZone), GlobalRouting.Status.out, GlobalRouting.Agent.tenant); + assertGlobalRouting(app.deploymentIdIn(westZone), RoutingStatus.Value.out, RoutingStatus.Agent.tenant); // DELETE global rotation override status tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/environment/prod/region/us-west-1/global-rotation/override", DELETE) .userIdentity(USER_ID) .data("{\"reason\":\"unit-test\"}"), new File("global-rotation-delete.json")); - assertGlobalRouting(app.deploymentIdIn(westZone), GlobalRouting.Status.in, GlobalRouting.Agent.tenant); + assertGlobalRouting(app.deploymentIdIn(westZone), RoutingStatus.Value.in, RoutingStatus.Agent.tenant); // SET global rotation override status by operator addUserToHostedOperatorRole(HostedAthenzIdentities.from(HOSTED_VESPA_OPERATOR)); @@ -972,7 +974,7 @@ public class ApplicationApiTest extends ControllerContainerTest { .userIdentity(HOSTED_VESPA_OPERATOR) .data("{\"reason\":\"unit-test\"}"), new File("global-rotation-put.json")); - assertGlobalRouting(app.deploymentIdIn(westZone), GlobalRouting.Status.out, GlobalRouting.Agent.operator); + assertGlobalRouting(app.deploymentIdIn(westZone), RoutingStatus.Value.out, RoutingStatus.Agent.operator); } @Test @@ -1865,14 +1867,14 @@ public class ApplicationApiTest extends ControllerContainerTest { "Failed to deploy: Out of capacity"); } - private void assertGlobalRouting(DeploymentId deployment, GlobalRouting.Status status, GlobalRouting.Agent agent) { + private void assertGlobalRouting(DeploymentId deployment, RoutingStatus.Value value, RoutingStatus.Agent agent) { var changedAt = tester.controller().clock().instant(); var westPolicies = tester.controller().routing().policies().get(deployment); assertEquals(1, westPolicies.size()); var westPolicy = westPolicies.values().iterator().next(); - assertEquals(status, westPolicy.status().globalRouting().status()); - assertEquals(agent, westPolicy.status().globalRouting().agent()); - assertEquals(changedAt.truncatedTo(ChronoUnit.MILLIS), westPolicy.status().globalRouting().changedAt()); + assertEquals(value, westPolicy.status().routingStatus().value()); + assertEquals(agent, westPolicy.status().routingStatus().agent()); + assertEquals(changedAt.truncatedTo(ChronoUnit.MILLIS), westPolicy.status().routingStatus().changedAt()); } private static class RequestBuilder implements Supplier<Request> { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application2-with-patches.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application2-with-patches.json index 5c073173544..28732acb1df 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application2-with-patches.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application2-with-patches.json @@ -18,7 +18,6 @@ "instances": [ { "instance": "default", - "globalRotations": [], "deployments": [] }, { @@ -40,9 +39,6 @@ } }, "changeBlockers": [], - "globalRotations": [ - "https://instance1--application2--tenant2.global.vespa.oath.cloud:4443/" - ], "rotationId": "rotation-id-2", "deployments": [] } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application2.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application2.json index 01fd88a599f..d009af005e4 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application2.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application2.json @@ -17,7 +17,6 @@ "instances": [ { "instance": "default", - "globalRotations": [], "deployments": [] }, { @@ -39,9 +38,6 @@ } }, "changeBlockers": [], - "globalRotations": [ - "https://instance1--application2--tenant2.global.vespa.oath.cloud:4443/" - ], "rotationId": "rotation-id-2", "deployments": [] } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment.json index 3ce83528b2c..fb6088f54b8 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment.json @@ -20,6 +20,14 @@ "scope": "global", "routingMethod": "shared", "legacy": false + }, + { + "cluster": "foo", + "tls": true, + "url": "https://a0--application1--tenant1.us-central-1-r.vespa.oath.cloud:4443/", + "scope": "application", + "routingMethod": "shared", + "legacy": false } ], "clusters": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/environment/prod/region/us-central-1/clusters", diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/instance-with-routing-policy.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/instance-with-routing-policy.json index 33b1d95b5ca..e3f70e84f43 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/instance-with-routing-policy.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/instance-with-routing-policy.json @@ -12,7 +12,6 @@ "commit": "commit1", "projectId": 1000, "changeBlockers": [], - "globalRotations": [], "instances": [ { "environment": "prod", diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/instance.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/instance.json index 745d0ad162a..bcbdf448ad5 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/instance.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/instance.json @@ -49,9 +49,6 @@ ] } ], - "globalRotations": [ - "https://instance1--application1--tenant1.global.vespa.oath.cloud:4443/" - ], "rotationId": "rotation-id-1", "instances": [ { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/instance1-recursive.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/instance1-recursive.json index 19676decdb8..0c4f046f45c 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/instance1-recursive.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/instance1-recursive.json @@ -49,9 +49,6 @@ ] } ], - "globalRotations": [ - "https://instance1--application1--tenant1.global.vespa.oath.cloud:4443/" - ], "rotationId": "rotation-id-1", "instances": [ @include(dev-us-east-1.json), diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/prod-us-central-1.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/prod-us-central-1.json index 4edbc58121b..409e97b063c 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/prod-us-central-1.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/prod-us-central-1.json @@ -23,6 +23,14 @@ "scope": "global", "routingMethod": "shared", "legacy": false + }, + { + "cluster": "foo", + "tls": true, + "url": "https://a0--application1--tenant1.us-central-1-r.vespa.oath.cloud:4443/", + "scope": "application", + "routingMethod": "shared", + "legacy": false } ], "clusters":"http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/environment/prod/region/us-central-1/clusters", diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiTest.java index 41607b17b52..5b2fabcaff8 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiTest.java @@ -251,48 +251,6 @@ public class RoutingApiTest extends ControllerContainerTest { new File("rotation/zone-status-in.json")); } - // TODO(mpolden): Remove this once a zone supports either of routing policy and rotation - @Test - public void mixed_routing_single_zone() { - var westZone = ZoneId.from("prod", "us-west-1"); - var eastZone = ZoneId.from("prod", "us-east-3"); - - // One zone supports multiple routing methods - deploymentTester.controllerTester().zoneRegistry().setRoutingMethod(ZoneApiMock.from(westZone), - RoutingMethod.shared, - RoutingMethod.exclusive); - - // Deploy application - var context = deploymentTester.newDeploymentContext(); - var applicationPackage = new ApplicationPackageBuilder() - .region(westZone.region()) - .region(eastZone.region()) - .endpoint("default", "default", eastZone.region().value(), westZone.region().value()) - .build(); - context.submit(applicationPackage).deploy(); - - // GET status with both policy and rotation assigned - tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/status/tenant/tenant/application/application/instance/default/environment/prod/region/us-west-1", - "", Request.Method.GET), - new File("multi-status-initial.json")); - - // POST sets deployment out - tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/inactive/tenant/tenant/application/application/instance/default/environment/prod/region/us-west-1", - "", Request.Method.POST), - "{\"message\":\"Set global routing status for tenant.application in prod.us-west-1 to OUT\"}"); - tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/status/tenant/tenant/application/application/instance/default/environment/prod/region/us-west-1", - "", Request.Method.GET), - new File("multi-status-out.json")); - - // DELETE sets deployment in - tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/inactive/tenant/tenant/application/application/instance/default/environment/prod/region/us-west-1", - "", Request.Method.DELETE), - "{\"message\":\"Set global routing status for tenant.application in prod.us-west-1 to IN\"}"); - tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/status/tenant/tenant/application/application/instance/default/environment/prod/region/us-west-1", - "", Request.Method.GET), - new File("multi-status-in.json")); - } - @Test public void mixed_routing_multiple_zones() { var westZone = ZoneId.from("prod", "us-west-1"); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/multi-status-in.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/multi-status-in.json deleted file mode 100644 index 1c23c6bb569..00000000000 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/multi-status-in.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "deployments": [ - { - "routingMethod": "shared", - "instance": "tenant:application:default", - "environment": "prod", - "region": "us-west-1", - "status": "in", - "agent": "operator", - "changedAt": "(ignore)" - }, - { - "routingMethod": "exclusive", - "instance": "tenant:application:default", - "environment": "prod", - "region": "us-west-1", - "status": "in", - "agent": "operator", - "changedAt": "(ignore)" - } - ] -} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/multi-status-initial.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/multi-status-initial.json deleted file mode 100644 index eea78c1b963..00000000000 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/multi-status-initial.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "deployments": [ - { - "routingMethod": "shared", - "instance": "tenant:application:default", - "environment": "prod", - "region": "us-west-1", - "status": "in", - "agent": "unknown", - "changedAt": "(ignore)" - }, - { - "routingMethod": "exclusive", - "instance": "tenant:application:default", - "environment": "prod", - "region": "us-west-1", - "status": "in", - "agent": "system", - "changedAt": "(ignore)" - } - ] -} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/multi-status-out.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/multi-status-out.json deleted file mode 100644 index 6cb90bdb673..00000000000 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/multi-status-out.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "deployments": [ - { - "routingMethod": "shared", - "instance": "tenant:application:default", - "environment": "prod", - "region": "us-west-1", - "status": "out", - "agent": "operator", - "changedAt": "(ignore)" - }, - { - "routingMethod": "exclusive", - "instance": "tenant:application:default", - "environment": "prod", - "region": "us-west-1", - "status": "out", - "agent": "operator", - "changedAt": "(ignore)" - } - ] -} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/rotation/RotationRepositoryTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/rotation/RotationRepositoryTest.java index 910e5943989..b767e8a791f 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/rotation/RotationRepositoryTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/rotation/RotationRepositoryTest.java @@ -63,7 +63,7 @@ public class RotationRepositoryTest { assertEquals(List.of(expected.id()), rotationIds(application.instance().rotations())); assertEquals(URI.create("https://app1--tenant1.global.vespa.oath.cloud:4443/"), - tester.controller().routing().endpointsOf(application.instanceId()).primary().get().url()); + tester.controller().routing().readDeclaredEndpointsOf(application.instanceId()).primary().get().url()); try (RotationLock lock = repository.lock()) { List<AssignedRotation> rotations = repository.getOrAssignRotations(application.application().deploymentSpec(), application.instance(), @@ -158,7 +158,7 @@ public class RotationRepositoryTest { application2.submit(applicationPackage); assertEquals(List.of(new RotationId("foo-1")), rotationIds(application2.instance().rotations())); assertEquals("https://cd--app2--tenant2.global.vespa.oath.cloud:4443/", - tester.controller().routing().endpointsOf(application2.instanceId()).primary().get().url().toString()); + tester.controller().routing().readDeclaredEndpointsOf(application2.instanceId()).primary().get().url().toString()); } @Test @@ -174,9 +174,9 @@ public class RotationRepositoryTest { assertEquals(List.of(new RotationId("foo-1")), rotationIds(instance1.instance().rotations())); assertEquals(List.of(new RotationId("foo-2")), rotationIds(instance2.instance().rotations())); assertEquals(URI.create("https://instance1--application1--tenant1.global.vespa.oath.cloud:4443/"), - tester.controller().routing().endpointsOf(instance1.instanceId()).primary().get().url()); + tester.controller().routing().readDeclaredEndpointsOf(instance1.instanceId()).primary().get().url()); assertEquals(URI.create("https://instance2--application1--tenant1.global.vespa.oath.cloud:4443/"), - tester.controller().routing().endpointsOf(instance2.instanceId()).primary().get().url()); + tester.controller().routing().readDeclaredEndpointsOf(instance2.instanceId()).primary().get().url()); } @Test @@ -194,9 +194,9 @@ public class RotationRepositoryTest { assertEquals(List.of(new RotationId("foo-2")), rotationIds(instance2.instance().rotations())); assertEquals(URI.create("https://instance1--application1--tenant1.global.vespa.oath.cloud:4443/"), - tester.controller().routing().endpointsOf(instance1.instanceId()).primary().get().url()); + tester.controller().routing().readDeclaredEndpointsOf(instance1.instanceId()).primary().get().url()); assertEquals(URI.create("https://instance2--application1--tenant1.global.vespa.oath.cloud:4443/"), - tester.controller().routing().endpointsOf(instance2.instanceId()).primary().get().url()); + tester.controller().routing().readDeclaredEndpointsOf(instance2.instanceId()).primary().get().url()); } private void assertSingleRotation(Rotation expected, List<AssignedRotation> assignedRotations, RotationRepository repository) { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java index e2ef27492af..c40cb20a0bc 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java @@ -27,6 +27,7 @@ import com.yahoo.vespa.hosted.controller.application.Endpoint; import com.yahoo.vespa.hosted.controller.application.EndpointId; import com.yahoo.vespa.hosted.controller.application.EndpointList; import com.yahoo.vespa.hosted.controller.application.SystemApplication; +import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; import com.yahoo.vespa.hosted.controller.deployment.DeploymentContext; @@ -165,8 +166,8 @@ public class RoutingPoliciesTest { tester.policiesOf(context.instance().id()).size()); // A zone in shared region is set out - tester.routingPolicies().setGlobalRoutingStatus(context.deploymentIdIn(zone4), GlobalRouting.Status.out, - GlobalRouting.Agent.tenant); + tester.routingPolicies().setRoutingStatus(context.deploymentIdIn(zone4), RoutingStatus.Value.out, + RoutingStatus.Agent.tenant); context.flushDnsUpdates(); // Weight of inactive zone is set to zero @@ -176,16 +177,16 @@ public class RoutingPoliciesTest { // Other zone in shared region is set out. Entire record group for the region is removed as all zones in the // region are out (weight sum = 0) - tester.routingPolicies().setGlobalRoutingStatus(context.deploymentIdIn(zone3), GlobalRouting.Status.out, - GlobalRouting.Agent.tenant); + tester.routingPolicies().setRoutingStatus(context.deploymentIdIn(zone3), RoutingStatus.Value.out, + RoutingStatus.Agent.tenant); context.flushDnsUpdates(); tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, ImmutableMap.of(zone1, 1L)); // Everything is set back in - tester.routingPolicies().setGlobalRoutingStatus(context.deploymentIdIn(zone3), GlobalRouting.Status.in, - GlobalRouting.Agent.tenant); - tester.routingPolicies().setGlobalRoutingStatus(context.deploymentIdIn(zone4), GlobalRouting.Status.in, - GlobalRouting.Agent.tenant); + tester.routingPolicies().setRoutingStatus(context.deploymentIdIn(zone3), RoutingStatus.Value.in, + RoutingStatus.Agent.tenant); + tester.routingPolicies().setRoutingStatus(context.deploymentIdIn(zone4), RoutingStatus.Value.in, + RoutingStatus.Agent.tenant); context.flushDnsUpdates(); tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, ImmutableMap.of(zone1, 1L, zone3, 1L, @@ -480,8 +481,8 @@ public class RoutingPoliciesTest { // Global routing status is overridden in one zone var changedAt = tester.controllerTester().clock().instant(); - tester.routingPolicies().setGlobalRoutingStatus(context.deploymentIdIn(zone1), GlobalRouting.Status.out, - GlobalRouting.Agent.tenant); + tester.routingPolicies().setRoutingStatus(context.deploymentIdIn(zone1), RoutingStatus.Value.out, + RoutingStatus.Agent.tenant); context.flushDnsUpdates(); // Inactive zone is removed from global DNS record @@ -490,15 +491,15 @@ public class RoutingPoliciesTest { // Status details is stored in policy var policy1 = tester.routingPolicies().get(context.deploymentIdIn(zone1)).values().iterator().next(); - assertEquals(GlobalRouting.Status.out, policy1.status().globalRouting().status()); - assertEquals(GlobalRouting.Agent.tenant, policy1.status().globalRouting().agent()); - assertEquals(changedAt.truncatedTo(ChronoUnit.MILLIS), policy1.status().globalRouting().changedAt()); + assertEquals(RoutingStatus.Value.out, policy1.status().routingStatus().value()); + assertEquals(RoutingStatus.Agent.tenant, policy1.status().routingStatus().agent()); + assertEquals(changedAt.truncatedTo(ChronoUnit.MILLIS), policy1.status().routingStatus().changedAt()); // Other zone remains in var policy2 = tester.routingPolicies().get(context.deploymentIdIn(zone2)).values().iterator().next(); - assertEquals(GlobalRouting.Status.in, policy2.status().globalRouting().status()); - assertEquals(GlobalRouting.Agent.system, policy2.status().globalRouting().agent()); - assertEquals(Instant.EPOCH, policy2.status().globalRouting().changedAt()); + assertEquals(RoutingStatus.Value.in, policy2.status().routingStatus().value()); + assertEquals(RoutingStatus.Agent.system, policy2.status().routingStatus().agent()); + assertEquals(Instant.EPOCH, policy2.status().routingStatus().changedAt()); // Next deployment does not affect status context.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy(); @@ -509,15 +510,15 @@ public class RoutingPoliciesTest { // Deployment is set back in tester.controllerTester().clock().advance(Duration.ofHours(1)); changedAt = tester.controllerTester().clock().instant(); - tester.routingPolicies().setGlobalRoutingStatus(context.deploymentIdIn(zone1), GlobalRouting.Status.in, GlobalRouting.Agent.tenant); + tester.routingPolicies().setRoutingStatus(context.deploymentIdIn(zone1), RoutingStatus.Value.in, RoutingStatus.Agent.tenant); context.flushDnsUpdates(); tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone1, zone2); tester.assertTargets(context.instanceId(), EndpointId.of("r1"), 0, zone1, zone2); policy1 = tester.routingPolicies().get(context.deploymentIdIn(zone1)).values().iterator().next(); - assertEquals(GlobalRouting.Status.in, policy1.status().globalRouting().status()); - assertEquals(GlobalRouting.Agent.tenant, policy1.status().globalRouting().agent()); - assertEquals(changedAt.truncatedTo(ChronoUnit.MILLIS), policy1.status().globalRouting().changedAt()); + assertEquals(RoutingStatus.Value.in, policy1.status().routingStatus().value()); + assertEquals(RoutingStatus.Agent.tenant, policy1.status().routingStatus().agent()); + assertEquals(changedAt.truncatedTo(ChronoUnit.MILLIS), policy1.status().routingStatus().changedAt()); // Deployment is set out through a new deployment.xml var applicationPackage2 = applicationPackageBuilder() @@ -562,37 +563,37 @@ public class RoutingPoliciesTest { } // Set zone out - tester.routingPolicies().setGlobalRoutingStatus(zone2, GlobalRouting.Status.out); + tester.routingPolicies().setRoutingStatus(zone2, RoutingStatus.Value.out); context1.flushDnsUpdates(); tester.assertTargets(context1.instanceId(), EndpointId.defaultId(), 0, zone1); tester.assertTargets(context2.instanceId(), EndpointId.defaultId(), 0, zone1); for (var context : contexts) { var policies = tester.routingPolicies().get(context.instanceId()); - assertTrue("Global routing status for policy remains " + GlobalRouting.Status.in, + assertTrue("Global routing status for policy remains " + RoutingStatus.Value.in, policies.values().stream() .map(RoutingPolicy::status) - .map(Status::globalRouting) - .map(GlobalRouting::status) - .allMatch(status -> status == GlobalRouting.Status.in)); + .map(RoutingPolicy.Status::routingStatus) + .map(RoutingStatus::value) + .allMatch(status -> status == RoutingStatus.Value.in)); } var changedAt = tester.controllerTester().clock().instant(); var zonePolicy = tester.controllerTester().controller().curator().readZoneRoutingPolicy(zone2); - assertEquals(GlobalRouting.Status.out, zonePolicy.globalRouting().status()); - assertEquals(GlobalRouting.Agent.operator, zonePolicy.globalRouting().agent()); - assertEquals(changedAt.truncatedTo(ChronoUnit.MILLIS), zonePolicy.globalRouting().changedAt()); + assertEquals(RoutingStatus.Value.out, zonePolicy.routingStatus().value()); + assertEquals(RoutingStatus.Agent.operator, zonePolicy.routingStatus().agent()); + assertEquals(changedAt.truncatedTo(ChronoUnit.MILLIS), zonePolicy.routingStatus().changedAt()); // Setting status per deployment does not affect status as entire zone is out - tester.routingPolicies().setGlobalRoutingStatus(context1.deploymentIdIn(zone2), GlobalRouting.Status.in, GlobalRouting.Agent.tenant); + tester.routingPolicies().setRoutingStatus(context1.deploymentIdIn(zone2), RoutingStatus.Value.in, RoutingStatus.Agent.tenant); context1.flushDnsUpdates(); tester.assertTargets(context1.instanceId(), EndpointId.defaultId(), 0, zone1); tester.assertTargets(context2.instanceId(), EndpointId.defaultId(), 0, zone1); // Set single deployment out - tester.routingPolicies().setGlobalRoutingStatus(context1.deploymentIdIn(zone2), GlobalRouting.Status.out, GlobalRouting.Agent.tenant); + tester.routingPolicies().setRoutingStatus(context1.deploymentIdIn(zone2), RoutingStatus.Value.out, RoutingStatus.Agent.tenant); context1.flushDnsUpdates(); // Set zone back in. Deployment set explicitly out, remains out, the rest are in - tester.routingPolicies().setGlobalRoutingStatus(zone2, GlobalRouting.Status.in); + tester.routingPolicies().setRoutingStatus(zone2, RoutingStatus.Value.in); context1.flushDnsUpdates(); tester.assertTargets(context1.instanceId(), EndpointId.defaultId(), 0, zone1); tester.assertTargets(context2.instanceId(), EndpointId.defaultId(), 0, zone1, zone2); @@ -646,41 +647,41 @@ public class RoutingPoliciesTest { tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone1, zone2); // Global routing status is overridden for one deployment - tester.routingPolicies().setGlobalRoutingStatus(context.deploymentIdIn(zone1), GlobalRouting.Status.out, - GlobalRouting.Agent.tenant); + tester.routingPolicies().setRoutingStatus(context.deploymentIdIn(zone1), RoutingStatus.Value.out, + RoutingStatus.Agent.tenant); context.flushDnsUpdates(); tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone2); // Setting other deployment out implicitly sets all deployments in. Weight is set to zero, but that has no // impact on routing decisions when the weight sum is zero - tester.routingPolicies().setGlobalRoutingStatus(context.deploymentIdIn(zone2), GlobalRouting.Status.out, - GlobalRouting.Agent.tenant); + tester.routingPolicies().setRoutingStatus(context.deploymentIdIn(zone2), RoutingStatus.Value.out, + RoutingStatus.Agent.tenant); context.flushDnsUpdates(); tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, ImmutableMap.of(zone1, 0L, zone2, 0L)); // One inactive deployment is put back in. Global DNS record now points to the only active deployment - tester.routingPolicies().setGlobalRoutingStatus(context.deploymentIdIn(zone1), GlobalRouting.Status.in, - GlobalRouting.Agent.tenant); + tester.routingPolicies().setRoutingStatus(context.deploymentIdIn(zone1), RoutingStatus.Value.in, + RoutingStatus.Agent.tenant); context.flushDnsUpdates(); tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone1); // Setting zone (containing active deployment) out puts all deployments in - tester.routingPolicies().setGlobalRoutingStatus(zone1, GlobalRouting.Status.out); + tester.routingPolicies().setRoutingStatus(zone1, RoutingStatus.Value.out); context.flushDnsUpdates(); - assertEquals(GlobalRouting.Status.out, tester.routingPolicies().get(zone1).globalRouting().status()); + assertEquals(RoutingStatus.Value.out, tester.routingPolicies().get(zone1).routingStatus().value()); tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, ImmutableMap.of(zone1, 0L, zone2, 0L)); // Setting zone back in removes the currently inactive deployment - tester.routingPolicies().setGlobalRoutingStatus(zone1, GlobalRouting.Status.in); + tester.routingPolicies().setRoutingStatus(zone1, RoutingStatus.Value.in); context.flushDnsUpdates(); tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone1); // Inactive deployment is set in - tester.routingPolicies().setGlobalRoutingStatus(context.deploymentIdIn(zone2), GlobalRouting.Status.in, - GlobalRouting.Agent.tenant); + tester.routingPolicies().setRoutingStatus(context.deploymentIdIn(zone2), RoutingStatus.Value.in, + RoutingStatus.Agent.tenant); context.flushDnsUpdates(); for (var policy : tester.routingPolicies().get(context.instanceId()).values()) { - assertSame(GlobalRouting.Status.in, policy.status().globalRouting().status()); + assertSame(RoutingStatus.Value.in, policy.status().routingStatus().value()); } tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone1, zone2); } @@ -701,12 +702,153 @@ public class RoutingPoliciesTest { records.get(0).data()); } + @Test + public void application_endpoint_routing_policy() { + RoutingPoliciesTester tester = new RoutingPoliciesTester(); + TenantAndApplicationId application = TenantAndApplicationId.from("tenant1", "app1"); + ApplicationId betaInstance = application.instance("beta"); + ApplicationId mainInstance = application.instance("main"); + + DeploymentContext betaContext = tester.newDeploymentContext(betaInstance); + DeploymentContext mainContext = tester.newDeploymentContext(mainInstance); + var applicationPackage = applicationPackageBuilder() + .instances("beta,main") + .region(zone1.region()) + .region(zone2.region()) + .applicationEndpoint("a0", "c0", "us-west-1", + Map.of(betaInstance.instance(), 2, + mainInstance.instance(), 8)) + .applicationEndpoint("a1", "c1", "us-central-1", + Map.of(betaInstance.instance(), 4, + mainInstance.instance(), 6)) + .build(); + for (var zone : List.of(zone1, zone2)) { + tester.provisionLoadBalancers(2, betaInstance, zone); + tester.provisionLoadBalancers(2, mainInstance, zone); + } + + // Deploy both instances + betaContext.submit(applicationPackage).deploy(); + + // Application endpoint points to both instances with correct weights + DeploymentId betaZone1 = betaContext.deploymentIdIn(zone1); + DeploymentId mainZone1 = mainContext.deploymentIdIn(zone1); + DeploymentId betaZone2 = betaContext.deploymentIdIn(zone2); + DeploymentId mainZone2 = mainContext.deploymentIdIn(zone2); + tester.assertTargets(application, EndpointId.of("a0"), ClusterSpec.Id.from("c0"), 0, + Map.of(betaZone1, 2, + mainZone1, 8)); + tester.assertTargets(application, EndpointId.of("a1"), ClusterSpec.Id.from("c1"), 1, + Map.of(betaZone2, 4, + mainZone2, 6)); + + // Weights are updated + applicationPackage = applicationPackageBuilder() + .instances("beta,main") + .region(zone1.region()) + .region(zone2.region()) + .applicationEndpoint("a0", "c0", "us-west-1", + Map.of(betaInstance.instance(), 3, + mainInstance.instance(), 7)) + .applicationEndpoint("a1", "c1", "us-central-1", + Map.of(betaInstance.instance(), 1, + mainInstance.instance(), 9)) + .build(); + betaContext.submit(applicationPackage).deploy(); + tester.assertTargets(application, EndpointId.of("a0"), ClusterSpec.Id.from("c0"), 0, + Map.of(betaZone1, 3, + mainZone1, 7)); + tester.assertTargets(application, EndpointId.of("a1"), ClusterSpec.Id.from("c1"), 1, + Map.of(betaZone2, 1, + mainZone2, 9)); + + // An endpoint is removed + applicationPackage = applicationPackageBuilder() + .instances("beta,main") + .region(zone1.region()) + .region(zone2.region()) + .applicationEndpoint("a0", "c0", "us-west-1", + Map.of(betaInstance.instance(), 1)) + .build(); + betaContext.submit(applicationPackage).deploy(); + + // Application endpoints now point to a single instance + tester.assertTargets(application, EndpointId.of("a0"), ClusterSpec.Id.from("c0"), 0, + Map.of(betaZone1, 1)); + assertTrue("Endpoint removed", + tester.controllerTester().controller().routing() + .readDeclaredEndpointsOf(application) + .named(EndpointId.of("a1")).isEmpty()); + } + + @Test + public void application_endpoint_routing_status() { + RoutingPoliciesTester tester = new RoutingPoliciesTester(); + TenantAndApplicationId application = TenantAndApplicationId.from("tenant1", "app1"); + ApplicationId betaInstance = application.instance("beta"); + ApplicationId mainInstance = application.instance("main"); + + DeploymentContext betaContext = tester.newDeploymentContext(betaInstance); + DeploymentContext mainContext = tester.newDeploymentContext(mainInstance); + var applicationPackage = applicationPackageBuilder() + .instances("beta,main") + .region(zone1.region()) + .region(zone2.region()) + .applicationEndpoint("a0", "c0", "us-west-1", + Map.of(betaInstance.instance(), 2, + mainInstance.instance(), 8)) + .applicationEndpoint("a1", "c1", "us-central-1", + Map.of(betaInstance.instance(), 4, + mainInstance.instance(), 0)) + .build(); + for (var zone : List.of(zone1, zone2)) { + tester.provisionLoadBalancers(2, betaInstance, zone); + tester.provisionLoadBalancers(2, mainInstance, zone); + } + + // Deploy both instances + betaContext.submit(applicationPackage).deploy(); + + // Application endpoint points to both instances with correct weights + DeploymentId betaZone1 = betaContext.deploymentIdIn(zone1); + DeploymentId mainZone1 = mainContext.deploymentIdIn(zone1); + DeploymentId betaZone2 = betaContext.deploymentIdIn(zone2); + DeploymentId mainZone2 = mainContext.deploymentIdIn(zone2); + tester.assertTargets(application, EndpointId.of("a0"), ClusterSpec.Id.from("c0"), 0, + Map.of(betaZone1, 2, + mainZone1, 8)); + tester.assertTargets(application, EndpointId.of("a1"), ClusterSpec.Id.from("c1"), 1, + Map.of(betaZone2, 4, + mainZone2, 0)); + + // Changing routing status updates weight + tester.routingPolicies().setRoutingStatus(mainZone1, RoutingStatus.Value.out, RoutingStatus.Agent.tenant); + betaContext.flushDnsUpdates(); + tester.assertTargets(application, EndpointId.of("a0"), ClusterSpec.Id.from("c0"), 0, + Map.of(betaZone1, 2, + mainZone1, 0)); + tester.routingPolicies().setRoutingStatus(mainZone1, RoutingStatus.Value.in, RoutingStatus.Agent.tenant); + betaContext.flushDnsUpdates(); + tester.assertTargets(application, EndpointId.of("a0"), ClusterSpec.Id.from("c0"), 0, + Map.of(betaZone1, 2, + mainZone1, 8)); + + // Changing routing status preserves weights if change in routing status would result in a zero weight sum + // Otherwise this would result in both targets have weight 0 and thus traffic would be distributed evenly across + // all targets which does not match intention of taking out a deployment + tester.routingPolicies().setRoutingStatus(betaZone2, RoutingStatus.Value.out, RoutingStatus.Agent.tenant); + betaContext.flushDnsUpdates(); + tester.assertTargets(application, EndpointId.of("a1"), ClusterSpec.Id.from("c1"), 1, + Map.of(betaZone2, 4, + mainZone2, 0)); + } + /** Returns an application package builder that satisfies requirements for a directly routed endpoint */ private static ApplicationPackageBuilder applicationPackageBuilder() { - return new ApplicationPackageBuilder() - .athenzIdentity(AthenzDomain.from("domain"), AthenzService.from("service")); + return new ApplicationPackageBuilder().athenzIdentity(AthenzDomain.from("domain"), + AthenzService.from("service")); } - + private static List<LoadBalancer> createLoadBalancers(ZoneId zone, ApplicationId application, boolean shared, int count) { List<LoadBalancer> loadBalancers = new ArrayList<>(); for (int i = 0; i < count; i++) { @@ -819,12 +961,40 @@ public class RoutingPoliciesTest { .collect(Collectors.toList()); } - private void assertTargets(ApplicationId instance, EndpointId endpointId, ClusterSpec.Id cluster, int loadBalancerId, Map<ZoneId, Long> zoneWeights) { + /** Assert that an application endpoint points to given targets and weights */ + private void assertTargets(TenantAndApplicationId application, EndpointId endpointId, ClusterSpec.Id cluster, + int loadBalancerId, Map<DeploymentId, Integer> deploymentWeights) { + Map<String, List<DeploymentId>> deploymentsByDnsName = new HashMap<>(); + for (var deployment : deploymentWeights.keySet()) { + EndpointList applicationEndpoints = tester.controller().routing().readDeclaredEndpointsOf(application) + .named(endpointId) + .targets(deployment) + .cluster(cluster); + assertEquals("Expected a single endpoint with ID '" + endpointId + "'", 1, + applicationEndpoints.size()); + String dnsName = applicationEndpoints.asList().get(0).dnsName(); + deploymentsByDnsName.computeIfAbsent(dnsName, (k) -> new ArrayList<>()) + .add(deployment); + } + assertEquals("Found " + endpointId + " for " + application, 1, deploymentsByDnsName.size()); + deploymentsByDnsName.forEach((dnsName, deployments) -> { + Set<String> weightedTargets = deployments.stream() + .map(d -> "weighted/lb-" + loadBalancerId + "--" + + d.applicationId().serializedForm() + "--" + d.zoneId().value() + + "/dns-zone-1/" + d.zoneId().value() + "/" + deploymentWeights.get(d)) + .collect(Collectors.toSet()); + assertEquals(dnsName + " has expected targets", weightedTargets, aliasDataOf(dnsName)); + }); + } + + /** Assert that an instance endpoint points to given targets and weights */ + private void assertTargets(ApplicationId instance, EndpointId endpointId, ClusterSpec.Id cluster, + int loadBalancerId, Map<ZoneId, Long> zoneWeights) { Set<String> latencyTargets = new HashSet<>(); Map<String, List<ZoneId>> zonesByRegionEndpoint = new HashMap<>(); for (var zone : zoneWeights.keySet()) { DeploymentId deployment = new DeploymentId(instance, zone); - EndpointList regionEndpoints = tester.controller().routing().endpointsOf(deployment) + EndpointList regionEndpoints = tester.controller().routing().readEndpointsOf(deployment) .cluster(cluster) .scope(Endpoint.Scope.weighted); Endpoint regionEndpoint = regionEndpoints.first().orElseThrow(() -> new IllegalArgumentException("No region endpoint found for " + cluster + " in " + deployment)); @@ -845,7 +1015,7 @@ public class RoutingPoliciesTest { latencyTargets.add(latencyTarget); }); List<DeploymentId> deployments = zoneWeights.keySet().stream().map(z -> new DeploymentId(instance, z)).collect(Collectors.toList()); - String globalEndpoint = tester.controller().routing().endpointsOf(instance) + String globalEndpoint = tester.controller().routing().readDeclaredEndpointsOf(instance) .named(endpointId) .targets(deployments) .primary() diff --git a/dist/vespa.spec b/dist/vespa.spec index 92713517988..81a180348c2 100644 --- a/dist/vespa.spec +++ b/dist/vespa.spec @@ -85,7 +85,7 @@ BuildRequires: vespa-lz4-devel >= 1.9.2-2 BuildRequires: vespa-onnxruntime-devel = 1.7.1 BuildRequires: vespa-openssl-devel >= 1.1.1l-1 %define _use_vespa_openssl 1 -BuildRequires: vespa-protobuf-devel = 3.17.3 +BuildRequires: vespa-protobuf-devel = 3.19.1 BuildRequires: vespa-libzstd-devel >= 1.4.5-2 %endif %if 0%{?el8} @@ -112,7 +112,7 @@ BuildRequires: vespa-gtest = 1.11.0 %define _use_vespa_gtest 1 BuildRequires: vespa-lz4-devel >= 1.9.2-2 BuildRequires: vespa-onnxruntime-devel = 1.7.1 -BuildRequires: vespa-protobuf-devel = 3.17.3 +BuildRequires: vespa-protobuf-devel = 3.19.1 BuildRequires: vespa-libzstd-devel >= 1.4.5-2 %endif %if 0%{?fedora} @@ -161,14 +161,14 @@ BuildRequires: gmock-devel %if 0%{?el7} && 0%{?amzn2} BuildRequires: vespa-xxhash-devel = 0.8.0 %define _use_vespa_xxhash 1 -BuildRequires: vespa-openblas-devel = 0.3.17 +BuildRequires: vespa-openblas-devel = 0.3.18 %define _use_vespa_openblas 1 BuildRequires: vespa-re2-devel = 20210801 %define _use_vespa_re2 1 %else BuildRequires: xxhash-devel >= 0.8.0 %if 0%{?el7} || 0%{?el8} -BuildRequires: vespa-openblas-devel = 0.3.17 +BuildRequires: vespa-openblas-devel = 0.3.18 %define _use_vespa_openblas 1 %else BuildRequires: openblas-devel @@ -335,7 +335,7 @@ Requires: openssl-libs Requires: vespa-lz4 >= 1.9.2-2 Requires: vespa-libzstd >= 1.4.5-2 %if 0%{?el8} || 0%{?el7} -Requires: vespa-openblas = 0.3.17 +Requires: vespa-openblas = 0.3.18 %else Requires: openblas-serial %endif @@ -360,7 +360,7 @@ Requires: %{name}-base-libs = %{version}-%{release} %if 0%{?el7} Requires: llvm7.0-libs Requires: vespa-icu >= 65.1.0-1 -Requires: vespa-protobuf = 3.17.3 +Requires: vespa-protobuf = 3.19.1 %else Requires: libicu %endif @@ -379,7 +379,7 @@ Requires: (llvm-libs >= 11.0.0 and llvm-libs < 12) %else Requires: (llvm-libs >= 10.0.1 and llvm-libs < 11) %endif -Requires: vespa-protobuf = 3.17.3 +Requires: vespa-protobuf = 3.19.1 %endif %if 0%{?fedora} Requires: protobuf diff --git a/docproc/abi-spec.json b/docproc/abi-spec.json index 34bf2c21824..e2bbfd8ec10 100644 --- a/docproc/abi-spec.json +++ b/docproc/abi-spec.json @@ -106,6 +106,7 @@ "public void <init>(java.lang.String, com.yahoo.statistics.Statistics, com.yahoo.jdisc.Metric)", "public void <init>(java.lang.String, com.yahoo.jdisc.Metric)", "public void <init>(com.yahoo.docproc.CallStack)", + "public void <init>(java.lang.String, java.util.Collection, com.yahoo.jdisc.Metric)", "public void <init>(java.lang.String, java.util.Collection, com.yahoo.statistics.Statistics, com.yahoo.jdisc.Metric)", "public java.lang.String getName()", "public void setName(java.lang.String)", diff --git a/docproc/src/main/java/com/yahoo/docproc/CallStack.java b/docproc/src/main/java/com/yahoo/docproc/CallStack.java index 7e0920f717c..9f59d9e869b 100644 --- a/docproc/src/main/java/com/yahoo/docproc/CallStack.java +++ b/docproc/src/main/java/com/yahoo/docproc/CallStack.java @@ -51,7 +51,7 @@ public class CallStack { public CallStack(Statistics statistics, Metric metric) { this(metric); } - + @Deprecated public CallStack(String name, Statistics manager, Metric metric) { this(name, metric); } @@ -81,12 +81,16 @@ public class CallStack { * @param name the name of the stack * @param docprocs the document processors to call */ - public CallStack(String name, Collection<DocumentProcessor> docprocs, Statistics manager, Metric metric) { - this(name, manager, metric); + public CallStack(String name, Collection<DocumentProcessor> docprocs, Metric metric) { + this(name, metric); for (DocumentProcessor docproc : docprocs) { addLast(docproc); } } + @Deprecated + public CallStack(String name, Collection<DocumentProcessor> docprocs, Statistics manager, Metric metric) { + this(name, docprocs, metric); + } /** Returns the name of this stack, or null if it is not named */ public String getName() { @@ -380,6 +384,7 @@ public class CallStack { return b.toString(); } + @Deprecated public Statistics getStatistics() { return null; } diff --git a/document/abi-spec.json b/document/abi-spec.json index 88b9fef86cb..39a93b2b2cb 100644 --- a/document/abi-spec.json +++ b/document/abi-spec.json @@ -51,8 +51,7 @@ ], "fields": [ "protected java.util.Map fieldIds", - "protected java.util.Map fields", - "protected com.yahoo.compress.Compressor compressor" + "protected java.util.Map fields" ] }, "com.yahoo.document.BucketDistribution": { diff --git a/document/src/main/java/com/yahoo/document/BaseStructDataType.java b/document/src/main/java/com/yahoo/document/BaseStructDataType.java index 97357f3fa7c..1bce5d716a4 100755 --- a/document/src/main/java/com/yahoo/document/BaseStructDataType.java +++ b/document/src/main/java/com/yahoo/document/BaseStructDataType.java @@ -18,8 +18,6 @@ public abstract class BaseStructDataType extends StructuredDataType { protected Map<Integer, Field> fieldIds = new LinkedHashMap<>(); protected Map<String, Field> fields = new LinkedHashMap<>(); - protected Compressor compressor = new Compressor(CompressionType.NONE); - BaseStructDataType(String name) { super(name); } @@ -101,27 +99,25 @@ public abstract class BaseStructDataType extends StructuredDataType { return fields.size(); } - /** Returns the compressor to use to compress data of this type */ - public Compressor getCompressor() { return compressor; } + /** Returns the compressor to use to compress data of this type + * @deprecated Will go away on Vespa 8 + */ + @Deprecated + public Compressor getCompressor() { return new Compressor(CompressionType.NONE); } - /** Returns a view of the configuration of the compressor used to compress this type */ + /** Returns a view of the configuration of the compressor used to compress this type + * @deprecated Will go away on Vespa 8 + */ + @Deprecated public CompressionConfig getCompressionConfig() { - // CompressionConfig accepts a percentage (but exposes a factor) ... - float compressionThresholdPercentage = (float)compressor.compressionThresholdFactor() * 100; - - return new CompressionConfig(compressor.type(), - compressor.level(), - compressionThresholdPercentage, - compressor.compressMinSizeBytes()); + return new CompressionConfig(); } - /** Set the config to the compressor used to compress data of this type */ - public void setCompressionConfig(CompressionConfig config) { - CompressionType type = config.type; - compressor = new Compressor(type, - config.compressionLevel, - config.thresholdFactor(), - (int)config.minsize); - } + /** + * Set the config to the compressor used to compress data of this type + * @deprecated Ignored and will go away on Vespa 8 + */ + @Deprecated + public void setCompressionConfig(CompressionConfig config) { } } diff --git a/document/src/main/java/com/yahoo/document/DocumentTypeManagerConfigurer.java b/document/src/main/java/com/yahoo/document/DocumentTypeManagerConfigurer.java index db0566aecc1..a0ac3cb6620 100644 --- a/document/src/main/java/com/yahoo/document/DocumentTypeManagerConfigurer.java +++ b/document/src/main/java/com/yahoo/document/DocumentTypeManagerConfigurer.java @@ -10,7 +10,6 @@ import java.util.logging.Level; import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; -import java.util.HashSet; import java.util.Map; import java.util.logging.Logger; import java.util.stream.Collectors; @@ -24,17 +23,14 @@ public class DocumentTypeManagerConfigurer implements ConfigSubscriber.SingleSub private final static Logger log = Logger.getLogger(DocumentTypeManagerConfigurer.class.getName()); - private DocumentTypeManager managerToConfigure; + private final DocumentTypeManager managerToConfigure; public DocumentTypeManagerConfigurer(DocumentTypeManager manager) { this.managerToConfigure = manager; } - private static CompressionConfig makeCompressionConfig(DocumentmanagerConfig.Datatype.Structtype cfg) { - return new CompressionConfig(toCompressorType(cfg.compresstype()), cfg.compresslevel(), - cfg.compressthreshold(), cfg.compressminsize()); - } - + /** Deprecated and will go away on Vespa 8 */ + @Deprecated public static CompressionType toCompressorType(DocumentmanagerConfig.Datatype.Structtype.Compresstype.Enum value) { switch (value) { case NONE: return CompressionType.NONE; @@ -43,7 +39,6 @@ public class DocumentTypeManagerConfigurer implements ConfigSubscriber.SingleSub } throw new IllegalArgumentException("Compression type " + value + " is not supported"); } - /** * <p>Makes the DocumentTypeManager subscribe on its config.</p> * @@ -73,8 +68,7 @@ public class DocumentTypeManagerConfigurer implements ConfigSubscriber.SingleSub setupAnnotationRefTypes(config, manager); log.log(Level.FINE, "Configuring document manager with " + config.datatype().size() + " data types."); - ArrayList<DocumentmanagerConfig.Datatype> failed = new ArrayList<>(); - failed.addAll(config.datatype()); + ArrayList<DocumentmanagerConfig.Datatype> failed = new ArrayList<>(config.datatype()); while (!failed.isEmpty()) { ArrayList<DocumentmanagerConfig.Datatype> tmp = failed; failed = new ArrayList<>(); @@ -82,7 +76,7 @@ public class DocumentTypeManagerConfigurer implements ConfigSubscriber.SingleSub DocumentmanagerConfig.Datatype thisDataType = tmp.get(i); int id = thisDataType.id(); try { - registerTypeIdMapping(config, manager, thisDataType, id); + registerTypeIdMapping(manager, thisDataType, id); } catch (IllegalArgumentException e) { failed.add(thisDataType); } @@ -95,24 +89,24 @@ public class DocumentTypeManagerConfigurer implements ConfigSubscriber.SingleSub manager.replaceTemporaryTypes(); } - private static void registerTypeIdMapping(DocumentmanagerConfig config, DocumentTypeManager manager, DocumentmanagerConfig.Datatype thisDataType, int id) { - for (Object o : thisDataType.arraytype()) { - registerArrayType(manager, id, (DocumentmanagerConfig.Datatype.Arraytype) o); + private static void registerTypeIdMapping(DocumentTypeManager manager, DocumentmanagerConfig.Datatype thisDataType, int id) { + for (var o : thisDataType.arraytype()) { + registerArrayType(manager, id, o); } - for (Object o : thisDataType.maptype()) { - registerMapType(manager, id, (DocumentmanagerConfig.Datatype.Maptype) o); + for (var o : thisDataType.maptype()) { + registerMapType(manager, id, o); } - for (Object o : thisDataType.weightedsettype()) { - registerWeightedSetType(manager, id, (DocumentmanagerConfig.Datatype.Weightedsettype) o); + for (var o : thisDataType.weightedsettype()) { + registerWeightedSetType(manager, id, o); } - for (Object o : thisDataType.structtype()) { - registerStructType(config, manager, id, (DocumentmanagerConfig.Datatype.Structtype) o); + for (var o : thisDataType.structtype()) { + registerStructType(manager, id, o); } - for (Object o : thisDataType.documenttype()) { - registerDocumentType(manager, (DocumentmanagerConfig.Datatype.Documenttype) o); + for (var o : thisDataType.documenttype()) { + registerDocumentType(manager, o); } - for (Object o : thisDataType.referencetype()) { - registerReferenceType(manager, id, (DocumentmanagerConfig.Datatype.Referencetype) o); + for (var o : thisDataType.referencetype()) { + registerReferenceType(manager, id, o); } } @@ -139,20 +133,17 @@ public class DocumentTypeManagerConfigurer implements ConfigSubscriber.SingleSub manager.register(type); } - @SuppressWarnings("deprecation") private static void registerDocumentType(DocumentTypeManager manager, DocumentmanagerConfig.Datatype.Documenttype doc) { StructDataType header = (StructDataType) manager.getDataType(doc.headerstruct(), ""); var importedFields = doc.importedfield().stream() .map(f -> f.name()) .collect(Collectors.toUnmodifiableSet()); DocumentType type = new DocumentType(doc.name(), header, importedFields); - for (Object j : doc.inherits()) { - DocumentmanagerConfig.Datatype.Documenttype.Inherits parent = - (DocumentmanagerConfig.Datatype.Documenttype.Inherits) j; + for (var parent : doc.inherits()) { DataTypeName name = new DataTypeName(parent.name()); DocumentType parentType = manager.getDocumentType(name); if (parentType == null) { - throw new IllegalArgumentException("Could not find document type '" + name.toString() + "'."); + throw new IllegalArgumentException("Could not find document type '" + name + "'."); } type.inherit(parentType); } @@ -164,18 +155,11 @@ public class DocumentTypeManagerConfigurer implements ConfigSubscriber.SingleSub manager.register(type); } - private static void registerStructType(DocumentmanagerConfig config, DocumentTypeManager manager, int id, + private static void registerStructType(DocumentTypeManager manager, int id, DocumentmanagerConfig.Datatype.Structtype struct) { StructDataType type = new StructDataType(id, struct.name()); - if (config.enablecompression()) { - CompressionConfig comp = makeCompressionConfig(struct); - type.setCompressionConfig(comp); - } - - for (Object j : struct.field()) { - DocumentmanagerConfig.Datatype.Structtype.Field field = - (DocumentmanagerConfig.Datatype.Structtype.Field) j; + for (var field : struct.field()) { DataType fieldType = (field.datatype() == id) ? manager.getDataTypeAndReturnTemporary(field.datatype(), field.detailedtype()) : manager.getDataType(field.datatype(), field.detailedtype()); @@ -231,8 +215,7 @@ public class DocumentTypeManagerConfigurer implements ConfigSubscriber.SingleSub for (int i = 0; i < config.datatype().size(); i++) { DocumentmanagerConfig.Datatype thisDataType = config.datatype(i); int id = thisDataType.id(); - for (Object o : thisDataType.annotationreftype()) { - DocumentmanagerConfig.Datatype.Annotationreftype annRefType = (DocumentmanagerConfig.Datatype.Annotationreftype) o; + for (var annRefType : thisDataType.annotationreftype()) { AnnotationType annotationType = manager.getAnnotationTypeRegistry().getType(annRefType.annotation()); if (annotationType == null) { throw new IllegalArgumentException("Found reference to " + annRefType.annotation() + ", which does not exist!"); @@ -275,11 +258,10 @@ public class DocumentTypeManagerConfigurer implements ConfigSubscriber.SingleSub for (int i = 0; i < config.datatype().size(); i++) { DocumentmanagerConfig.Datatype thisDataType = config.datatype(i); int id = thisDataType.id(); - for (Object o : thisDataType.structtype()) { - DocumentmanagerConfig.Datatype.Structtype struct = (DocumentmanagerConfig.Datatype.Structtype) o; + for (var struct : thisDataType.structtype()) { StructDataType thisStruct = (StructDataType) manager.getDataType(id, ""); - for (DocumentmanagerConfig.Datatype.Structtype.Inherits parent : struct.inherits()) { + for (var parent : struct.inherits()) { StructDataType parentStruct = (StructDataType) manager.getDataType(parent.name()); thisStruct.inherit(parentStruct); } diff --git a/document/src/main/java/com/yahoo/document/datatypes/Struct.java b/document/src/main/java/com/yahoo/document/datatypes/Struct.java index fd13885ac36..b47dbe69199 100644 --- a/document/src/main/java/com/yahoo/document/datatypes/Struct.java +++ b/document/src/main/java/com/yahoo/document/datatypes/Struct.java @@ -65,25 +65,20 @@ public class Struct extends StructuredFieldValue { return this.version; } + /** @deprecated Will go away on Vespa 8 */ + @Deprecated public com.yahoo.compress.CompressionType getCompressionType() { - if (getDataType().getCompressionConfig() == null) { - return com.yahoo.compress.CompressionType.NONE; - } - return getDataType().getCompressionConfig().type; + return com.yahoo.compress.CompressionType.NONE; } - public int getCompressionLevel() { - if ( getDataType().getCompressionConfig() == null) { - return 9; - } - return getDataType().getCompressionConfig().compressionLevel; - } + /** @deprecated Will go away on Vespa 8 */ + @Deprecated + public int getCompressionLevel() { return 9; } + /** @deprecated Will go away on Vespa 8 */ + @Deprecated public float getCompressionThreshold() { - if (getDataType().getCompressionConfig() == null) { - return .95f; - } - return getDataType().getCompressionConfig().threshold; + return .95f; } @Override diff --git a/document/src/main/java/com/yahoo/document/serialization/VespaDocumentSerializer6.java b/document/src/main/java/com/yahoo/document/serialization/VespaDocumentSerializer6.java index cbcd0c64bd0..09e41a0e8bf 100644 --- a/document/src/main/java/com/yahoo/document/serialization/VespaDocumentSerializer6.java +++ b/document/src/main/java/com/yahoo/document/serialization/VespaDocumentSerializer6.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.document.serialization; -import com.yahoo.compress.Compressor; +import com.yahoo.compress.CompressionType; import com.yahoo.document.ArrayDataType; import com.yahoo.document.CollectionDataType; @@ -53,7 +53,6 @@ import com.yahoo.document.update.TensorModifyUpdate; import com.yahoo.document.update.TensorRemoveUpdate; import com.yahoo.document.update.ValueUpdate; import com.yahoo.io.GrowableByteBuffer; -import com.yahoo.tensor.serialization.TypedBinaryFormat; import com.yahoo.vespa.objects.BufferSerializer; import com.yahoo.vespa.objects.FieldBase; @@ -344,19 +343,10 @@ public class VespaDocumentSerializer6 extends BufferSerializer implements Docume buffer.flip(); buf = bigBuffer; - int uncompressedSize = buffer.remaining(); - Compressor.Compression compression = - s.getDataType().getCompressor().compress(buffer.getByteBuffer().array(), buffer.remaining()); - // Actual serialization starts here. int lenPos = buf.position(); putInt(null, 0); // Move back to this after compression is done. - buf.put(compression.type().getCode()); - - if (compression.data() != null && compression.type().isCompressed()) { - buf.putInt2_4_8Bytes(uncompressedSize); - } - + buf.put(CompressionType.NONE.getCode()); buf.putInt1_4Bytes(s.getFieldCount()); for (int i = 0; i < s.getFieldCount(); ++i) { @@ -365,11 +355,7 @@ public class VespaDocumentSerializer6 extends BufferSerializer implements Docume } int pos = buf.position(); - if (compression.data() != null && compression.type().isCompressed()) { - put(null, compression.data()); - } else { - put(null, buffer.getByteBuffer()); - } + put(null, buffer.getByteBuffer()); int dataLength = buf.position() - pos; int posNow = buf.position(); diff --git a/document/src/test/java/com/yahoo/document/DocumentSerializationTestCase.java b/document/src/test/java/com/yahoo/document/DocumentSerializationTestCase.java index 5f283df9614..db877bb9712 100644 --- a/document/src/test/java/com/yahoo/document/DocumentSerializationTestCase.java +++ b/document/src/test/java/com/yahoo/document/DocumentSerializationTestCase.java @@ -112,21 +112,11 @@ public class DocumentSerializationTestCase extends AbstractTypesTest { weightedSet.put(new StringFieldValue("Weighted 1"), 199); doc.setFieldValue("wsfield", weightedSet); - CompressionConfig noncomp = new CompressionConfig(); - CompressionConfig lz4comp = new CompressionConfig(CompressionType.LZ4); { - doc.getDataType().contentStruct().setCompressionConfig(noncomp); FileOutputStream fout = new FileOutputStream(path + "document-java-currentversion-uncompressed.dat", false); doc.serialize(fout); fout.close(); } - { - doc.getDataType().contentStruct().setCompressionConfig(lz4comp); - FileOutputStream fout = new FileOutputStream(path + "document-java-currentversion-lz4-9.dat", false); - doc.serialize(fout); - doc.getDataType().contentStruct().setCompressionConfig(noncomp); - fout.close(); - } } class TestDoc { diff --git a/document/src/test/java/com/yahoo/document/DocumentTestCase.java b/document/src/test/java/com/yahoo/document/DocumentTestCase.java index 5a694d21dd8..144c7d62894 100644 --- a/document/src/test/java/com/yahoo/document/DocumentTestCase.java +++ b/document/src/test/java/com/yahoo/document/DocumentTestCase.java @@ -428,7 +428,7 @@ public class DocumentTestCase extends DocumentTestCaseBase { } } - class ModifyIteratorHandler extends FieldPathIteratorHandler { + static class ModifyIteratorHandler extends FieldPathIteratorHandler { public ModificationStatus doModify(FieldValue fv) { if (fv instanceof StringFieldValue) { @@ -444,7 +444,7 @@ public class DocumentTestCase extends DocumentTestCaseBase { } } - class AddIteratorHandler extends FieldPathIteratorHandler { + static class AddIteratorHandler extends FieldPathIteratorHandler { @SuppressWarnings("unchecked") public ModificationStatus doModify(FieldValue fv) { @@ -658,7 +658,7 @@ public class DocumentTestCase extends DocumentTestCaseBase { validateCppDoc(doc); } - public void validateCppDoc(Document doc) throws IOException { + public void validateCppDoc(Document doc) { validateCppDocNotMap(doc); MapFieldValue map = (MapFieldValue)doc.getFieldValue("mapfield"); assertEquals(map.get(new StringFieldValue("foo1")), new StringFieldValue("bar1")); @@ -666,7 +666,7 @@ public class DocumentTestCase extends DocumentTestCaseBase { } @SuppressWarnings("unchecked") - public void validateCppDocNotMap(Document doc) throws IOException { + public void validateCppDocNotMap(Document doc) { // in practice to validate v6 serialization assertEquals("id:ns:serializetest::http://test.doc.id/", doc.getId().toString()); assertEquals(new IntegerFieldValue(5), doc.getFieldValue("intfield")); @@ -694,7 +694,6 @@ public class DocumentTestCase extends DocumentTestCaseBase { } @Test - @SuppressWarnings("deprecation") public void testGenerateSerializedFile() throws IOException { docMan = setUpCppDocType(); @@ -744,18 +743,6 @@ public class DocumentTestCase extends DocumentTestCaseBase { FileOutputStream fos = new FileOutputStream("src/tests/data/serializejava.dat"); fos.write(buf.array(), 0, size); fos.close(); - - CompressionConfig noncomp = new CompressionConfig(); - CompressionConfig lz4comp = new CompressionConfig(CompressionType.LZ4); - - doc.getDataType().contentStruct().setCompressionConfig(lz4comp); - buf = new GrowableByteBuffer(size, 2.0f); - - doc.serialize(buf); - doc.getDataType().contentStruct().setCompressionConfig(noncomp); - fos = new FileOutputStream("src/tests/data/serializejava-compressed.dat"); - fos.write(buf.array(), 0, buf.position()); - fos.close(); } @Test @@ -801,53 +788,6 @@ public class DocumentTestCase extends DocumentTestCaseBase { } @Test - @SuppressWarnings("deprecation") - public void testSerializeDeserializeCompressed() { - setUpSertestDocType(); - Document doc = getSertestDocument(); - - CompressionConfig noncomp = new CompressionConfig(); - CompressionConfig lz4comp = new CompressionConfig(CompressionType.LZ4); - - doc.getDataType().contentStruct().setCompressionConfig(lz4comp); - - GrowableByteBuffer data = new GrowableByteBuffer(); - doc.serialize(data); - int size = doc.getSerializedSize(); - doc.getDataType().contentStruct().setCompressionConfig(noncomp); - - assertEquals(size, data.position()); - - data.flip(); - - try { - FileOutputStream fos = new FileOutputStream("src/test/files/testser.dat"); - fos.write(data.array(), 0, data.remaining()); - fos.close(); - } catch (Exception e) { - } - - Document doc2 = docMan.createDocument(data); - - assertEquals(doc.getFieldValue("mailid"), doc2.getFieldValue("mailid")); - assertEquals(doc.getFieldValue("date"), doc2.getFieldValue("date")); - assertEquals(doc.getFieldValue("from"), doc2.getFieldValue("from")); - assertEquals(doc.getFieldValue("to"), doc2.getFieldValue("to")); - assertEquals(doc.getFieldValue("subject"), doc2.getFieldValue("subject")); - assertEquals(doc.getFieldValue("body"), doc2.getFieldValue("body")); - assertEquals(doc.getFieldValue("attachmentcount"), doc2.getFieldValue("attachmentcount")); - assertEquals(doc.getFieldValue("attachments"), doc2.getFieldValue("attachments")); - byte[] docRawBytes = ((Raw)doc.getFieldValue("rawfield")).getByteBuffer().array(); - byte[] doc2RawBytes = ((Raw)doc2.getFieldValue("rawfield")).getByteBuffer().array(); - assertEquals(docRawBytes.length, doc2RawBytes.length); - for (int i = 0; i < docRawBytes.length; i++) { - assertEquals(docRawBytes[i], doc2RawBytes[i]); - } - assertEquals(doc.getFieldValue("weightedfield"), doc2.getFieldValue("weightedfield")); - assertEquals(doc.getFieldValue("mapfield"), doc2.getFieldValue("mapfield")); - } - - @Test public void testDeserialize() { setUpSertestDocType(); @@ -937,7 +877,7 @@ public class DocumentTestCase extends DocumentTestCaseBase { } @Test - public void testCompressionConfigured() { + public void testCompressionConfiguredIsIgnored() { int size_uncompressed; { @@ -966,7 +906,7 @@ public class DocumentTestCase extends DocumentTestCaseBase { doc.serialize(data); int size_compressed = data.position(); - assertTrue(size_compressed + " < " + size_uncompressed, size_compressed < size_uncompressed); + assertEquals(size_compressed, size_uncompressed); } @Test diff --git a/document/src/test/java/com/yahoo/document/serialization/VespaDocumentSerializerTestCase.java b/document/src/test/java/com/yahoo/document/serialization/VespaDocumentSerializerTestCase.java index 379089f1c79..9814e765a89 100644 --- a/document/src/test/java/com/yahoo/document/serialization/VespaDocumentSerializerTestCase.java +++ b/document/src/test/java/com/yahoo/document/serialization/VespaDocumentSerializerTestCase.java @@ -1,27 +1,18 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.document.serialization; -import com.yahoo.compress.CompressionType; -import com.yahoo.compress.Compressor; -import com.yahoo.document.CompressionConfig; import com.yahoo.document.DataType; import com.yahoo.document.Document; import com.yahoo.document.DocumentType; -import com.yahoo.document.DocumentTypeManager; import com.yahoo.document.Field; -import com.yahoo.document.MapDataType; -import com.yahoo.document.StructDataType; import com.yahoo.document.datatypes.IntegerFieldValue; -import com.yahoo.document.datatypes.MapFieldValue; import com.yahoo.document.datatypes.PredicateFieldValue; import com.yahoo.document.datatypes.StringFieldValue; -import com.yahoo.document.datatypes.Struct; import com.yahoo.io.GrowableByteBuffer; import org.junit.Test; import org.mockito.Mockito; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; /** * @author Simon Thoresen Hult @@ -57,76 +48,4 @@ public class VespaDocumentSerializerTestCase { Mockito.verify(predicate, Mockito.times(1)).serialize(Mockito.same(field), Mockito.any(FieldWriter.class)); } - static class CompressionFixture { - - static final String COMPRESSABLE_STRING = "zippy zip mc zippington the 3rd zippy zip"; - - final DocumentTypeManager manager; - final DocumentType docType; - final StructDataType nestedType; - final MapDataType mapType; - - CompressionFixture() { - docType = new DocumentType("map_of_structs"); - docType.contentStruct().setCompressionConfig(new CompressionConfig(CompressionType.LZ4)); - - nestedType = new StructDataType("nested_type"); - nestedType.addField(new Field("str", DataType.STRING)); - - mapType = new MapDataType(DataType.STRING, nestedType); - docType.addField(new Field("map", mapType)); - - manager = new DocumentTypeManager(); - manager.registerDocumentType(docType); - } - - static GrowableByteBuffer asSerialized(Document inputDoc) { - GrowableByteBuffer buf = new GrowableByteBuffer(); - inputDoc.serialize(buf); - buf.flip(); - return buf; - } - - Document roundtripSerialize(Document inputDoc) { - return manager.createDocument(asSerialized(inputDoc)); - } - } - - @Test - public void compressed_map_of_compressed_structs_is_supported() { - CompressionFixture fixture = new CompressionFixture(); - - Document doc = new Document(fixture.docType, "id:foo:map_of_structs::flarn"); - Struct nested = new Struct(fixture.nestedType); - nested.setFieldValue("str", new StringFieldValue(CompressionFixture.COMPRESSABLE_STRING)); - - MapFieldValue<StringFieldValue, Struct> map = new MapFieldValue<StringFieldValue, Struct>(fixture.mapType); - map.put(new StringFieldValue("foo"), nested); - map.put(new StringFieldValue("bar"), nested); - doc.setFieldValue("map", map); - - // Should _not_ throw any deserialization exceptions - Document result = fixture.roundtripSerialize(doc); - assertEquals(doc, result); - } - - @Test - public void incompressable_structs_are_serialized_without_buffer_size_overhead_bug() { - CompressionFixture fixture = new CompressionFixture(); - - Document doc = new Document(fixture.docType, "id:foo:map_of_structs::flarn"); - Struct nested = new Struct(fixture.nestedType); - nested.setFieldValue("str", new StringFieldValue(CompressionFixture.COMPRESSABLE_STRING)); - - MapFieldValue<StringFieldValue, Struct> map = new MapFieldValue<StringFieldValue, Struct>(fixture.mapType); - // Only 1 struct added. Not enough redundant information that header struct containing map itself - // can be compressed. - map.put(new StringFieldValue("foo"), nested); - doc.setFieldValue("map", map); - - GrowableByteBuffer buf = CompressionFixture.asSerialized(doc); - // Explanation of arbitrary value: buffer copy bug meant that incompressable structs were all serialized - // rounded up to 4096 bytes. - assertTrue(buf.remaining() < 4096); - } } diff --git a/document/src/test/serializeddocuments/.gitignore b/document/src/test/serializeddocuments/.gitignore index 7ef24eb7f2c..23eba06a7b0 100644 --- a/document/src/test/serializeddocuments/.gitignore +++ b/document/src/test/serializeddocuments/.gitignore @@ -1,2 +1 @@ -document-java-currentversion-lz4-9.dat document-java-currentversion-uncompressed.dat diff --git a/document/src/test/serializeddocuments/document-java-currentversion-lz4-9.dat b/document/src/test/serializeddocuments/document-java-currentversion-lz4-9.dat Binary files differnew file mode 100644 index 00000000000..033844ac09b --- /dev/null +++ b/document/src/test/serializeddocuments/document-java-currentversion-lz4-9.dat diff --git a/document/src/tests/documenttestcase.cpp b/document/src/tests/documenttestcase.cpp index 9fa72b7c1e7..aae4b5c6f40 100644 --- a/document/src/tests/documenttestcase.cpp +++ b/document/src/tests/documenttestcase.cpp @@ -19,7 +19,6 @@ #include <gmock/gmock.h> using vespalib::nbostream; -using vespalib::compression::CompressionConfig; using namespace ::testing; using namespace document::config_builder; @@ -699,7 +698,6 @@ TEST(DocumentTest,testReadSerializedAllVersions) doc.setValue("rawfield", RawFieldValue("RAW DATA", 8)); Document docInDoc(*docInDocType, DocumentId("id:ns:docindoc::http://doc.in.doc/")); docInDoc.set("stringindocfield", "Elvis is dead"); - //docInDoc.setCompression(CompressionConfig(CompressionConfig::NONE, 0, 0)); doc.setValue("docfield", docInDoc); ArrayFieldValue floatArray(*arrayOfFloatDataType); floatArray.add(1.0); @@ -713,7 +711,6 @@ TEST(DocumentTest,testReadSerializedAllVersions) // Write document to disk, (when you bump version and alter stuff, // you can copy this current to new test for new version) { - //doc.setCompression(CompressionConfig(CompressionConfig::NONE, 0, 0)); nbostream buf = doc.serialize(); int fd = open(TEST_PATH("data/document-cpp-currentversion-uncompressed.dat").c_str(), O_WRONLY | O_CREAT | O_TRUNC, 0644); @@ -722,20 +719,6 @@ TEST(DocumentTest,testReadSerializedAllVersions) EXPECT_EQ(buf.size(), len); close(fd); } - { - CompressionConfig oldCfg(doc.getType().getFieldsType().getCompressionConfig()); - CompressionConfig newCfg(CompressionConfig::LZ4, 9, 95); - const_cast<StructDataType &>(doc.getType().getFieldsType()).setCompressionConfig(newCfg); - nbostream buf = doc.serialize(); - EXPECT_TRUE(buf.size() <= buf.capacity()); - int fd = open(TEST_PATH("data/document-cpp-currentversion-lz4-9.dat").c_str(), - O_WRONLY | O_CREAT | O_TRUNC, 0644); - EXPECT_TRUE(fd > 0); - size_t len = write(fd, buf.peek(), buf.size()); - EXPECT_EQ(buf.size(), len); - close(fd); - const_cast<StructDataType &>(doc.getType().getFieldsType()).setCompressionConfig(oldCfg); - } } std::string jpath = TEST_PATH("../test/serializeddocuments/"); @@ -877,18 +860,6 @@ TEST(DocumentTest, testGenerateSerializedFile) throw vespalib::Exception("write failed"); } close(fd); - - CompressionConfig newCfg(CompressionConfig::LZ4, 9, 95); - const_cast<StructDataType &>(doc.getType().getFieldsType()).setCompressionConfig(newCfg); - - nbostream lz4buf = doc.serialize(); - - fd = open((serializedDir + "/serializecpp-lz4-level9.dat").c_str(), - O_WRONLY | O_TRUNC | O_CREAT, 0644); - if (write(fd, lz4buf.data(), lz4buf.size()) != (ssize_t)lz4buf.size()) { - throw vespalib::Exception("write failed"); - } - close(fd); } TEST(DocumentTest, testBogusserialize) @@ -1013,84 +984,6 @@ TEST(DocumentTest, testSliceSerialize) EXPECT_EQ(*doc2, doc4); } -TEST(DocumentTest, testCompression) -{ - TestDocMan testDocMan; - Document::UP doc = testDocMan.createDocument(); - - std::string bigString("compress me"); - for (int i = 0; i < 8; ++i) { bigString += bigString; } - - doc->setValue("hstringval", StringFieldValue(bigString)); - - nbostream buf_uncompressed = doc->serialize(); - - CompressionConfig oldCfg(doc->getType().getFieldsType().getCompressionConfig()); - - CompressionConfig newCfg(CompressionConfig::LZ4, 9, 95); - const_cast<StructDataType &>(doc->getType().getFieldsType()).setCompressionConfig(newCfg); - nbostream buf_lz4 = doc->serialize(); - - const_cast<StructDataType &>(doc->getType().getFieldsType()).setCompressionConfig(oldCfg); - - EXPECT_TRUE(buf_lz4.size() < buf_uncompressed.size()); - - Document doc_lz4(testDocMan.getTypeRepo(), buf_lz4); - - EXPECT_EQ(*doc, doc_lz4); -} - -TEST(DocumentTest, testCompressionConfigured) -{ - DocumenttypesConfigBuilderHelper builder; - builder.document(43, "serializetest", - Struct("serializetest.header").setId(44), - Struct("serializetest.body").setId(45) - .addField("stringfield", DataType::T_STRING)); - DocumentTypeRepo repo(builder.config()); - Document doc_uncompressed(*repo.getDocumentType("serializetest"), - DocumentId("id:ns:serializetest::1")); - - std::string bigString("compress me"); - for (int i = 0; i < 8; ++i) { bigString += bigString; } - - doc_uncompressed.setValue("stringfield", StringFieldValue(bigString)); - nbostream buf_uncompressed; - doc_uncompressed.serialize(buf_uncompressed); - - size_t uncompressedSize = buf_uncompressed.size(); - - DocumenttypesConfigBuilderHelper builder2; - builder2.document(43, "serializetest", - Struct("serializetest.header").setId(44), - Struct("serializetest.body").setId(45) - .addField("stringfield", DataType::T_STRING) - .setCompression(DocumenttypesConfig::Documenttype:: - Datatype::Sstruct::Compression::Type::LZ4, - 9, 99, 0)); - DocumentTypeRepo repo2(builder2.config()); - - Document doc(repo2, buf_uncompressed); - - nbostream buf_compressed; - doc.serialize(buf_compressed); - size_t compressedSize = buf_compressed.size(); - - EXPECT_TRUE(compressedSize < uncompressedSize); - - Document doc2(repo2, buf_compressed); - - nbostream buf_compressed2; - doc2.serialize(buf_compressed2); - - EXPECT_EQ(compressedSize, buf_compressed2.size()); - - Document doc3(repo2, buf_compressed2); - - EXPECT_EQ(doc2, doc_uncompressed); - EXPECT_EQ(doc2, doc3); -} - TEST(DocumentTest, testUnknownEntries) { // We should be able to deserialize a document with unknown values in it. diff --git a/document/src/tests/repo/documenttyperepo_test.cpp b/document/src/tests/repo/documenttyperepo_test.cpp index d1456fb40c2..6619d53e609 100644 --- a/document/src/tests/repo/documenttyperepo_test.cpp +++ b/document/src/tests/repo/documenttyperepo_test.cpp @@ -17,7 +17,6 @@ #include <vespa/vespalib/testkit/testapp.h> #include <vespa/vespalib/util/exceptions.h> #include <set> -#include <vespa/config/helper/configgetter.h> using config::AsciiConfigWriter; @@ -26,7 +25,6 @@ using std::vector; using vespalib::Identifiable; using vespalib::IllegalArgumentException; using vespalib::string; -using vespalib::compression::CompressionConfig; using namespace document::config_builder; using namespace document; @@ -42,9 +40,6 @@ const int32_t body_id = 31; const string type_name_2 = "test_2"; const string header_name_2 = type_name_2 + ".header"; const string body_name_2 = type_name_2 + ".body"; -const int32_t comp_level = 10; -const int32_t comp_minres = 80; -const size_t comp_minsize = 120; const string field_name = "field_name"; const string derived_name = "derived"; @@ -80,25 +75,6 @@ TEST("requireThatDocumentTypeCanBeLookedUpWhenIdIsNotAHash") { ASSERT_TRUE(type); } -TEST("requireThatStructsCanConfigureCompression") { - DocumenttypesConfigBuilderHelper builder; - typedef DocumenttypesConfig::Documenttype::Datatype::Sstruct Sstruct; - builder.document(doc_type_id, type_name, - Struct(header_name), - Struct(body_name).setCompression( - Sstruct::Compression::Type::LZ4, - comp_level, comp_minres, comp_minsize)); - DocumentTypeRepo repo(builder.config()); - - const CompressionConfig &comp_config = - repo.getDocumentType(type_name)->getFieldsType() - .getCompressionConfig(); - EXPECT_EQUAL(CompressionConfig::LZ4, comp_config.type); - EXPECT_EQUAL(comp_level, comp_config.compressionLevel); - EXPECT_EQUAL(comp_minres, comp_config.threshold); - EXPECT_EQUAL(comp_minsize, comp_config.minSize); -} - TEST("requireThatStructsCanHaveFields") { DocumenttypesConfigBuilderHelper builder; builder.document(doc_type_id, type_name, diff --git a/document/src/tests/serialization/vespadocumentserializer_test.cpp b/document/src/tests/serialization/vespadocumentserializer_test.cpp index 3d71371e155..443c7d1885a 100644 --- a/document/src/tests/serialization/vespadocumentserializer_test.cpp +++ b/document/src/tests/serialization/vespadocumentserializer_test.cpp @@ -44,6 +44,7 @@ #include <vespa/vespalib/objects/nbostream.h> #include <vespa/vespalib/testkit/testapp.h> #include <vespa/document/base/exceptions.h> +#include <vespa/vespalib/util/compressionconfig.h> using document::DocumenttypesConfig; using vespalib::File; @@ -488,21 +489,11 @@ TEST("requireThatUncompressedStructFieldValueCanBeSerialized") { checkStructSerialization(value, CompressionConfig::NONE); } -TEST("requireThatCompressedStructFieldValueCanBeSerialized") { +TEST("requireThatReserializationIsUnompressedIfUnmodified") { StructDataType structType(getStructDataType()); StructFieldValue value = getStructFieldValue(structType); - const_cast<StructDataType *>(static_cast<const StructDataType *>(value.getDataType())) - ->setCompressionConfig(CompressionConfig(CompressionConfig::LZ4, 0, 95)); - checkStructSerialization(value, CompressionConfig::LZ4); -} - -TEST("requireThatReserializationPreservesCompressionIfUnmodified") { - StructDataType structType(getStructDataType()); - StructFieldValue value = getStructFieldValue(structType); - const_cast<StructDataType *>(static_cast<const StructDataType *>(value.getDataType())) - ->setCompressionConfig(CompressionConfig(CompressionConfig::LZ4, 0, 95)); - TEST_DO(checkStructSerialization(value, CompressionConfig::LZ4)); + TEST_DO(checkStructSerialization(value, CompressionConfig::NONE)); nbostream os; VespaDocumentSerializer serializer(os); @@ -512,9 +503,9 @@ TEST("requireThatReserializationPreservesCompressionIfUnmodified") { StructFieldValue value2(struct_type); VespaDocumentDeserializer deserializer(repo, os, serialization_version); deserializer.read(value2); - TEST_DO(checkStructSerialization(value, CompressionConfig::LZ4)); + TEST_DO(checkStructSerialization(value, CompressionConfig::NONE)); // Lazy serialization of structs.... - TEST_DO(checkStructSerialization(value2, CompressionConfig::LZ4)); + TEST_DO(checkStructSerialization(value2, CompressionConfig::NONE)); EXPECT_EQUAL(value, value2); } diff --git a/document/src/vespa/document/datatype/structdatatype.cpp b/document/src/vespa/document/datatype/structdatatype.cpp index bb927e6e872..e4b39099d08 100644 --- a/document/src/vespa/document/datatype/structdatatype.cpp +++ b/document/src/vespa/document/datatype/structdatatype.cpp @@ -22,22 +22,19 @@ IMPLEMENT_IDENTIFIABLE(StructDataType, StructuredDataType); StructDataType::StructDataType() : StructuredDataType(), _nameFieldMap(), - _idFieldMap(), - _compressionConfig() + _idFieldMap() { } StructDataType::StructDataType(vespalib::stringref name) : StructuredDataType(name), _nameFieldMap(), - _idFieldMap(), - _compressionConfig() + _idFieldMap() { } StructDataType::StructDataType(vespalib::stringref name, int32_t dataTypeId) : StructuredDataType(name, dataTypeId), _nameFieldMap(), - _idFieldMap(), - _compressionConfig() + _idFieldMap() { } StructDataType::~StructDataType() = default; @@ -54,11 +51,6 @@ StructDataType::print(std::ostream& out, bool verbose, out << "StructDataType(" << getName(); if (verbose) { out << ", id " << getId(); - if (_compressionConfig.type != CompressionConfig::NONE) { - out << ", Compression(" << _compressionConfig.type << "," - << int(_compressionConfig.compressionLevel) << "," - << int(_compressionConfig.threshold) << ")"; - } } out << ")"; if (verbose) { diff --git a/document/src/vespa/document/datatype/structdatatype.h b/document/src/vespa/document/datatype/structdatatype.h index 624ce3011ff..ace9edfb0ab 100644 --- a/document/src/vespa/document/datatype/structdatatype.h +++ b/document/src/vespa/document/datatype/structdatatype.h @@ -11,7 +11,6 @@ #include <vespa/document/datatype/structureddatatype.h> #include <vespa/vespalib/stllike/hash_map.h> -#include <vespa/vespalib/util/compressionconfig.h> #include <memory> namespace document { @@ -20,7 +19,6 @@ class StructDataType final : public StructuredDataType { public: using UP = std::unique_ptr<StructDataType>; using SP = std::shared_ptr<StructDataType>; - using CompressionConfig = vespalib::compression::CompressionConfig; StructDataType(); StructDataType(vespalib::stringref name); @@ -65,9 +63,6 @@ public: Field::Set getFieldSet() const override; StructDataType* clone() const override; - void setCompressionConfig(const CompressionConfig& cfg) { _compressionConfig = cfg; }; - const CompressionConfig& getCompressionConfig() const { return _compressionConfig; } - DECLARE_IDENTIFIABLE(StructDataType); private: @@ -75,7 +70,6 @@ private: using IntFieldMap = vespalib::hash_map<int32_t, Field::SP>; StringFieldMap _nameFieldMap; IntFieldMap _idFieldMap; - CompressionConfig _compressionConfig; /** @return "" if not conflicting. Error message otherwise. */ vespalib::string containsConflictingField(const Field& field) const; diff --git a/document/src/vespa/document/fieldvalue/serializablearray.cpp b/document/src/vespa/document/fieldvalue/serializablearray.cpp index afb89ba0acf..605e4a698df 100644 --- a/document/src/vespa/document/fieldvalue/serializablearray.cpp +++ b/document/src/vespa/document/fieldvalue/serializablearray.cpp @@ -2,11 +2,9 @@ #include "serializablearray.h" #include <vespa/document/util/serializableexceptions.h> #include <vespa/document/util/bytebuffer.h> -#include <vespa/vespalib/util/compressor.h> #include <vespa/vespalib/stllike/hash_map.hpp> -#include <vespa/vespalib/data/databuffer.h> #include <algorithm> - +#include <cassert> #include <vespa/log/log.h> LOG_SETUP(".document.serializable-array"); @@ -27,22 +25,13 @@ public: } void -SerializableArray::set(EntryMap entries, ByteBuffer buffer, - CompressionConfig::Type comp_type, uint32_t uncompressed_length) +SerializableArray::set(EntryMap entries, ByteBuffer buffer) { _entries = std::move(entries); - if (CompressionConfig::isCompressed(comp_type)) { - _unlikely = std::make_unique<RarelyUsedBuffers>(); - _unlikely->_compSerData = std::move(buffer); - _unlikely->_serializedCompression = comp_type; - _unlikely->_uncompressedLength = uncompressed_length; - _uncompSerData = ByteBuffer(); - } else { - _uncompSerData = std::move(buffer); - _unlikely.reset(); - } + _uncompSerData = std::move(buffer); } +SerializableArray::SerializableArray() = default; SerializableArray::SerializableArray(SerializableArray &&) noexcept = default; SerializableArray& SerializableArray::operator=(SerializableArray &&) noexcept = default; SerializableArray::~SerializableArray() = default; @@ -60,25 +49,9 @@ ensure(std::unique_ptr<T> &owned) { } -SerializableArray::RarelyUsedBuffers::RarelyUsedBuffers() - : _owned(), - _compSerData(nullptr, 0), - _serializedCompression(CompressionConfig::NONE), - _uncompressedLength(0) -{ } -SerializableArray::RarelyUsedBuffers::~RarelyUsedBuffers() = default; - -SerializableArray::RarelyUsedBuffers::RarelyUsedBuffers(const RarelyUsedBuffers & rhs) - : _owned(), - _compSerData(rhs._compSerData), - _serializedCompression(rhs._serializedCompression), - _uncompressedLength(rhs._uncompressedLength) -{ } - SerializableArray::SerializableArray(const SerializableArray& rhs) : _entries(rhs._entries), - _uncompSerData(rhs._uncompSerData), - _unlikely(rhs._unlikely ? new RarelyUsedBuffers(*rhs._unlikely) : nullptr) + _uncompSerData(rhs._uncompSerData) { for (size_t i(0); i < _entries.size(); i++) { Entry & e(_entries[i]); @@ -86,7 +59,7 @@ SerializableArray::SerializableArray(const SerializableArray& rhs) // Pointing to a buffer in the _owned structure. ByteBuffer buf(ByteBuffer::copyBuffer(e.getBuffer(&_uncompSerData), e.size())); e.setBuffer(buf.getBuffer()); - ensure(_unlikely->_owned)[e.id()] = std::move(buf); + ensure(_owned)[e.id()] = std::move(buf); } else { // If not it is relative to the buffer _uncompSerData, and hence it is valid as is. } @@ -106,31 +79,20 @@ void SerializableArray::clear() { _entries.clear(); _uncompSerData = ByteBuffer(nullptr, 0); - _unlikely.reset(); -} - -void -SerializableArray::invalidate() -{ - if (_unlikely) { - _unlikely->_compSerData = ByteBuffer(nullptr, 0);; - } } void SerializableArray::set(int id, ByteBuffer buffer) { - maybeDecompress(); Entry e(id, buffer.getRemaining(), buffer.getBuffer()); assert(buffer.getRemaining() < 0x80000000ul); - ensure(ensure(_unlikely)._owned)[id] = std::move(buffer); + ensure(_owned)[id] = std::move(buffer); auto it = find(id); if (it == _entries.end()) { _entries.push_back(e); } else { *it = e; } - invalidate(); } void SerializableArray::set(int id, const char* value, int len) @@ -160,90 +122,25 @@ vespalib::ConstBufferRef SerializableArray::get(int id) const { vespalib::ConstBufferRef buf; - if ( !maybeDecompressAndCatch() ) { - auto found = find(id); + auto found = find(id); - if (found != _entries.end()) { - const Entry& entry = *found; - buf = vespalib::ConstBufferRef(entry.getBuffer(&_uncompSerData), entry.size()); - } - } else { - // should we clear all or what? + if (found != _entries.end()) { + const Entry& entry = *found; + buf = vespalib::ConstBufferRef(entry.getBuffer(&_uncompSerData), entry.size()); } return buf; } -bool -SerializableArray::deCompressAndCatch() const -{ - try { - const_cast<SerializableArray *>(this)->deCompress(); - return false; - } catch (const std::exception & e) { - LOG(warning, "Deserializing compressed content failed: %s", e.what()); - return true; - } -} - void SerializableArray::clear(int id) { - maybeDecompress(); auto it = find(id); if (it != _entries.end()) { _entries.erase(it); - if (_unlikely && _unlikely->_owned) { - _unlikely->_owned->erase(id); - } - invalidate(); } } -void -SerializableArray::deCompress() // throw (DeserializeException) -{ - using vespalib::compression::decompress; - // will only do this once - - assert(_unlikely && (_unlikely->_compSerData.getRemaining() != 0)); - assert(_uncompSerData.getRemaining() == 0); - assert(CompressionConfig::isCompressed(_unlikely->_serializedCompression)); - uint32_t uncompressedLength = _unlikely->_uncompressedLength; - - ByteBuffer newSerialization(vespalib::alloc::Alloc::alloc(uncompressedLength), uncompressedLength); - vespalib::DataBuffer unCompressed(newSerialization.getBuffer(), newSerialization.getLength()); - unCompressed.clear(); - try { - decompress(_unlikely->_serializedCompression, - uncompressedLength, - vespalib::ConstBufferRef(_unlikely->_compSerData.getBufferAtPos(), _unlikely->_compSerData.getRemaining()), - unCompressed, - false); - } catch (const std::runtime_error & e) { - throw DeserializeException( - make_string( "Document was compressed with code unknown code %d", _unlikely->_serializedCompression), - VESPA_STRLOC); - } - - if (unCompressed.getDataLen() != (size_t)uncompressedLength) { - throw DeserializeException( - make_string("Did not decompress to the expected length: had %u, wanted %d, got %zu", - _unlikely->_compSerData.getRemaining(), uncompressedLength, unCompressed.getDataLen()), - VESPA_STRLOC); - } - assert(newSerialization.getBuffer() == unCompressed.getData()); - _uncompSerData = std::move(newSerialization); - LOG_ASSERT(uncompressedLength == _uncompSerData.getRemaining()); -} - -vespalib::compression::CompressionInfo -SerializableArray::getCompressionInfo() const { - return _unlikely - ? CompressionInfo(_unlikely->_uncompressedLength, _unlikely->_compSerData.getRemaining()) - : CompressionInfo(_uncompSerData.getRemaining(), CompressionConfig::NONE); -} - const char * SerializableArray::Entry::getBuffer(const ByteBuffer * readOnlyBuffer) const { return hasBuffer() ? _data._buffer : readOnlyBuffer->getBuffer() + getOffset(); diff --git a/document/src/vespa/document/fieldvalue/serializablearray.h b/document/src/vespa/document/fieldvalue/serializablearray.h index 126f7072d6a..3fef1d21c3c 100644 --- a/document/src/vespa/document/fieldvalue/serializablearray.h +++ b/document/src/vespa/document/fieldvalue/serializablearray.h @@ -16,7 +16,6 @@ #pragma once -#include <vespa/vespalib/util/compressionconfig.h> #include <vespa/vespalib/util/buffer.h> #include <vespa/vespalib/util/memory.h> #include <vespa/document/util/bytebuffer.h> @@ -80,19 +79,15 @@ public: using CP = vespalib::CloneablePtr<SerializableArray>; using UP = std::unique_ptr<SerializableArray>; - using ByteBufferUP = std::unique_ptr<ByteBuffer>; - using CompressionConfig = vespalib::compression::CompressionConfig; - using CompressionInfo = vespalib::compression::CompressionInfo; - SerializableArray() = default; + SerializableArray(); SerializableArray(const SerializableArray&); SerializableArray& operator=(const SerializableArray&); SerializableArray(SerializableArray &&) noexcept; SerializableArray& operator=(SerializableArray &&) noexcept; ~SerializableArray(); - void set(EntryMap entries, ByteBuffer buffer, - CompressionConfig::Type comp_type, uint32_t uncompressed_length); + void set(EntryMap entries, ByteBuffer buffer); /** * Stores a value in the array. * @@ -129,57 +124,20 @@ public: /** Deletes all stored attributes. */ void clear(); - CompressionConfig::Type getCompression() const { - return _unlikely ? _unlikely->_serializedCompression : CompressionConfig::NONE; - } - CompressionInfo getCompressionInfo() const; - bool empty() const { return _entries.empty(); } const ByteBuffer* getSerializedBuffer() const { - return CompressionConfig::isCompressed(getCompression()) - ? &_unlikely->_compSerData - : &_uncompSerData; + return &_uncompSerData; } const EntryMap & getEntries() const { return _entries; } private: - bool shouldDecompress() const { - return _unlikely && (_unlikely->_compSerData.getRemaining() != 0) && (_uncompSerData.getBuffer() == 0); - } - bool maybeDecompressAndCatch() const { - if ( shouldDecompress() ) { - return deCompressAndCatch(); - } - return false; - } - - bool deCompressAndCatch() const; - void maybeDecompress() const { - if ( shouldDecompress() ) { - const_cast<SerializableArray *>(this)->deCompress(); - } - } - void deCompress(); // throw (DeserializeException); - - struct RarelyUsedBuffers { - /** The buffers we own. */ - RarelyUsedBuffers(); - RarelyUsedBuffers(const RarelyUsedBuffers &); - ~RarelyUsedBuffers(); - std::unique_ptr<serializablearray::BufferMap> _owned; - ByteBuffer _compSerData; - CompressionConfig::Type _serializedCompression; - uint32_t _uncompressedLength; - }; /** Contains the stored attributes, with reference to the real data.. */ EntryMap _entries; /** Data we deserialized from, if applicable. */ ByteBuffer _uncompSerData; - std::unique_ptr<RarelyUsedBuffers> _unlikely; - + std::unique_ptr<serializablearray::BufferMap> _owned; - VESPA_DLL_LOCAL void invalidate(); VESPA_DLL_LOCAL EntryMap::const_iterator find(int id) const; VESPA_DLL_LOCAL EntryMap::iterator find(int id); }; diff --git a/document/src/vespa/document/fieldvalue/structfieldvalue.cpp b/document/src/vespa/document/fieldvalue/structfieldvalue.cpp index fc87fbe3a59..555964d8b34 100644 --- a/document/src/vespa/document/fieldvalue/structfieldvalue.cpp +++ b/document/src/vespa/document/fieldvalue/structfieldvalue.cpp @@ -23,7 +23,6 @@ using std::vector; using vespalib::nbostream; using vespalib::nbostream_longlivedbuf; using vespalib::make_string; -using vespalib::compression::CompressionConfig; using namespace vespalib::xml; namespace document { @@ -50,24 +49,14 @@ StructFieldValue::getStructType() const { return static_cast<const StructDataType &>(getType()); } -const CompressionConfig & -StructFieldValue::getCompressionConfig() const { - return getStructType().getCompressionConfig(); -} - void -StructFieldValue::lazyDeserialize(const FixedTypeRepo &repo, - uint16_t version, - SerializableArray::EntryMap && fm, - ByteBuffer buffer, - CompressionConfig::Type comp_type, - int32_t uncompressed_length) +StructFieldValue::lazyDeserialize(const FixedTypeRepo &repo, uint16_t version, SerializableArray::EntryMap && fm, ByteBuffer buffer) { _repo = &repo.getDocumentTypeRepo(); _doc_type = &repo.getDocumentType(); _version = version; - _fields.set(std::move(fm), std::move(buffer), comp_type, uncompressed_length); + _fields.set(std::move(fm), std::move(buffer)); _hasChanged = false; } diff --git a/document/src/vespa/document/fieldvalue/structfieldvalue.h b/document/src/vespa/document/fieldvalue/structfieldvalue.h index cd8bd0fea0f..ab35dc04421 100644 --- a/document/src/vespa/document/fieldvalue/structfieldvalue.h +++ b/document/src/vespa/document/fieldvalue/structfieldvalue.h @@ -34,7 +34,6 @@ private: public: using UP = std::unique_ptr<StructFieldValue>; - using CompressionConfig = vespalib::compression::CompressionConfig; StructFieldValue(const DataType &type); StructFieldValue(const StructFieldValue & rhs); @@ -48,12 +47,7 @@ public: void setDocumentType(const DocumentType & docType) { _doc_type = & docType; } const SerializableArray & getFields() const { return _fields; } - void lazyDeserialize(const FixedTypeRepo &repo, - uint16_t version, - SerializableArray::EntryMap && fields, - ByteBuffer buffer, - CompressionConfig::Type comp_type, - int32_t uncompressed_length); + void lazyDeserialize(const FixedTypeRepo &repo, uint16_t version, SerializableArray::EntryMap && fields, ByteBuffer buffer); // returns false if the field could not be serialized. bool serializeField(int raw_field_id, uint16_t version, FieldValueWriter &writer) const; @@ -70,8 +64,6 @@ public: const Field& getField(vespalib::stringref name) const override; void clear() override; - const CompressionConfig &getCompressionConfig() const; - // FieldValue implementation. FieldValue& assign(const FieldValue&) override; int compare(const FieldValue& other) const override; diff --git a/document/src/vespa/document/repo/documenttyperepo.cpp b/document/src/vespa/document/repo/documenttyperepo.cpp index b993f50d7b6..15730d14a86 100644 --- a/document/src/vespa/document/repo/documenttyperepo.cpp +++ b/document/src/vespa/document/repo/documenttyperepo.cpp @@ -31,7 +31,6 @@ using vespalib::hash_map; using vespalib::make_string; using vespalib::string; using vespalib::stringref; -using vespalib::compression::CompressionConfig; namespace document { @@ -304,14 +303,6 @@ void addStruct(int32_t id, const Datatype::Sstruct &s, Repo &repo) { } } - CompressionConfig::Type type = CompressionConfig::NONE; - if (s.compression.type == Datatype::Sstruct::Compression::Type::LZ4) { - type = CompressionConfig::LZ4; - } - - struct_type->setCompressionConfig( - CompressionConfig(type, s.compression.level, s.compression.threshold, s.compression.minsize)); - for (size_t i = 0; i < s.field.size(); ++i) { addField(s.field[i], repo, *struct_type); } diff --git a/document/src/vespa/document/serialization/vespadocumentdeserializer.cpp b/document/src/vespa/document/serialization/vespadocumentdeserializer.cpp index 6dd6a4c21bd..747dfbbcee6 100644 --- a/document/src/vespa/document/serialization/vespadocumentdeserializer.cpp +++ b/document/src/vespa/document/serialization/vespadocumentdeserializer.cpp @@ -22,6 +22,8 @@ #include <vespa/vespalib/data/slime/slime.h> #include <vespa/vespalib/stllike/asciistream.h> #include <vespa/vespalib/util/backtrace.h> +#include <vespa/vespalib/util/compressor.h> +#include <vespa/vespalib/data/databuffer.h> #include <vespa/eval/eval/fast_value.h> #include <vespa/eval/eval/value_codec.h> #include <vespa/eval/eval/value.h> @@ -42,6 +44,8 @@ using vespalib::nbostream; using vespalib::Memory; using vespalib::stringref; using vespalib::compression::CompressionConfig; +using vespalib::ConstBufferRef; +using vespalib::make_string_short::fmt; using vespalib::eval::FastValueBuilderFactory; namespace document { @@ -67,7 +71,8 @@ getChunkCount(uint8_t contentCode) } // namespace -void VespaDocumentDeserializer::readDocument(Document &value) { +void +VespaDocumentDeserializer::readDocument(Document &value) { read(value.getId()); uint8_t content_code = readValue<uint8_t>(_stream); @@ -90,7 +95,8 @@ void VespaDocumentDeserializer::readDocument(Document &value) { } } -void VespaDocumentDeserializer::read(FieldValue &value) { +void +VespaDocumentDeserializer::read(FieldValue &value) { value.accept(*this); } @@ -113,20 +119,23 @@ VespaDocumentDeserializer::readDocType(const DocumentType &guess) return 0; } -void VespaDocumentDeserializer::read(DocumentId &value) { +void +VespaDocumentDeserializer::read(DocumentId &value) { stringref s(_stream.peek()); value.set(s); _stream.adjustReadPos(s.size() + 1); } -void VespaDocumentDeserializer::read(DocumentType &value) { +void +VespaDocumentDeserializer::read(DocumentType &value) { const DocumentType *doc_type = readDocType(value); if (doc_type) { value = *doc_type; } } -void VespaDocumentDeserializer::read(Document &value) { +void +VespaDocumentDeserializer::read(Document &value) { uint16_t version = readValue<uint16_t>(_stream); VarScope<uint16_t> version_scope(_version, version); @@ -150,11 +159,13 @@ void VespaDocumentDeserializer::read(Document &value) { } -void VespaDocumentDeserializer::read(AnnotationReferenceFieldValue &value) { +void +VespaDocumentDeserializer::read(AnnotationReferenceFieldValue &value) { value.setAnnotationIndex(getInt1_2_4Bytes(_stream)); } -void VespaDocumentDeserializer::read(ArrayFieldValue &value) { +void +VespaDocumentDeserializer::read(ArrayFieldValue &value) { uint32_t size = readSize(_stream); value.clear(); value.resize(size); @@ -163,7 +174,8 @@ void VespaDocumentDeserializer::read(ArrayFieldValue &value) { } } -void VespaDocumentDeserializer::read(MapFieldValue &value) { +void +VespaDocumentDeserializer::read(MapFieldValue &value) { value.clear(); uint32_t size = readSize(_stream); value.resize(size); @@ -190,31 +202,38 @@ void readFieldValue(nbostream &input, T &value) { } // namespace -void VespaDocumentDeserializer::read(BoolFieldValue &value) { +void +VespaDocumentDeserializer::read(BoolFieldValue &value) { readFieldValue(_stream, value); } -void VespaDocumentDeserializer::read(ByteFieldValue &value) { +void +VespaDocumentDeserializer::read(ByteFieldValue &value) { readFieldValue(_stream, value); } -void VespaDocumentDeserializer::read(DoubleFieldValue &value) { +void +VespaDocumentDeserializer::read(DoubleFieldValue &value) { readFieldValue(_stream, value); } -void VespaDocumentDeserializer::read(FloatFieldValue &value) { +void +VespaDocumentDeserializer::read(FloatFieldValue &value) { readFieldValue(_stream, value); } -void VespaDocumentDeserializer::read(IntFieldValue &value) { +void +VespaDocumentDeserializer::read(IntFieldValue &value) { readFieldValue(_stream, value); } -void VespaDocumentDeserializer::read(LongFieldValue &value) { +void +VespaDocumentDeserializer::read(LongFieldValue &value) { readFieldValue(_stream, value); } -void VespaDocumentDeserializer::read(PredicateFieldValue &value) { +void +VespaDocumentDeserializer::read(PredicateFieldValue &value) { uint32_t stored_size = readValue<uint32_t>(_stream); Memory memory(_stream.peek(), _stream.size()); std::unique_ptr<Slime> slime(new Slime); @@ -238,18 +257,21 @@ void setValue(FV &field_value, stringref val, bool use_ref) { } } // namespace -void VespaDocumentDeserializer::read(RawFieldValue &value) { +void +VespaDocumentDeserializer::read(RawFieldValue &value) { uint32_t size = readValue<uint32_t>(_stream); stringref val(_stream.peek(), size); setValue(value, val, _stream.isLongLivedBuffer()); _stream.adjustReadPos(size); } -void VespaDocumentDeserializer::read(ShortFieldValue &value) { +void +VespaDocumentDeserializer::read(ShortFieldValue &value) { readFieldValue(_stream, value); } -void VespaDocumentDeserializer::read(StringFieldValue &value) { +void +VespaDocumentDeserializer::read(StringFieldValue &value) { uint8_t coding = readValue<uint8_t>(_stream); size_t size = getInt1_4Bytes(_stream); if (size == 0) { @@ -274,7 +296,8 @@ typedef SerializableArray::EntryMap FieldInfo; void readFieldInfo(nbostream& input, SerializableArray::EntryMap & field_info) __attribute__((noinline)); -void readFieldInfo(nbostream& input, SerializableArray::EntryMap & field_info) { +void +readFieldInfo(nbostream& input, SerializableArray::EntryMap & field_info) { size_t field_count = getInt1_4Bytes(input); field_info.reserve(field_count); uint32_t offset = 0; @@ -285,9 +308,40 @@ void readFieldInfo(nbostream& input, SerializableArray::EntryMap & field_info) { offset += size; } } + +ByteBuffer +deCompress(CompressionConfig::Type compression, uint32_t uncompressedLength, vespalib::ConstBufferRef compressed) __attribute__((noinline)); + +ByteBuffer +deCompress(CompressionConfig::Type compression, uint32_t uncompressedLength, vespalib::ConstBufferRef compressed) +{ + using vespalib::compression::decompress; + + assert(compressed.size() != 0); + + ByteBuffer newSerialization(vespalib::alloc::Alloc::alloc(uncompressedLength), uncompressedLength); + vespalib::DataBuffer unCompressed(newSerialization.getBuffer(), newSerialization.getLength()); + unCompressed.clear(); + try { + decompress(compression, uncompressedLength, compressed,unCompressed,false); + } catch (const std::runtime_error & e) { + throw DeserializeException(fmt( "Document was compressed with code unknown code %d", compression), VESPA_STRLOC); + } + + if (unCompressed.getDataLen() != (size_t)uncompressedLength) { + throw DeserializeException(fmt("Did not decompress to the expected length: had %lu, wanted %d, got %zu", + compressed.size(), uncompressedLength, unCompressed.getDataLen()), + VESPA_STRLOC); + } + assert(newSerialization.getBuffer() == unCompressed.getData()); + LOG_ASSERT(uncompressedLength == newSerialization.getRemaining()); + return newSerialization; +} + } // namespace -void VespaDocumentDeserializer::readStructNoReset(StructFieldValue &value) { +void +VespaDocumentDeserializer::readStructNoReset(StructFieldValue &value) { size_t data_size = readValue<uint32_t>(_stream); CompressionConfig::Type compression_type = CompressionConfig::Type(readValue<uint8_t>(_stream)); @@ -308,19 +362,18 @@ void VespaDocumentDeserializer::readStructNoReset(StructFieldValue &value) { } if (data_size > 0) { - ByteBuffer buffer(_stream.isLongLivedBuffer() - ? ByteBuffer(_stream.peek(), data_size) - : ByteBuffer::copyBuffer(_stream.peek(), data_size)); + ByteBuffer buffer = CompressionConfig::isCompressed(compression_type) + ? deCompress(compression_type, uncompressed_size, ConstBufferRef(_stream.peek(), data_size)) + : (_stream.isLongLivedBuffer() + ? ByteBuffer(_stream.peek(), data_size) + : ByteBuffer::copyBuffer(_stream.peek(), data_size)); if (value.getFields().empty()) { - LOG(spam, "Lazy deserializing into %s with _version %u", - value.getDataType()->getName().c_str(), _version); - value.lazyDeserialize(_repo, _version, std::move(field_info), - std::move(buffer), compression_type, uncompressed_size); + LOG(spam, "Lazy deserializing into %s with _version %u", value.getDataType()->getName().c_str(), _version); + value.lazyDeserialize(_repo, _version, std::move(field_info), std::move(buffer)); } else { LOG(debug, "Legacy dual header/body format. -> Merging."); StructFieldValue tmp(*value.getDataType()); - tmp.lazyDeserialize(_repo, _version, std::move(field_info), - std::move(buffer), compression_type, uncompressed_size); + tmp.lazyDeserialize(_repo, _version, std::move(field_info), std::move(buffer)); for (const auto & entry : tmp) { try { FieldValue::UP decoded = tmp.getValue(entry); @@ -342,7 +395,8 @@ VespaDocumentDeserializer::read(StructFieldValue& value) readStructNoReset(value); } -void VespaDocumentDeserializer::read(WeightedSetFieldValue &value) { +void +VespaDocumentDeserializer::read(WeightedSetFieldValue &value) { value.clear(); readValue<uint32_t>(_stream); // skip type id uint32_t size = readValue<uint32_t>(_stream); @@ -367,7 +421,7 @@ VespaDocumentDeserializer::readTensor() { size_t length = _stream.getInt1_4Bytes(); if (length > _stream.size()) { - throw DeserializeException(vespalib::make_string("Stream failed size(%zu), needed(%zu) to deserialize tensor field value", _stream.size(), length), + throw DeserializeException(fmt("Stream failed size(%zu), needed(%zu) to deserialize tensor field value", _stream.size(), length), VESPA_STRLOC); } std::unique_ptr<vespalib::eval::Value> tensor; @@ -386,7 +440,8 @@ VespaDocumentDeserializer::readTensor() return tensor; } -void VespaDocumentDeserializer::read(ReferenceFieldValue& value) { +void +VespaDocumentDeserializer::read(ReferenceFieldValue& value) { const bool hasId(readValue<uint8_t>(_stream) == 1); if (hasId) { DocumentId id; diff --git a/document/src/vespa/document/serialization/vespadocumentserializer.cpp b/document/src/vespa/document/serialization/vespadocumentserializer.cpp index f68a2202788..b6a7dabd87d 100644 --- a/document/src/vespa/document/serialization/vespadocumentserializer.cpp +++ b/document/src/vespa/document/serialization/vespadocumentserializer.cpp @@ -26,7 +26,6 @@ #include <vespa/document/update/fieldpathupdates.h> #include <vespa/document/update/updates.h> #include <vespa/document/util/bytebuffer.h> -#include <vespa/eval/eval/value.h> #include <vespa/eval/eval/value_codec.h> #include <vespa/vespalib/data/databuffer.h> #include <vespa/vespalib/data/slime/binary_format.h> @@ -236,36 +235,6 @@ serializeFields(const StructFieldValue &value, nbostream &stream, } } -bool compressionSufficient(const CompressionConfig &config, uint64_t old_size, size_t new_size) -{ - return (new_size + 8) < (old_size * config.threshold / 100); -} - -bool bigEnough(size_t size, const CompressionConfig &config) -{ - return (size >= config.minSize); -} - -vespalib::ConstBufferRef -compressStream(const CompressionConfig &config, nbostream &stream, vespalib::DataBuffer & compressed_data) -{ - using vespalib::compression::compress; - vespalib::ConstBufferRef buf(stream.data(), stream.size()); - if (config.useCompression() && bigEnough(stream.size(), config)) { - CompressionConfig::Type compressedType = compress(config, - vespalib::ConstBufferRef(stream.data(), stream.size()), - compressed_data, false); - if (compressedType != config.type || - ! compressionSufficient(config, stream.size(), compressed_data.getDataLen())) - { - compressed_data.clear(); - } else { - buf = vespalib::ConstBufferRef(compressed_data.getData(), compressed_data.getDataLen()); - } - } - return buf; -} - void putFieldInfo(nbostream &output, const vector<pair<uint32_t, uint32_t> > &field_info) { putInt1_4Bytes(output, field_info.size()); @@ -294,15 +263,11 @@ VespaDocumentSerializer::structNeedsReserialization(const StructFieldValue &valu return true; } - if (value.getCompressionConfig().type == CompressionConfig::NONE) { - return false; - } - - return (value.getFields().getCompression() != value.getCompressionConfig().type && - value.getFields().getCompression() != CompressionConfig::UNCOMPRESSABLE); + return false; } -void VespaDocumentSerializer::writeUnchanged(const SerializableArray &value) { +void +VespaDocumentSerializer::writeUnchanged(const SerializableArray &value) { vector<pair<uint32_t, uint32_t> > field_info; const std::vector<SerializableArray::Entry>& entries = value.getEntries(); @@ -316,38 +281,24 @@ void VespaDocumentSerializer::writeUnchanged(const SerializableArray &value) { size_t estimatedRequiredSpace = sz + 4 + 1 + 8 + 4 + field_info.size()*12; _stream.reserve(_stream.size() + estimatedRequiredSpace); _stream << sz; - _stream << static_cast<uint8_t>(value.getCompression()); - if (CompressionConfig::isCompressed(value.getCompression())) { - putInt2_4_8Bytes(_stream, value.getCompressionInfo().getUncompressedSize()); - } + _stream << static_cast<uint8_t>(CompressionConfig::NONE); putFieldInfo(_stream, field_info); if (sz) { _stream.write(buffer->getBuffer(), buffer->getLength()); } } -void VespaDocumentSerializer::write(const StructFieldValue &value, const FieldSet& fieldSet) +void +VespaDocumentSerializer::write(const StructFieldValue &value, const FieldSet& fieldSet) { nbostream value_stream; vector<pair<uint32_t, uint32_t> > field_info; serializeFields(value, value_stream, field_info, fieldSet); - const CompressionConfig &comp_config = value.getCompressionConfig(); - vespalib::DataBuffer compressed_data; - vespalib::ConstBufferRef toSerialize = compressStream(comp_config, value_stream, compressed_data); - - uint8_t comp_type = (compressed_data.getDataLen() == 0) - ? (comp_config.type == CompressionConfig::NONE - ? CompressionConfig::NONE - : CompressionConfig::UNCOMPRESSABLE) - : comp_config.type; - _stream << static_cast<uint32_t>(toSerialize.size()); - _stream << comp_type; - if (compressed_data.getDataLen() != 0) { - putInt2_4_8Bytes(_stream, value_stream.size()); - } + _stream << static_cast<uint32_t>(value_stream.size()); + _stream << static_cast<uint8_t>(CompressionConfig::NONE); putFieldInfo(_stream, field_info); - _stream.write(toSerialize.c_str(), toSerialize.size()); + _stream.write(value_stream.data(), value_stream.size()); } void diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusDocumentAccess.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusDocumentAccess.java index f703b9ad59a..de09049b49e 100644 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusDocumentAccess.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusDocumentAccess.java @@ -64,8 +64,9 @@ public class MessageBusDocumentAccess extends DocumentAccess { bus = new NetworkMessageBus(network, new MessageBus(network, mbusParams)); } else { - if (params.getRPCNetworkParams().getSlobroksConfig() != null && mbusParams.getMessageBusConfig() != null) + if (mbusParams.getMessageBusConfig() != null) { bus = new RPCMessageBus(mbusParams, params.getRPCNetworkParams()); + } else { log.log(Level.FINE, () -> "Setting up self-subscription to config because explicit config was missing; try to avoid this in containers"); bus = new RPCMessageBus(mbusParams, params.getRPCNetworkParams(), params.getRoutingConfigId()); diff --git a/documentgen-test/src/test/java/com/yahoo/vespa/config/DocumentGenPluginTest.java b/documentgen-test/src/test/java/com/yahoo/vespa/config/DocumentGenPluginTest.java index c0a480f548c..0bc2b53e121 100644 --- a/documentgen-test/src/test/java/com/yahoo/vespa/config/DocumentGenPluginTest.java +++ b/documentgen-test/src/test/java/com/yahoo/vespa/config/DocumentGenPluginTest.java @@ -863,7 +863,6 @@ public class DocumentGenPluginTest { public void testSerialization() { final Book book = getBook(); assertEquals(book.getMystruct().getD1(), (Double)56.777); - assertEquals(book.getMystruct().getCompressionType(), CompressionType.NONE); assertEquals(book.getFieldCount(), 13); assertEquals(book.getMystruct().getFieldCount(), 4); assertEquals(book.getContent().get(0), 3); diff --git a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReceiver.java b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReceiver.java index fbc17293e8d..89a77599909 100644 --- a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReceiver.java +++ b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReceiver.java @@ -187,7 +187,7 @@ public class FileReceiver { private static void moveFileToDestination(File tempFile, File destination) { try { Files.move(tempFile.toPath(), destination.toPath()); - log.log(Level.FINE, () -> "File moved from " + tempFile.getAbsolutePath()+ " to " + destination.getAbsolutePath()); + log.log(Level.FINEST, () -> "File moved from " + tempFile.getAbsolutePath()+ " to " + destination.getAbsolutePath()); } catch (FileAlreadyExistsException e) { // Don't fail if it already exists (we might get the file from several config servers when retrying, servers are down etc. // so it might be written already). Delete temp file/dir in that case, to avoid filling the disk. @@ -239,7 +239,7 @@ public class FileReceiver { } private void receiveFilePart(Request req) { - log.log(Level.FINE, () -> "Received method call '" + req.methodName() + "' with parameters : " + req.parameters()); + log.log(Level.FINEST, () -> "Received method call '" + req.methodName() + "' with parameters : " + req.parameters()); FileReference reference = new FileReference(req.parameters().get(0).asString()); int sessionId = req.parameters().get(1).asInt32(); diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index d72582287a9..ed12529f91e 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -295,7 +295,7 @@ public class Flags { public static final UnboundBooleanFlag DELETE_UNMAINTAINED_CERTIFICATES = defineFeatureFlag( "delete-unmaintained-certificates", false, - List.of("andreer"), "2021-09-23", "2021-11-11", + List.of("andreer"), "2021-09-23", "2021-12-11", "Whether to delete certificates that are known by provider but not by controller", "Takes effect on next run of EndpointCertificateMaintainer" ); diff --git a/hosted-tenant-base/pom.xml b/hosted-tenant-base/pom.xml index 739b8e60a9f..66b8cb56443 100644 --- a/hosted-tenant-base/pom.xml +++ b/hosted-tenant-base/pom.xml @@ -130,6 +130,10 @@ <artifactId>maven-surefire-plugin</artifactId> <version>${maven-surefire-plugin.version}</version> <configuration> + <!-- Illegal reflective access by LogFileHandler via com.yahoo.io.NativeIO --> + <argLine> + --add-opens=java.base/java.io=ALL-UNNAMED + </argLine> <groups>${test.categories}</groups> <redirectTestOutputToFile>false</redirectTestOutputToFile> <trimStackTrace>false</trimStackTrace> @@ -353,8 +357,10 @@ <artifactId>maven-compiler-plugin</artifactId> <version>${maven-compiler-plugin.version}</version> <configuration> - <source>${target_jdk_version}</source> - <target>${target_jdk_version}</target> + <jdkToolchain> + <version>${target_jdk_version}</version> + </jdkToolchain> + <release>${target_jdk_version}</release> <showWarnings>true</showWarnings> <showDeprecation>true</showDeprecation> <compilerArgs> diff --git a/hosted-zone-api/abi-spec.json b/hosted-zone-api/abi-spec.json index 6a6da57a2a1..b1b8eb84705 100644 --- a/hosted-zone-api/abi-spec.json +++ b/hosted-zone-api/abi-spec.json @@ -1,4 +1,21 @@ { + "ai.vespa.cloud.ApplicationId": { + "superClass": "java.lang.Object", + "interfaces": [], + "attributes": [ + "public" + ], + "methods": [ + "public void <init>(java.lang.String, java.lang.String, java.lang.String)", + "public java.lang.String tenant()", + "public java.lang.String application()", + "public java.lang.String instance()", + "public boolean equals(java.lang.Object)", + "public int hashCode()", + "public java.lang.String toString()" + ], + "fields": [] + }, "ai.vespa.cloud.Cluster": { "superClass": "java.lang.Object", "interfaces": [], @@ -55,7 +72,9 @@ "public" ], "methods": [ + "public void <init>(ai.vespa.cloud.ApplicationId, ai.vespa.cloud.Zone, ai.vespa.cloud.Cluster, ai.vespa.cloud.Node)", "public void <init>(ai.vespa.cloud.Zone, ai.vespa.cloud.Cluster, ai.vespa.cloud.Node)", + "public ai.vespa.cloud.ApplicationId application()", "public ai.vespa.cloud.Zone zone()", "public ai.vespa.cloud.Cluster cluster()", "public ai.vespa.cloud.Node node()" diff --git a/hosted-zone-api/src/main/java/ai/vespa/cloud/ApplicationId.java b/hosted-zone-api/src/main/java/ai/vespa/cloud/ApplicationId.java new file mode 100644 index 00000000000..46780d17a13 --- /dev/null +++ b/hosted-zone-api/src/main/java/ai/vespa/cloud/ApplicationId.java @@ -0,0 +1,46 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package ai.vespa.cloud; + +import java.util.Objects; + +/** + * The application id this is running as. + * This is a combination of a tenant, application, and instance name. + * + * @author bratseth + */ +public class ApplicationId { + + private final String tenant; + private final String application; + private final String instance; + + public ApplicationId(String tenant, String application, String instance) { + this.tenant = tenant; + this.application = application; + this.instance = instance; + } + + public String tenant() { return tenant; } + public String application() { return application; } + public String instance() { return instance; } + + @Override + public boolean equals(Object o) { + if (o == this) return true; + if ( ! (o instanceof ApplicationId)) return false; + ApplicationId other = (ApplicationId)o; + if ( ! other.tenant.equals(this.tenant)) return false; + if ( ! other.application.equals(this.application)) return false; + if ( ! other.instance.equals(this.instance)) return false; + return true; + } + + @Override + public int hashCode() { return Objects.hash(tenant, application, instance); } + + /** Returns the string tenant.application.instance */ + @Override + public String toString() { return tenant + "." + application + "." + instance; } + +} diff --git a/hosted-zone-api/src/main/java/ai/vespa/cloud/SystemInfo.java b/hosted-zone-api/src/main/java/ai/vespa/cloud/SystemInfo.java index 15b06d40948..3789c49fe82 100644 --- a/hosted-zone-api/src/main/java/ai/vespa/cloud/SystemInfo.java +++ b/hosted-zone-api/src/main/java/ai/vespa/cloud/SystemInfo.java @@ -11,19 +11,27 @@ import java.util.Objects; */ public class SystemInfo { + private final ApplicationId application; private final Zone zone; private final Cluster cluster; private final Node node; + public SystemInfo(ApplicationId application, Zone zone, Cluster cluster, Node node) { + this.application = Objects.requireNonNull(application, "Application cannot be null"); + this.zone = Objects.requireNonNull(zone, "Zone cannot be null"); + this.cluster = Objects.requireNonNull(cluster, "Cluster cannot be null"); + this.node = Objects.requireNonNull(node, "Node cannot be null"); + } + + /** @deprecated pass an application id */ + @Deprecated // Remove on Vespa 8 public SystemInfo(Zone zone, Cluster cluster, Node node) { - Objects.requireNonNull(zone, "Zone cannot be null!"); - Objects.requireNonNull(cluster, "Cluster cannot be null!"); - Objects.requireNonNull(node, "Node cannot be null!"); - this.zone = zone; - this.cluster = cluster; - this.node = node; + this(new ApplicationId("default", "default", "default"), zone, cluster, node); } + /** Returns the application this is running as a part of */ + public ApplicationId application() { return application; } + /** Returns the zone this is running in */ public Zone zone() { return zone; } diff --git a/hosted-zone-api/src/test/java/ai/vespa/cloud/SystemInfoTest.java b/hosted-zone-api/src/test/java/ai/vespa/cloud/SystemInfoTest.java index 7f6921ab114..c14955d6614 100644 --- a/hosted-zone-api/src/test/java/ai/vespa/cloud/SystemInfoTest.java +++ b/hosted-zone-api/src/test/java/ai/vespa/cloud/SystemInfoTest.java @@ -15,11 +15,13 @@ public class SystemInfoTest { @Test public void testSystemInfo() { + ApplicationId application = new ApplicationId("tenant1", "application1", "instance1"); Zone zone = new Zone(Environment.dev, "us-west-1"); Cluster cluster = new Cluster(1, List.of()); Node node = new Node(0); - SystemInfo info = new SystemInfo(zone, cluster, node); + SystemInfo info = new SystemInfo(application, zone, cluster, node); + assertEquals(application, info.application()); assertEquals(zone, info.zone()); assertEquals(cluster, info.cluster()); assertEquals(node, info.node()); diff --git a/jdisc-cloud-aws/src/main/java/com/yahoo/jdisc/cloud/aws/VespaAwsCredentialsProvider.java b/jdisc-cloud-aws/src/main/java/com/yahoo/jdisc/cloud/aws/VespaAwsCredentialsProvider.java index 4424b63dcc4..1991ff3ad88 100644 --- a/jdisc-cloud-aws/src/main/java/com/yahoo/jdisc/cloud/aws/VespaAwsCredentialsProvider.java +++ b/jdisc-cloud-aws/src/main/java/com/yahoo/jdisc/cloud/aws/VespaAwsCredentialsProvider.java @@ -13,24 +13,19 @@ import com.yahoo.slime.SlimeUtils; import java.io.IOException; import java.io.UncheckedIOException; import java.nio.file.Files; -import java.nio.file.NoSuchFileException; import java.nio.file.Path; import java.util.concurrent.atomic.AtomicReference; public class VespaAwsCredentialsProvider implements AWSCredentialsProvider { private static final String DEFAULT_CREDENTIALS_PATH = "/opt/vespa/var/vespa/aws/credentials.json"; - // TODO (freva): Remove when host-admin writes to the new path above - private static final String DEFAULT_CREDENTIALS_PATH_OLD = "/opt/vespa/var/container-data/opt/vespa/conf/vespa/credentials.json"; private final AtomicReference<AWSCredentials> credentials = new AtomicReference<>(); private final Path credentialsPath; - private final Path credentialsPathOld; public VespaAwsCredentialsProvider() { this.credentialsPath = Path.of(DEFAULT_CREDENTIALS_PATH); - this.credentialsPathOld = Path.of(DEFAULT_CREDENTIALS_PATH_OLD); refresh(); } @@ -50,12 +45,7 @@ public class VespaAwsCredentialsProvider implements AWSCredentialsProvider { private AWSSessionCredentials readCredentials() { try { - Slime slime; - try { - slime = SlimeUtils.jsonToSlime(Files.readAllBytes(credentialsPath)); - } catch (NoSuchFileException ignored) { - slime = SlimeUtils.jsonToSlime(Files.readAllBytes(credentialsPathOld)); - } + Slime slime = SlimeUtils.jsonToSlime(Files.readAllBytes(credentialsPath)); Cursor cursor = slime.get(); String accessKey = cursor.field("awsAccessKey").asString(); String secretKey = cursor.field("awsSecretKey").asString(); diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/application/package-info.java b/jdisc_core/src/main/java/com/yahoo/jdisc/application/package-info.java index a633e4bb291..950b3569319 100644 --- a/jdisc_core/src/main/java/com/yahoo/jdisc/application/package-info.java +++ b/jdisc_core/src/main/java/com/yahoo/jdisc/application/package-info.java @@ -3,7 +3,8 @@ * <p>Provides classes and interfaces for implementing an {@link com.yahoo.jdisc.application.Application * Application}.</p> * - * <h3>Application</h3> + * <h2>Application</h2> + * * <p>In every jDISC process there is exactly one Application instance, it is created during jDISC startup, and it is * destroyed during jDISC shutdown. The Application uses the {@link com.yahoo.jdisc.application.ContainerBuilder * ContainerBuilder} interface to load OSGi {@link org.osgi.framework.Bundle Bundles}, install Guice {@link @@ -52,7 +53,7 @@ void reconfigureApplication() { } </pre> * - * <h3>Application and OSGi</h3> + * <h2>Application and OSGi</h2> * <p>At the heart of jDISC is an OSGi framework. An Application is always packaged as an OSGi bundle. The OSGi * technology itself is a set of specifications that define a dynamic component system for Java. These specifications * enable a development model where applications are (dynamically) composed of many different (reusable) components. The @@ -105,7 +106,7 @@ void reconfigureApplication() { * into the application itself. Unless incompatible API changes are made to 3rd party jDISC components, it should be * possible to upgrade dependencies without having to recompile and redeploy the Application.</p> * - * <h3>Application deployment</h3> + * <h2>Application deployment</h2> * <p>jDISC allows a single binary to execute any application without having to change the command line parameters. * Instead of * modifying the parameters of the single application binary, changing the application is achieved by setting a single diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/handler/package-info.java b/jdisc_core/src/main/java/com/yahoo/jdisc/handler/package-info.java index 1b08ef5e60c..088c9d5592d 100644 --- a/jdisc_core/src/main/java/com/yahoo/jdisc/handler/package-info.java +++ b/jdisc_core/src/main/java/com/yahoo/jdisc/handler/package-info.java @@ -3,7 +3,7 @@ * <p>Provides classes and interfaces for implementing a {@link com.yahoo.jdisc.handler.RequestHandler * RequestHandler}.</p> * - * <h3>RequestHandler</h3> + * <h2>RequestHandler</h2> * <p>All {@link com.yahoo.jdisc.Request Requests} in a jDISC application are processed by RequestHandlers. These are * components created by the {@link com.yahoo.jdisc.application.Application Application}, and bound to one or more URI * patterns through the {@link com.yahoo.jdisc.application.ContainerBuilder ContainerBuilder} API. Upon receiving a @@ -39,7 +39,7 @@ MyApplication(ContainerActivator activator, CurrentContainer container) { * otherwise, a jDISC application that is intended to forward large streams of data can do so without having to make any * copies of that data as it is passing through.</p> * - * <h3>ResponseHandler</h3> + * <h2>ResponseHandler</h2> * <p>The complement of the Request is the Response. A Response is a numeric status code and a set of header fields. * Just as Requests are processed by RequestHandlers, Responses are processed by ResponseHandlers. The ResponseHandler * interface is fully asynchronous, and uses the ContentChannel class to encapsulate the asynchronous passing of diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/service/package-info.java b/jdisc_core/src/main/java/com/yahoo/jdisc/service/package-info.java index 532d2921469..cc2cd4abbef 100644 --- a/jdisc_core/src/main/java/com/yahoo/jdisc/service/package-info.java +++ b/jdisc_core/src/main/java/com/yahoo/jdisc/service/package-info.java @@ -3,7 +3,7 @@ * <p>Provides classes and interfaces for implementing a {@link com.yahoo.jdisc.service.ClientProvider ClientProvider} or * a {@link com.yahoo.jdisc.service.ServerProvider ServerProvider}.</p> * - * <h3>ServerProvider</h3> + * <h2>ServerProvider</h2> * <p>All {@link com.yahoo.jdisc.Request Requests} that are processed in a jDISC application are created by * ServerProviders. These are components created by the {@link com.yahoo.jdisc.application.Application Application}, and * they are the parts of jDISC that accept incoming connections. The ServerProvider creates and dispatches Request diff --git a/jrt/src/com/yahoo/jrt/slobrok/api/Mirror.java b/jrt/src/com/yahoo/jrt/slobrok/api/Mirror.java index 47c9c545d29..51a3c6b6146 100644 --- a/jrt/src/com/yahoo/jrt/slobrok/api/Mirror.java +++ b/jrt/src/com/yahoo/jrt/slobrok/api/Mirror.java @@ -33,11 +33,12 @@ public class Mirror implements IMirror { private static final Logger log = Logger.getLogger(Mirror.class.getName()); - private final Supervisor orb; + private final Supervisor orb; private final SlobrokList slobroks; private String currSlobrok; private final BackOffPolicy backOff; private volatile int updates = 0; + private volatile long iterations = 0; private boolean requestDone = false; private boolean logOnSuccess = true; private final AtomicReference<Entry[]> specs = new AtomicReference<>(new Entry[0]); @@ -169,6 +170,7 @@ public class Mirror implements IMirror { * Invoked by the update task. */ private void checkForUpdate() { + ++iterations; if (requestDone) { handleUpdate(); requestDone = false; @@ -327,4 +329,12 @@ public class Mirror implements IMirror { } + public void dumpState() { + log.log(Level.INFO, "location broker mirror state: " + + " iterations: " + iterations + + ", connected to: " + target + + ", seen " + updates + " updates" + + ", current server: "+ currSlobrok + + ", list of servers: " + slobroks); + } } diff --git a/logserver/src/main/java/com/yahoo/logserver/filter/LogFilterManager.java b/logserver/src/main/java/com/yahoo/logserver/filter/LogFilterManager.java index 66de14a071b..b343758b2bf 100644 --- a/logserver/src/main/java/com/yahoo/logserver/filter/LogFilterManager.java +++ b/logserver/src/main/java/com/yahoo/logserver/filter/LogFilterManager.java @@ -1,11 +1,11 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.logserver.filter; +import com.yahoo.log.LogLevel; + import java.util.HashMap; import java.util.Map; -import com.yahoo.log.LogLevel; - /** * The LogFilterManager keeps track of associations between * LogFilter names and instances, so that access to filters @@ -23,8 +23,6 @@ public class LogFilterManager { LevelFilter allEvents = new LevelFilter(); allEvents.addLevel(LogLevel.EVENT); instance.addLogFilterInternal("system.allevents", allEvents); - instance.addLogFilterInternal("system.metricsevents", new MetricsFilter()); - instance.addLogFilterInternal("system.nometricsevents", new NoMetricsFilter()); instance.addLogFilterInternal("system.all", new NullFilter()); instance.addLogFilterInternal("system.mute", MuteFilter.getInstance()); } diff --git a/logserver/src/main/java/com/yahoo/logserver/filter/MetricsFilter.java b/logserver/src/main/java/com/yahoo/logserver/filter/MetricsFilter.java deleted file mode 100644 index 38c3d4ebee1..00000000000 --- a/logserver/src/main/java/com/yahoo/logserver/filter/MetricsFilter.java +++ /dev/null @@ -1,50 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.logserver.filter; - -import com.yahoo.log.LogLevel; -import com.yahoo.log.event.Count; -import com.yahoo.log.event.CountGroup; -import com.yahoo.log.event.Event; -import com.yahoo.log.event.Histogram; -import com.yahoo.log.event.MalformedEventException; -import com.yahoo.log.event.Value; -import com.yahoo.log.event.ValueGroup; -import com.yahoo.log.LogMessage; - -/** - * This filter matches events that are used for monitoring, specificly - * the Count and Value events. - * - * @author Bjorn Borud - */ -public class MetricsFilter implements LogFilter { - public boolean isLoggable (LogMessage msg) { - if (msg.getLevel() != LogLevel.EVENT) { - return false; - } - - Event event; - try { - event = msg.getEvent(); - } - catch (MalformedEventException e) { - return false; - } - - // if it is not Count, Value or something which will generate - // Count or Value we don't care - if (! ((event instanceof Count) - || (event instanceof Value) - || (event instanceof Histogram) - || (event instanceof CountGroup) - || (event instanceof ValueGroup))) { - return false; - } - - return true; - } - - public String description () { - return "Match all events representing system metrics (Counts, Values, etc)."; - } -} diff --git a/logserver/src/main/java/com/yahoo/logserver/filter/NoMetricsFilter.java b/logserver/src/main/java/com/yahoo/logserver/filter/NoMetricsFilter.java deleted file mode 100644 index 71782ed8b0a..00000000000 --- a/logserver/src/main/java/com/yahoo/logserver/filter/NoMetricsFilter.java +++ /dev/null @@ -1,21 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.logserver.filter; - -import com.yahoo.log.LogMessage; - -/** - * This filter is the complement of MetricsFilter - * - * @author Bjorn Borud - */ -public class NoMetricsFilter implements LogFilter { - private final MetricsFilter filter = new MetricsFilter(); - - public boolean isLoggable (LogMessage msg) { - return (! filter.isLoggable(msg)); - } - - public String description () { - return "Matches all log messages except Count and Value events"; - } -} diff --git a/logserver/src/main/java/com/yahoo/logserver/handlers/archive/ArchiverHandler.java b/logserver/src/main/java/com/yahoo/logserver/handlers/archive/ArchiverHandler.java index 0e9eeefcc46..0b44e47f183 100644 --- a/logserver/src/main/java/com/yahoo/logserver/handlers/archive/ArchiverHandler.java +++ b/logserver/src/main/java/com/yahoo/logserver/handlers/archive/ArchiverHandler.java @@ -1,10 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.logserver.handlers.archive; -import java.util.logging.Level; import com.yahoo.log.LogMessage; -import com.yahoo.logserver.filter.LogFilter; -import com.yahoo.logserver.filter.LogFilterManager; import com.yahoo.logserver.handlers.AbstractLogHandler; import java.io.File; @@ -14,6 +11,7 @@ import java.util.Calendar; import java.util.GregorianCalendar; import java.util.Iterator; import java.util.TimeZone; +import java.util.logging.Level; import java.util.logging.Logger; @@ -44,7 +42,7 @@ public class ArchiverHandler extends AbstractLogHandler { /** * Max number of log files open at any given time */ - private final int maxFilesOpen = 100; + private static final int maxFilesOpen = 100; /** * The maximum number of bytes we allow a file to grow to @@ -68,11 +66,6 @@ public class ArchiverHandler extends AbstractLogHandler { */ private final LogWriterLRUCache logWriterLRUCache; - /** - * Filtering - */ - private LogFilter filter = null; - private FilesArchived filesArchived; /** @@ -83,17 +76,10 @@ public class ArchiverHandler extends AbstractLogHandler { dateformat = new SimpleDateFormat("yyyy/MM/dd/HH"); dateformat.setTimeZone(TimeZone.getTimeZone("UTC")); - // Set up filtering - String archiveMetrics = System.getProperty("vespa_log_server__archive_metric"); - if ("off".equals(archiveMetrics)) { - filter = LogFilterManager.getLogFilter("system.nometricsevents"); - } - - setLogFilter(filter); + setLogFilter(null); // set up LRU for files - logWriterLRUCache = new LogWriterLRUCache(maxFilesOpen, - (float) 0.75); + logWriterLRUCache = new LogWriterLRUCache(maxFilesOpen, (float) 0.75); } /** diff --git a/logserver/src/test/files/value-events.txt b/logserver/src/test/files/value-events.txt deleted file mode 100644 index d08452ecff9..00000000000 --- a/logserver/src/test/files/value-events.txt +++ /dev/null @@ -1,818 +0,0 @@ -1107248423.987624 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0 -1107248423.987754 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0 -1107248423.987809 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0 -1107248439.235755 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="queued" value=0 -1107248439.235871 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="active" value=0 -1107248439.236632 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0 -1107248439.236684 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0 -1107248423.987624 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0 -1107248423.987754 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0 -1107248423.987809 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0 -1107248439.235755 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="queued" value=0 -1107248439.235871 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="active" value=0 -1107248439.236632 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0 -1107248439.236684 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0 -1107248454.346566 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0 -1107248454.346701 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0 -1107248454.346755 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0 -1107248484.708180 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0 -1107248484.708309 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0 -1107248484.708362 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0 -1107248499.377732 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="queued" value=0 -1107248499.377855 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="active" value=0 -1107248499.378618 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0 -1107248499.378670 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0 -1107248515.068667 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0 -1107248515.068795 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0 -1107248515.068848 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0 -1107248545.429750 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0 -1107248545.429879 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0 -1107248545.429933 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0 -1107248559.510318 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="queued" value=0 -1107248559.510382 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="active" value=0 -1107248559.511135 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0 -1107248559.511187 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0 -1107248575.790604 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0 -1107248575.790728 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0 -1107248575.790780 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0 -1107248606.151589 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0 -1107248606.151717 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0 -1107248606.151769 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0 -1107248619.602250 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="queued" value=0 -1107248619.602314 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="active" value=0 -1107248619.603067 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0 -1107248619.603119 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0 -1107248636.512734 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0 -1107248636.512861 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0 -1107248636.512915 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0 -1107248666.873517 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0 -1107248666.873740 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0 -1107248666.873796 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0 -1107248679.704182 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="queued" value=0 -1107248679.704244 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="active" value=0 -1107248679.704995 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0 -1107248679.705048 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0 -1107248697.234524 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0 -1107248697.234653 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0 -1107248697.234708 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0 -1107248727.595687 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0 -1107248727.595817 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0 -1107248727.595870 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0 -1107248739.755755 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="queued" value=0 -1107248739.755880 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="active" value=0 -1107248739.756771 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0 -1107248739.756824 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0 -1107248757.956408 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0 -1107248757.956542 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0 -1107248757.956595 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0 -1107248788.317656 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0 -1107248788.317784 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0 -1107248788.317838 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0 -1107248799.887618 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="queued" value=0 -1107248799.887747 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="active" value=0 -1107248799.888510 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0 -1107248799.888563 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0 -1107248818.678404 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0 -1107248818.678542 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0 -1107248818.678607 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0 -1107248849.039894 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0 -1107248849.040024 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0 -1107248849.040079 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0 -1107248859.959613 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="queued" value=0 -1107248859.959735 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="active" value=0 -1107248859.960498 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0 -1107248859.960550 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0 -1107248879.400552 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0 -1107248879.400678 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0 -1107248879.400732 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0 -1107248909.761338 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0 -1107248909.761479 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0 -1107248909.761537 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0 -1107248920.061583 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="queued" value=0 -1107248920.061705 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="active" value=0 -1107248920.062472 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0 -1107248920.062525 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0 -1107248940.122556 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0 -1107248940.122697 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0 -1107248940.122751 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0 -1107248970.483540 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0 -1107248970.483669 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0 -1107248970.483723 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0 -1107248980.223600 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="queued" value=0 -1107248980.223719 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="active" value=0 -1107248980.224483 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0 -1107248980.224536 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0 -1107249000.844582 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0 -1107249000.844711 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0 -1107249000.844765 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0 -1107249031.205764 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0 -1107249031.205990 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0 -1107249031.206047 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0 -1107249040.295520 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="queued" value=0 -1107249040.295643 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="active" value=0 -1107249040.296405 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0 -1107249040.296457 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0 -1107249061.566458 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0 -1107249061.566586 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0 -1107249061.566640 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0 -1107249091.927910 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0 -1107249091.928037 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0 -1107249091.928091 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0 -1107249100.327467 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="queued" value=0 -1107249100.327593 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="active" value=0 -1107249100.328494 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0 -1107249100.328547 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0 -1107249122.288543 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0 -1107249122.288671 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0 -1107249122.288724 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0 -1107249152.649685 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0 -1107249152.649816 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0 -1107249152.649869 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0 -1107249160.509467 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="queued" value=0 -1107249160.509615 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="active" value=0 -1107249160.510802 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0 -1107249160.510855 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0 -1107249183.011765 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0 -1107249183.011895 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0 -1107249183.011948 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0 -1107249213.371426 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0 -1107249213.371565 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0 -1107249213.371622 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0 -1107249220.551434 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="queued" value=0 -1107249220.551563 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="active" value=0 -1107249220.552329 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0 -1107249220.552381 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0 -1107249243.732724 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0 -1107249243.732852 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0 -1107249243.732906 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0 -1107249274.093649 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0 -1107249274.093780 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0 -1107249274.093833 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0 -1107249280.663427 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="queued" value=0 -1107249280.663550 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="active" value=0 -1107249280.664313 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0 -1107249280.664365 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0 -1107249304.454704 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0 -1107249304.454833 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0 -1107249304.454886 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0 -1107249334.815726 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0 -1107249334.815858 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0 -1107249334.815912 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0 -1107249340.825357 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="queued" value=0 -1107249340.825488 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="active" value=0 -1107249340.826289 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0 -1107249340.826342 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0 -1107249365.177542 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0 -1107249365.177669 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0 -1107249365.177723 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0 -1107249395.541404 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0 -1107249395.541529 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0 -1107249395.541582 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0 -1107249482.760126 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="queued" value=0 -1107249482.760259 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="active" value=0 -1107249482.760989 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0 -1107249482.761042 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0 -1107249521.879344 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 -1107249521.879842 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 -1107249521.879913 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 -1107249542.731983 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="queued" value=0 -1107249542.732513 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="active" value=0 -1107249542.733591 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0 -1107249542.733644 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0 -1107249552.232878 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 -1107249552.233293 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 -1107249552.233356 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 -1107249582.593375 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 -1107249582.594022 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 -1107249582.594304 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 -1107249602.934015 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="queued" value=0 -1107249602.935138 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="active" value=0 -1107249602.936242 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0 -1107249602.936310 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0 -1107249612.954767 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 -1107249612.954976 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 -1107249612.955162 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 -1107249643.315410 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 -1107249643.315971 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 -1107249643.316213 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 -1107249662.938231 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="queued" value=0 -1107249662.938636 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="active" value=0 -1107249662.939500 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0 -1107249662.939552 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0 -1107249673.676574 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 -1107249673.677044 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 -1107249673.677282 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 -1107249704.037945 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 -1107249704.038370 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 -1107249704.038433 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 -1107249723.107946 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="queued" value=0 -1107249723.108405 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="active" value=0 -1107249723.109464 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0 -1107249723.109518 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0 -1107249734.398557 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 -1107249734.399061 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 -1107249734.399424 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 -1107249764.760077 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 -1107249764.760499 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 -1107249764.760562 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 -1107249783.149953 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="queued" value=0 -1107249783.150509 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="active" value=0 -1107249783.151530 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0 -1107249783.151610 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0 -1107249795.120517 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 -1107249795.121115 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 -1107249795.121470 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 -1107249825.481800 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 -1107249825.482123 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 -1107249825.482265 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 -1107249843.301904 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="queued" value=0 -1107249843.303003 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="active" value=0 -1107249843.304400 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0 -1107249843.304486 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0 -1107249855.842709 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 -1107249855.843038 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 -1107249855.843176 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 -1107249886.203512 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 -1107249886.203987 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 -1107249886.204214 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 -1107249903.433833 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="queued" value=0 -1107249903.434091 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="active" value=0 -1107249903.435239 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0 -1107249903.435292 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0 -1107249916.564719 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 -1107249916.565133 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 -1107249916.565277 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 -1107249946.925416 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 -1107249946.925938 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 -1107249946.926367 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 -1107249963.525789 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="queued" value=0 -1107249963.526251 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="active" value=0 -1107249963.527498 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0 -1107249963.527562 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0 -1107249977.286716 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 -1107249977.286985 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 -1107249977.287134 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 -1107250007.647650 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 -1107250007.648010 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 -1107250007.648158 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 -1107250023.557752 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="queued" value=0 -1107250023.558227 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="active" value=0 -1107250023.559250 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0 -1107250023.559334 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0 -1107250038.008626 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 -1107250038.008972 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 -1107250038.009117 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 -1107250068.369715 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 -1107250068.370110 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 -1107250068.370631 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 -1107250083.689727 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="queued" value=0 -1107250083.690303 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="active" value=0 -1107250083.691188 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0 -1107250083.691300 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0 -1107250098.730564 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 -1107250098.732151 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 -1107250098.732322 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 -1107250129.092925 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 -1107250129.093340 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 -1107250129.093403 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 -1107250143.761756 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="queued" value=0 -1107250143.762207 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="active" value=0 -1107250143.763278 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0 -1107250143.763355 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0 -1107250159.453836 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 -1107250159.454250 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 -1107250159.454313 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 -1107250189.813551 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 -1107250189.814359 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 -1107250189.814797 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 -1107250203.923674 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="queued" value=0 -1107250203.924112 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="active" value=0 -1107250203.925109 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0 -1107250203.925196 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0 -1107250220.176334 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 -1107250220.176748 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 -1107250220.176811 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 -1107250250.535671 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 -1107250250.536188 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 -1107250250.536510 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 -1107250263.945671 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="queued" value=0 -1107250263.945967 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="active" value=0 -1107250263.947065 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0 -1107250263.947151 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0 -1107250280.896680 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 -1107250280.897147 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 -1107250280.897565 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 -1107250311.257655 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 -1107250311.258113 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 -1107250311.258344 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 -1107250324.067765 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="queued" value=0 -1107250324.068159 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="active" value=0 -1107250324.068986 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0 -1107250324.069111 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0 -1107248423.987624 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0 -1107248423.987754 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0 -1107248423.987809 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0 -1107248439.235755 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="queued" value=0 -1107248439.235871 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="active" value=0 -1107248439.236632 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0 -1107248439.236684 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0 -1107248454.346566 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0 -1107248454.346701 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0 -1107248454.346755 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0 -1107248484.708180 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0 -1107248484.708309 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0 -1107248484.708362 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0 -1107248499.377732 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="queued" value=0 -1107248499.377855 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="active" value=0 -1107248499.378618 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0 -1107248499.378670 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0 -1107248515.068667 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0 -1107248515.068795 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0 -1107248515.068848 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0 -1107248545.429750 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0 -1107248545.429879 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0 -1107248545.429933 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0 -1107248559.510318 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="queued" value=0 -1107248559.510382 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="active" value=0 -1107248559.511135 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0 -1107248559.511187 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0 -1107248575.790604 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0 -1107248575.790728 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0 -1107248575.790780 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0 -1107248606.151589 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0 -1107248606.151717 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0 -1107248606.151769 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0 -1107248619.602250 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="queued" value=0 -1107248619.602314 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="active" value=0 -1107248619.603067 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0 -1107248619.603119 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0 -1107248636.512734 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0 -1107248636.512861 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0 -1107248636.512915 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0 -1107248666.873517 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0 -1107248666.873740 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0 -1107248666.873796 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0 -1107248679.704182 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="queued" value=0 -1107248679.704244 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="active" value=0 -1107248679.704995 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0 -1107248679.705048 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0 -1107248697.234524 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0 -1107248697.234653 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0 -1107248697.234708 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0 -1107248727.595687 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0 -1107248727.595817 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0 -1107248727.595870 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0 -1107248739.755755 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="queued" value=0 -1107248739.755880 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="active" value=0 -1107248739.756771 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0 -1107248739.756824 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0 -1107248757.956408 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0 -1107248757.956542 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0 -1107248757.956595 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0 -1107248788.317656 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0 -1107248788.317784 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0 -1107248788.317838 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0 -1107248799.887618 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="queued" value=0 -1107248799.887747 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="active" value=0 -1107248799.888510 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0 -1107248799.888563 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0 -1107248818.678404 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0 -1107248818.678542 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0 -1107248818.678607 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0 -1107248849.039894 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0 -1107248849.040024 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0 -1107248849.040079 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0 -1107248859.959613 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="queued" value=0 -1107248859.959735 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="active" value=0 -1107248859.960498 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0 -1107248859.960550 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0 -1107248879.400552 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0 -1107248879.400678 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0 -1107248879.400732 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0 -1107248909.761338 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0 -1107248909.761479 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0 -1107248909.761537 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0 -1107248920.061583 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="queued" value=0 -1107248920.061705 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="active" value=0 -1107248920.062472 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0 -1107248920.062525 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0 -1107248940.122556 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0 -1107248940.122697 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0 -1107248940.122751 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0 -1107248970.483540 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0 -1107248970.483669 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0 -1107248970.483723 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0 -1107248980.223600 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="queued" value=0 -1107248980.223719 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="active" value=0 -1107248980.224483 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0 -1107248980.224536 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0 -1107249000.844582 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0 -1107249000.844711 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0 -1107249000.844765 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0 -1107249031.205764 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0 -1107249031.205990 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0 -1107249031.206047 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0 -1107249040.295520 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="queued" value=0 -1107249040.295643 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="active" value=0 -1107249040.296405 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0 -1107249040.296457 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0 -1107249061.566458 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0 -1107249061.566586 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0 -1107249061.566640 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0 -1107249091.927910 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0 -1107249091.928037 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0 -1107249091.928091 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0 -1107249100.327467 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="queued" value=0 -1107249100.327593 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="active" value=0 -1107249100.328494 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0 -1107249100.328547 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0 -1107249122.288543 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0 -1107249122.288671 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0 -1107249122.288724 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0 -1107249152.649685 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0 -1107249152.649816 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0 -1107249152.649869 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0 -1107249160.509467 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="queued" value=0 -1107249160.509615 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="active" value=0 -1107249160.510802 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0 -1107249160.510855 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0 -1107249183.011765 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0 -1107249183.011895 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0 -1107249183.011948 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0 -1107249213.371426 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0 -1107249213.371565 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0 -1107249213.371622 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0 -1107249220.551434 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="queued" value=0 -1107249220.551563 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="active" value=0 -1107249220.552329 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0 -1107249220.552381 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0 -1107249243.732724 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0 -1107249243.732852 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0 -1107249243.732906 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0 -1107249274.093649 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0 -1107249274.093780 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0 -1107249274.093833 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0 -1107249280.663427 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="queued" value=0 -1107249280.663550 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="active" value=0 -1107249280.664313 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0 -1107249280.664365 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0 -1107249304.454704 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0 -1107249304.454833 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0 -1107249304.454886 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0 -1107249334.815726 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0 -1107249334.815858 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0 -1107249334.815912 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0 -1107249340.825357 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="queued" value=0 -1107249340.825488 example.yahoo.com 28555 vsm fsearch.queryperf event value/1 name="active" value=0 -1107249340.826289 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0 -1107249340.826342 example.yahoo.com 28555 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0 -1107249365.177542 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0 -1107249365.177669 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0 -1107249365.177723 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0 -1107249395.541404 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numcollections" value=0 -1107249395.541529 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numdocuments" value=0 -1107249395.541582 example.yahoo.com 28577 storage storageserver.collection-man event value/1 name="numbytes" value=0 -1107249482.760126 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="queued" value=0 -1107249482.760259 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="active" value=0 -1107249482.760989 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0 -1107249482.761042 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0 -1107249521.879344 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 -1107249521.879842 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 -1107249521.879913 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 -1107249542.731983 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="queued" value=0 -1107249542.732513 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="active" value=0 -1107249542.733591 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0 -1107249542.733644 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0 -1107249552.232878 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 -1107249552.233293 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 -1107249552.233356 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 -1107249582.593375 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 -1107249582.594022 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 -1107249582.594304 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 -1107249602.934015 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="queued" value=0 -1107249602.935138 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="active" value=0 -1107249602.936242 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0 -1107249602.936310 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0 -1107249612.954767 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 -1107249612.954976 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 -1107249612.955162 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 -1107249643.315410 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 -1107249643.315971 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 -1107249643.316213 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 -1107249662.938231 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="queued" value=0 -1107249662.938636 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="active" value=0 -1107249662.939500 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0 -1107249662.939552 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0 -1107249673.676574 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 -1107249673.677044 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 -1107249673.677282 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 -1107249704.037945 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 -1107249704.038370 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 -1107249704.038433 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 -1107249723.107946 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="queued" value=0 -1107249723.108405 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="active" value=0 -1107249723.109464 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0 -1107249723.109518 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0 -1107249734.398557 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 -1107249734.399061 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 -1107249734.399424 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 -1107249764.760077 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 -1107249764.760499 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 -1107249764.760562 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 -1107249783.149953 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="queued" value=0 -1107249783.150509 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="active" value=0 -1107249783.151530 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0 -1107249783.151610 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0 -1107249795.120517 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 -1107249795.121115 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 -1107249795.121470 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 -1107249825.481800 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 -1107249825.482123 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 -1107249825.482265 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 -1107249843.301904 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="queued" value=0 -1107249843.303003 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="active" value=0 -1107249843.304400 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0 -1107249843.304486 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0 -1107249855.842709 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 -1107249855.843038 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 -1107249855.843176 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 -1107249886.203512 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 -1107249886.203987 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 -1107249886.204214 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 -1107249903.433833 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="queued" value=0 -1107249903.434091 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="active" value=0 -1107249903.435239 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0 -1107249903.435292 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0 -1107249916.564719 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 -1107249916.565133 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 -1107249916.565277 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 -1107249946.925416 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 -1107249946.925938 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 -1107249946.926367 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 -1107249963.525789 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="queued" value=0 -1107249963.526251 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="active" value=0 -1107249963.527498 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0 -1107249963.527562 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0 -1107249977.286716 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 -1107249977.286985 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 -1107249977.287134 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 -1107250007.647650 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 -1107250007.648010 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 -1107250007.648158 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 -1107250023.557752 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="queued" value=0 -1107250023.558227 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="active" value=0 -1107250023.559250 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0 -1107250023.559334 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0 -1107250038.008626 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 -1107250038.008972 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 -1107250038.009117 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 -1107250068.369715 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 -1107250068.370110 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 -1107250068.370631 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 -1107250083.689727 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="queued" value=0 -1107250083.690303 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="active" value=0 -1107250083.691188 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0 -1107250083.691300 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0 -1107250098.730564 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 -1107250098.732151 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 -1107250098.732322 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 -1107250129.092925 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 -1107250129.093340 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 -1107250129.093403 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 -1107250143.761756 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="queued" value=0 -1107250143.762207 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="active" value=0 -1107250143.763278 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0 -1107250143.763355 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0 -1107250159.453836 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 -1107250159.454250 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 -1107250159.454313 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 -1107250189.813551 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 -1107250189.814359 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 -1107250189.814797 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 -1107250203.923674 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="queued" value=0 -1107250203.924112 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="active" value=0 -1107250203.925109 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0 -1107250203.925196 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0 -1107250220.176334 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 -1107250220.176748 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 -1107250220.176811 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 -1107250250.535671 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 -1107250250.536188 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 -1107250250.536510 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 -1107250263.945671 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="queued" value=0 -1107250263.945967 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="active" value=0 -1107250263.947065 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0 -1107250263.947151 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0 -1107250280.896680 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 -1107250280.897147 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 -1107250280.897565 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 -1107250311.257655 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 -1107250311.258113 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 -1107250311.258344 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 -1107250324.067765 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="queued" value=0 -1107250324.068159 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="active" value=0 -1107250324.068986 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0 -1107250324.069111 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0 -1107250341.618469 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 -1107250341.619001 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 -1107250341.619367 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 -1107250371.979550 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 -1107250371.980003 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 -1107250371.980254 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 -1107250384.209615 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="queued" value=0 -1107250384.210153 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="active" value=0 -1107250384.211269 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0 -1107250384.211322 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0 -1107250402.340723 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 -1107250402.341145 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 -1107250402.341208 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 -1107250432.701722 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 -1107250432.702052 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 -1107250432.702193 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 -1107250444.241654 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="queued" value=0 -1107250444.242282 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="active" value=0 -1107250444.243410 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0 -1107250444.243524 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0 -1107250463.062779 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 -1107250463.063193 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 -1107250463.063257 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 -1107250493.423518 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 -1107250493.423869 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 -1107250493.424011 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 -1107250504.433587 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="queued" value=0 -1107250504.434091 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="active" value=0 -1107250504.435178 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0 -1107250504.435231 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0 -1107250523.791832 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 -1107250523.791966 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 -1107250523.792019 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 -1107250554.145291 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 -1107250554.145429 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 -1107250554.145494 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 -1107250564.525521 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="queued" value=0 -1107250564.525632 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="active" value=0 -1107250564.526387 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0 -1107250564.526439 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0 -1107250584.506801 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 -1107250584.506957 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 -1107250584.507011 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 -1107250614.868677 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 -1107250614.868807 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 -1107250614.868861 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 -1107250624.567617 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="queued" value=0 -1107250624.567734 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="active" value=0 -1107250624.568485 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0 -1107250624.568536 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0 -1107250645.229659 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 -1107250645.229812 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 -1107250645.229866 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 -1107250675.589502 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 -1107250675.589644 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 -1107250675.589708 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 -1107250684.729465 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="queued" value=0 -1107250684.729580 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="active" value=0 -1107250684.730461 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0 -1107250684.730512 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0 -1107250705.950505 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 -1107250705.950692 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 -1107250705.950769 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 -1107250736.311593 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 -1107250736.311734 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 -1107250736.311798 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 -1107250744.761494 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="queued" value=0 -1107250744.761626 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="active" value=0 -1107250744.762381 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0 -1107250744.762433 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0 -1107250766.672522 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 -1107250766.672708 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 -1107250766.672763 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 -1107250797.034486 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 -1107250797.034617 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 -1107250797.034670 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 -1107250804.863485 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="queued" value=0 -1107250804.863613 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="active" value=0 -1107250804.864360 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0 -1107250804.864411 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0 -1107250827.395696 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 -1107250827.395823 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 -1107250827.395876 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 -1107250857.756404 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 -1107250857.756533 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 -1107250857.756585 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 -1107250865.035459 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="queued" value=0 -1107250865.035581 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="active" value=0 -1107250865.036337 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0 -1107250865.036388 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0 -1107250888.116534 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 -1107250888.116674 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 -1107250888.116737 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 -1107250918.479148 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 -1107250918.479277 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 -1107250918.479332 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 -1107250925.137420 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="queued" value=0 -1107250925.137544 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="active" value=0 -1107250925.138351 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0 -1107250925.138403 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0 -1107250948.838645 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 -1107250948.838779 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 -1107250948.838836 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 -1107250979.199468 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 -1107250979.199645 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 -1107250979.199705 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 -1107250985.130997 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="queued" value=0 -1107250985.131117 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="active" value=0 -1107250985.131975 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0 -1107250985.132027 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0 -1107251009.560525 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 -1107251009.560664 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 -1107251009.560727 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 -1107251039.921739 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 -1107251039.921872 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 -1107251039.921926 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 -1107251045.311481 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="queued" value=0 -1107251045.311598 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="active" value=0 -1107251045.312351 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0 -1107251045.312403 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0 -1107251070.282709 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 -1107251070.282849 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 -1107251070.282918 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 -1107251100.643735 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 -1107251100.643875 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 -1107251100.643928 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 -1107251105.403383 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="queued" value=0 -1107251105.403514 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="active" value=0 -1107251105.404268 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0 -1107251105.404320 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0 -1107251131.004511 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 -1107251131.004656 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 -1107251131.004753 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 -1107251161.365529 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 -1107251161.365664 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 -1107251161.365721 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 -1107251165.455342 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="queued" value=0 -1107251165.455454 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="active" value=0 -1107251165.456208 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0 -1107251165.456260 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0 -1107251191.726752 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 -1107251191.726890 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 -1107251191.726957 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 -1107251222.087961 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 -1107251222.088092 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 -1107251222.088146 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 -1107251225.587446 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="queued" value=0 -1107251225.587559 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="active" value=0 -1107251225.588311 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0 -1107251225.588363 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0 -1107251252.448823 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 -1107251252.448959 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 -1107251252.449014 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 -1107251282.809626 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 -1107251282.809768 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 -1107251282.809831 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 -1107251285.629323 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="queued" value=0 -1107251285.629447 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="active" value=0 -1107251285.630309 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0 -1107251285.630361 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0 -1107251313.170437 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 -1107251313.170575 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 -1107251313.170631 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 -1107251343.531690 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 -1107251343.531826 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 -1107251343.531880 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 -1107251345.811348 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="queued" value=0 -1107251345.811464 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="active" value=0 -1107251345.812220 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0 -1107251345.812272 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0 -1107251373.892751 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 -1107251373.892890 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 -1107251373.892954 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 -1107251404.253471 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 -1107251404.253618 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 -1107251404.253675 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 -1107251405.923299 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="queued" value=0 -1107251405.923418 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="active" value=0 -1107251405.924170 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0 -1107251405.924221 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0 -1107251434.614618 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 -1107251434.614759 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 -1107251434.614823 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 -1107251464.975470 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 -1107251464.975607 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 -1107251464.975662 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 -1107251466.005315 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="queued" value=0 -1107251466.005448 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="active" value=0 -1107251466.006208 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0 -1107251466.006260 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0 -1107251495.336518 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 -1107251495.336658 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 -1107251495.336721 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 -1107251525.697793 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 -1107251525.697922 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 -1107251525.697975 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 -1107251526.047229 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="queued" value=0 -1107251526.047353 example.yahoo.com 59768 vsm fsearch.queryperf event value/1 name="active" value=0 -1107251526.048109 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.1st.latency.ms.max" value=0 -1107251526.048161 example.yahoo.com 59768 vsm fsearch.vsm.vsmmanager event value/1 name="chunk.all.latency.ms.max" value=0 -1107251556.058595 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numcollections" value=0 -1107251556.058737 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numdocuments" value=0 -1107251556.058848 example.yahoo.com 59791 storage storageserver.collection-man event value/1 name="numbytes" value=0 diff --git a/logserver/src/test/java/com/yahoo/logserver/filter/test/LogFilterManagerTestCase.java b/logserver/src/test/java/com/yahoo/logserver/filter/test/LogFilterManagerTestCase.java index c8fad351c28..a88a9b9646e 100644 --- a/logserver/src/test/java/com/yahoo/logserver/filter/test/LogFilterManagerTestCase.java +++ b/logserver/src/test/java/com/yahoo/logserver/filter/test/LogFilterManagerTestCase.java @@ -1,17 +1,15 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.logserver.filter.test; +import com.yahoo.logserver.filter.LevelFilter; import com.yahoo.logserver.filter.LogFilter; import com.yahoo.logserver.filter.LogFilterManager; -import com.yahoo.logserver.filter.LevelFilter; -import com.yahoo.logserver.filter.MetricsFilter; -import com.yahoo.logserver.filter.NoMetricsFilter; -import com.yahoo.logserver.filter.NullFilter; import com.yahoo.logserver.filter.MuteFilter; +import com.yahoo.logserver.filter.NullFilter; +import org.junit.Test; -import org.junit.*; - -import static org.junit.Assert.*; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; public class LogFilterManagerTestCase { @@ -23,15 +21,6 @@ public class LogFilterManagerTestCase { assertNotNull(f); assertTrue(f instanceof LevelFilter); - f = LogFilterManager.getLogFilter("system.metricsevents"); - assertNotNull(f); - assertTrue(f instanceof MetricsFilter); - - f = LogFilterManager.getLogFilter("system.nometricsevents"); - assertNotNull(f); - assertTrue(f instanceof NoMetricsFilter); - - f = LogFilterManager.getLogFilter("system.all"); assertNotNull(f); assertTrue(f instanceof NullFilter); diff --git a/logserver/src/test/java/com/yahoo/logserver/filter/test/MetricsFilterTestCase.java b/logserver/src/test/java/com/yahoo/logserver/filter/test/MetricsFilterTestCase.java deleted file mode 100644 index c04e44e474b..00000000000 --- a/logserver/src/test/java/com/yahoo/logserver/filter/test/MetricsFilterTestCase.java +++ /dev/null @@ -1,43 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.logserver.filter.test; - -import java.io.BufferedReader; -import java.io.FileReader; -import java.io.IOException; - -import com.yahoo.log.event.Event; -import com.yahoo.log.event.MalformedEventException; -import com.yahoo.log.InvalidLogFormatException; -import com.yahoo.log.LogMessage; -import com.yahoo.logserver.filter.MetricsFilter; - -import org.junit.*; - -import static org.junit.Assert.*; - -public class MetricsFilterTestCase { - - @Test - public void testValueEvents() throws InvalidLogFormatException, IOException { - MetricsFilter filter = new MetricsFilter(); - String filename = "src/test/files/value-events.txt"; - BufferedReader br = new BufferedReader(new FileReader(filename)); - for (String line = br.readLine(); line != null; line = br.readLine()) { - LogMessage m = LogMessage.parseNativeFormat(line); - assertNotNull(m); - try { - Event event = m.getEvent(); - assertNotNull(event); - } catch (MalformedEventException e) { - fail(); - } - - if (filter.isLoggable(m)) { - assertTrue(true); - } else { - fail(); - } - } - - } -} diff --git a/logserver/src/test/java/com/yahoo/logserver/filter/test/NoMetricsFilterTestCase.java b/logserver/src/test/java/com/yahoo/logserver/filter/test/NoMetricsFilterTestCase.java deleted file mode 100644 index f8dcc294ce7..00000000000 --- a/logserver/src/test/java/com/yahoo/logserver/filter/test/NoMetricsFilterTestCase.java +++ /dev/null @@ -1,56 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.logserver.filter.test; - -import java.io.BufferedReader; -import java.io.FileReader; -import java.io.IOException; - -import com.yahoo.log.event.Event; -import com.yahoo.log.event.MalformedEventException; -import com.yahoo.log.InvalidLogFormatException; -import com.yahoo.log.LogMessage; -import com.yahoo.logserver.filter.MetricsFilter; -import com.yahoo.logserver.filter.NoMetricsFilter; - -import org.junit.*; - -import static org.junit.Assert.*; - -/** - * Ensure that the NoMetricsFilter does the opposite of MetricsFilter - */ -public class NoMetricsFilterTestCase { - - @Test - public void testValueEvents() throws InvalidLogFormatException, IOException { - NoMetricsFilter filter = new NoMetricsFilter(); - MetricsFilter metricsFilter = new MetricsFilter(); - - String filename = "src/test/files/value-events.txt"; - BufferedReader br = new BufferedReader(new FileReader(filename)); - for (String line = br.readLine(); line != null; line = br.readLine()) { - LogMessage m = LogMessage.parseNativeFormat(line); - assertNotNull(m); - - try { - Event event = m.getEvent(); - assertNotNull(event); - } catch (MalformedEventException e) { - fail(); - } - - if (filter.isLoggable(m)) { - fail(); - } else { - assertTrue(true); - } - - - if (metricsFilter.isLoggable(m)) { - assertTrue(true); - } else { - fail(); - } - } - } -} diff --git a/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/RPCNetwork.java b/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/RPCNetwork.java index 6b4a0853e29..7ed1b17a5b4 100644 --- a/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/RPCNetwork.java +++ b/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/RPCNetwork.java @@ -42,6 +42,8 @@ import java.util.concurrent.SynchronousQueue; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.logging.Level; +import java.util.logging.Logger; import java.util.stream.Collectors; /** @@ -51,6 +53,8 @@ import java.util.stream.Collectors; */ public class RPCNetwork implements Network, MethodHandler { + private static Logger log = Logger.getLogger(RPCNetwork.class.getName()); + private final AtomicBoolean destroyed = new AtomicBoolean(false); private final Identity identity; private final Supervisor orb; @@ -84,7 +88,7 @@ public class RPCNetwork implements Network, MethodHandler { * @param params a complete set of parameters * @param slobrokConfig subscriber for slobroks config */ - public RPCNetwork(RPCNetworkParams params, SlobrokConfigSubscriber slobrokConfig) { + private RPCNetwork(RPCNetworkParams params, SlobrokConfigSubscriber slobrokConfig) { this.slobroksConfig = slobrokConfig; identity = params.getIdentity(); orb = new Supervisor(new Transport("mbus-rpc-" + identity.getServicePrefix(), params.getNumNetworkThreads(), @@ -147,6 +151,10 @@ public class RPCNetwork implements Network, MethodHandler { if (mirror.ready()) { return true; } + if ((i % 100) == 0) { + log.log(Level.INFO, "waiting for network to become ready ("+(i/100)+" of "+((int)seconds)+" seconds)"); + mirror.dumpState(); + } try { Thread.sleep(10); } catch (InterruptedException e) { diff --git a/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/SlobrokConfigSubscriber.java b/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/SlobrokConfigSubscriber.java index fac1fc0cefd..903a31d3f3a 100755 --- a/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/SlobrokConfigSubscriber.java +++ b/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/SlobrokConfigSubscriber.java @@ -5,6 +5,9 @@ import com.yahoo.config.subscription.ConfigSubscriber; import com.yahoo.jrt.slobrok.api.SlobrokList; import com.yahoo.cloud.config.SlobroksConfig; +import java.util.logging.Level; +import java.util.logging.Logger; + /** * This class implements subscription to slobrok config. * @@ -12,6 +15,7 @@ import com.yahoo.cloud.config.SlobroksConfig; */ public class SlobrokConfigSubscriber implements ConfigSubscriber.SingleSubscriber<SlobroksConfig>{ + private static final Logger log = Logger.getLogger(SlobrokConfigSubscriber.class.getName()); private final SlobrokList slobroks = new SlobrokList(); private ConfigSubscriber subscriber; @@ -21,11 +25,13 @@ public class SlobrokConfigSubscriber implements ConfigSubscriber.SingleSubscribe * @param configId the id of the config to subscribe to */ public SlobrokConfigSubscriber(String configId) { + log.log(Level.FINE, "new slobrok config subscriber with config id: "+configId); subscriber = new ConfigSubscriber(); subscriber.subscribe(this, SlobroksConfig.class, configId); } public SlobrokConfigSubscriber(SlobroksConfig slobroksConfig) { + log.log(Level.FINE, "new slobrok config subscriber with fixed list: "+slobroksConfig); configure(slobroksConfig); } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeSpec.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeSpec.java index 7ac8d43f881..acb6ece6fc1 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeSpec.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeSpec.java @@ -791,7 +791,8 @@ public class NodeSpec { .type(NodeType.tenant) .flavor("d-2-8-50") .resources(new NodeResources(2, 8, 50, 10)) - .realResources(new NodeResources(2, 8, 50, 10)); + .realResources(new NodeResources(2, 8, 50, 10)) + .events(List.of(new Event("operator", "rebooted", Instant.EPOCH))); // Set the required allocated fields if (EnumSet.of(NodeState.active, NodeState.inactive, NodeState.reserved).contains(state)) { diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/Container.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/Container.java index f2f8ffb8d22..dd725a11364 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/Container.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/Container.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.hosted.node.admin.container; import com.yahoo.config.provision.DockerImage; +import java.time.Instant; import java.util.List; import java.util.Map; import java.util.Objects; @@ -19,10 +20,10 @@ public class Container extends PartialContainer { private final int conmonPid; private final List<Network> networks; - public Container(ContainerId id, ContainerName name, State state, String imageId, DockerImage image, + public Container(ContainerId id, ContainerName name, Instant createdAt, State state, String imageId, DockerImage image, Map<String, String> labels, int pid, int conmonPid, String hostname, ContainerResources resources, List<Network> networks, boolean managed) { - super(id, name, state, imageId, image, labels, pid, managed); + super(id, name, createdAt, state, imageId, image, labels, pid, managed); this.hostname = Objects.requireNonNull(hostname); this.resources = Objects.requireNonNull(resources); this.conmonPid = conmonPid; diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/PartialContainer.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/PartialContainer.java index d320ffd294c..22146767e01 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/PartialContainer.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/PartialContainer.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.hosted.node.admin.container; import com.yahoo.config.provision.DockerImage; +import java.time.Instant; import java.util.Map; import java.util.Objects; @@ -15,6 +16,7 @@ public class PartialContainer { private final ContainerId id; private final ContainerName name; + private final Instant createdAt; private final State state; private final String imageId; private final DockerImage image; @@ -22,10 +24,11 @@ public class PartialContainer { private final int pid; private final boolean managed; - public PartialContainer(ContainerId id, ContainerName name, State state, String imageId, + public PartialContainer(ContainerId id, ContainerName name, Instant createdAt, State state, String imageId, DockerImage image, Map<String, String> labels, int pid, boolean managed) { this.id = Objects.requireNonNull(id); this.name = Objects.requireNonNull(name); + this.createdAt = Objects.requireNonNull(createdAt); this.state = Objects.requireNonNull(state); this.imageId = Objects.requireNonNull(imageId); this.image = Objects.requireNonNull(image); @@ -44,6 +47,11 @@ public class PartialContainer { return name; } + /** Timestamp when this container was created */ + public Instant createdAt() { + return createdAt; + } + /** Current state of this */ public State state() { return state; @@ -86,12 +94,12 @@ public class PartialContainer { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; PartialContainer that = (PartialContainer) o; - return pid == that.pid && managed == that.managed && id.equals(that.id) && name.equals(that.name) && state == that.state && imageId.equals(that.imageId) && image.equals(that.image) && labels.equals(that.labels); + return pid == that.pid && managed == that.managed && id.equals(that.id) && name.equals(that.name) && createdAt.equals(that.createdAt) && state == that.state && imageId.equals(that.imageId) && image.equals(that.image) && labels.equals(that.labels); } @Override public int hashCode() { - return Objects.hash(id, name, state, imageId, image, labels, pid, managed); + return Objects.hash(id, name, createdAt, state, imageId, image, labels, pid, managed); } /** The state of a container */ diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/image/ContainerImagePruner.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/image/ContainerImagePruner.java index 82a898b4d4b..236a4d6718f 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/image/ContainerImagePruner.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/image/ContainerImagePruner.java @@ -1,7 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.node.admin.container.image; -import com.google.common.base.Strings; import com.yahoo.collections.Pair; import com.yahoo.vespa.hosted.node.admin.component.TaskContext; import com.yahoo.vespa.hosted.node.admin.container.ContainerEngine; @@ -10,12 +9,11 @@ import com.yahoo.vespa.hosted.node.admin.container.PartialContainer; import java.time.Clock; import java.time.Duration; import java.time.Instant; +import java.util.ArrayList; import java.util.Collections; -import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Objects; -import java.util.Optional; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.function.Function; @@ -78,61 +76,39 @@ public class ContainerImagePruner { Map<String, Image> imageByImageId = images.stream().collect(Collectors.toMap(Image::id, Function.identity())); - // Find all the ancestors for every local image id, this includes the image id itself - Map<String, Set<String>> ancestorsByImageId = images.stream() - .map(Image::id) - .collect(Collectors.toMap( - Function.identity(), - imageId -> { - Set<String> ancestors = new HashSet<>(); - while (!Strings.isNullOrEmpty(imageId)) { - ancestors.add(imageId); - imageId = Optional.of(imageId).map(imageByImageId::get).flatMap(Image::parentId).orElse(null); - } - return ancestors; - } - )); - // The set of images that we want to keep is: // 1. The images that were recently used // 2. The images that were explicitly excluded - // 3. All of the ancestors of from images in 1 & 2 Set<String> imagesToKeep = Stream .concat( updateRecentlyUsedImageIds(images, containers, minAge).stream(), // 1 referencesToImages(excludedRefs, images).stream()) // 2 - .flatMap(imageId -> ancestorsByImageId.getOrDefault(imageId, Collections.emptySet()).stream()) // 3 .collect(Collectors.toSet()); // Now take all the images we have locally - return imageByImageId.keySet().stream() - - // filter out images we want to keep - .filter(imageId -> !imagesToKeep.contains(imageId)) - - // Sort images in an order is safe to delete (children before parents) - .sorted((o1, o2) -> { - // If image2 is parent of image1, image1 comes before image2 - if (imageIsDescendantOf(imageByImageId, o1, o2)) return -1; - // If image1 is parent of image2, image2 comes before image1 - else if (imageIsDescendantOf(imageByImageId, o2, o1)) return 1; - // Otherwise, sort lexicographically by image name (For testing) - else return o1.compareTo(o2); - }) - - // Map back to image - .map(imageByImageId::get) - - // Delete image, if successful also remove last usage time to prevent re-download being instantly deleted - .peek(image -> { - // Deleting an image by image ID with multiple tags will fail -> delete by tags instead - referencesOf(image).forEach(imageReference -> { - LOG.info("Deleting unused image " + imageReference); - containerEngine.removeImage(context, imageReference); - }); - lastTimeUsedByImageId.remove(image.id()); - }) - .count() > 0; + List<Image> imagesToRemove = imageByImageId.keySet().stream() + // filter out images we want to keep + .filter(imageId -> !imagesToKeep.contains(imageId)) + .map(imageByImageId::get) + .collect(Collectors.toCollection(ArrayList::new)); + + // We cannot delete an image that is referenced by other images as parent. Computing parent image is complicated, see + // https://github.com/containers/podman/blob/d7b2f03f8a5d0e3789ac185ea03989463168fb76/vendor/github.com/containers/common/libimage/layer_tree.go#L235:L299 + // https://github.com/containers/podman/blob/d7b2f03f8a5d0e3789ac185ea03989463168fb76/vendor/github.com/containers/common/libimage/oci.go#L30:L97 + // In practice, our images do not have any parents on prod machines, so we should be able to delete in any + // order. In case we ever do get a parent on a host somehow, we could get stuck if we always attempt to delete + // in wrong order, so shuffle first to ensure this eventually converges + Collections.shuffle(imagesToRemove); + + imagesToRemove.forEach(image -> { + // Deleting an image by image ID with multiple tags will fail -> delete by tags instead + referencesOf(image).forEach(imageReference -> { + LOG.info("Deleting unused image " + imageReference); + containerEngine.removeImage(context, imageReference); + }); + lastTimeUsedByImageId.remove(image.id()); + }); + return !imagesToRemove.isEmpty(); } private Set<String> updateRecentlyUsedImageIds(List<Image> images, List<PartialContainer> containers, Duration minImageAgeToDelete) { @@ -169,18 +145,6 @@ public class ContainerImagePruner { } /** - * @return true if ancestor is a parent or grand-parent or grand-grand-parent, etc. of img - */ - private boolean imageIsDescendantOf(Map<String, Image> imageIdToImage, String img, String ancestor) { - while (imageIdToImage.containsKey(img)) { - img = imageIdToImage.get(img).parentId().orElse(null); - if (img == null) return false; - if (ancestor.equals(img)) return true; - } - return false; - } - - /** * Returns list of references to given image, preferring image tag(s), if any exist. * * If image is untagged, its ID is returned instead. diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/image/Image.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/image/Image.java index 5a47e5d335b..0f8e5cc6381 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/image/Image.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/image/Image.java @@ -3,7 +3,6 @@ package com.yahoo.vespa.hosted.node.admin.container.image; import java.util.List; import java.util.Objects; -import java.util.Optional; /** * This represents a container image that exists locally. @@ -13,12 +12,10 @@ import java.util.Optional; public class Image { private final String id; - private final Optional<String> parentId; private final List<String> names; - public Image(String id, Optional<String> parentId, List<String> names) { + public Image(String id, List<String> names) { this.id = Objects.requireNonNull(id); - this.parentId = Objects.requireNonNull(parentId); this.names = List.copyOf(Objects.requireNonNull(names)); } @@ -27,11 +24,6 @@ public class Image { return id; } - /** ID of the parent image of this, if any */ - public Optional<String> parentId() { - return parentId; - } - /** Names for this image, such as tags or digests */ public List<String> names() { return names; @@ -42,12 +34,12 @@ public class Image { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; Image image = (Image) o; - return id.equals(image.id) && parentId.equals(image.parentId) && names.equals(image.names); + return id.equals(image.id) && names.equals(image.names); } @Override public int hashCode() { - return Objects.hash(id, parentId, names); + return Objects.hash(id, names); } @Override diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java index 3ab196a052e..92aacf8827b 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java @@ -189,19 +189,29 @@ public class NodeAgentImpl implements NodeAgent { } } - private void updateNodeRepoWithCurrentAttributes(NodeAgentContext context) { + private void updateNodeRepoWithCurrentAttributes(NodeAgentContext context, Optional<Instant> containerCreatedAt) { final NodeAttributes currentNodeAttributes = new NodeAttributes(); final NodeAttributes newNodeAttributes = new NodeAttributes(); + boolean changed = false; if (context.node().wantedRestartGeneration().isPresent() && !Objects.equals(context.node().currentRestartGeneration(), currentRestartGeneration)) { currentNodeAttributes.withRestartGeneration(context.node().currentRestartGeneration()); newNodeAttributes.withRestartGeneration(currentRestartGeneration); + changed = true; } - if (!Objects.equals(context.node().currentRebootGeneration(), currentRebootGeneration)) { + boolean createdAtAfterRebootedEvent = context.node().events().stream() + .filter(event -> event.type().equals("rebooted")) + .map(event -> containerCreatedAt + .map(createdAt -> createdAt.isAfter(event.at())) + .orElse(false)) // Container not created + .findFirst() + .orElse(containerCreatedAt.isPresent()); // No rebooted event + if (!Objects.equals(context.node().currentRebootGeneration(), currentRebootGeneration) || createdAtAfterRebootedEvent) { currentNodeAttributes.withRebootGeneration(context.node().currentRebootGeneration()); newNodeAttributes.withRebootGeneration(currentRebootGeneration); + changed = true; } Optional<DockerImage> actualDockerImage = context.node().wantedDockerImage().filter(n -> containerState == UNKNOWN); @@ -213,16 +223,13 @@ public class NodeAgentImpl implements NodeAgent { currentNodeAttributes.withVespaVersion(currentImage.tagAsVersion()); newNodeAttributes.withDockerImage(newImage); newNodeAttributes.withVespaVersion(newImage.tagAsVersion()); + changed = true; } - publishStateToNodeRepoIfChanged(context, currentNodeAttributes, newNodeAttributes); - } - - private void publishStateToNodeRepoIfChanged(NodeAgentContext context, NodeAttributes currentAttributes, NodeAttributes newAttributes) { - if (!currentAttributes.equals(newAttributes)) { + if (changed) { context.log(logger, "Publishing new set of attributes to node repo: %s -> %s", - currentAttributes, newAttributes); - nodeRepository.updateNodeAttributes(context.hostname().value(), newAttributes); + currentNodeAttributes, newNodeAttributes); + nodeRepository.updateNodeAttributes(context.hostname().value(), newNodeAttributes); } } @@ -454,7 +461,7 @@ public class NodeAgentImpl implements NodeAgent { case inactive: case parked: removeContainerIfNeededUpdateContainerState(context, container); - updateNodeRepoWithCurrentAttributes(context); + updateNodeRepoWithCurrentAttributes(context, Optional.empty()); stopServicesIfNeeded(context); break; case active: @@ -501,7 +508,7 @@ public class NodeAgentImpl implements NodeAgent { // has been successfully rolled out. // - Slobrok and internal orchestrator state is used to determine whether // to allow upgrade (suspend). - updateNodeRepoWithCurrentAttributes(context); + updateNodeRepoWithCurrentAttributes(context, container.map(Container::createdAt)); if (suspendedInOrchestrator || node.orchestratorStatus().isSuspended()) { context.log(logger, "Call resume against Orchestrator"); orchestrator.resume(context.hostname().value()); @@ -517,7 +524,7 @@ public class NodeAgentImpl implements NodeAgent { credentialsMaintainers.forEach(maintainer -> maintainer.clearCredentials(context)); storageMaintainer.syncLogs(context, false); storageMaintainer.archiveNodeStorage(context); - updateNodeRepoWithCurrentAttributes(context); + updateNodeRepoWithCurrentAttributes(context, Optional.empty()); nodeRepository.setNodeState(context.hostname().value(), NodeState.ready); break; default: diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerPath.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerPath.java index 4f12c9439f2..dfeb3d7b3ee 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerPath.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerPath.java @@ -44,7 +44,7 @@ public class ContainerPath implements Path { public Path pathOnHost() { return pathOnHost; } public String pathInContainer() { return '/' + String.join("/", parts); } public ContainerPath withUser(UnixUser user) { return new ContainerPath(containerFs, pathOnHost, parts, user); } - UnixUser user() { return user; } + public UnixUser user() { return user; } @Override public ContainerFileSystem getFileSystem() { diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/ContainerEngineMock.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/ContainerEngineMock.java index 4507c424c98..3eab24a7a66 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/ContainerEngineMock.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/ContainerEngineMock.java @@ -9,6 +9,7 @@ import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext; import com.yahoo.vespa.hosted.node.admin.task.util.process.CommandResult; import java.time.Duration; +import java.time.Instant; import java.util.List; import java.util.Map; import java.util.Objects; @@ -84,7 +85,7 @@ public class ContainerEngineMock implements ContainerEngine { @Override public void updateContainer(NodeAgentContext context, ContainerId containerId, ContainerResources containerResources) { Container container = requireContainer(context.containerName()); - containers.put(container.name(), new Container(containerId, container.name(), container.state(), + containers.put(container.name(), new Container(containerId, container.name(), container.createdAt(), container.state(), container.imageId(), container.image(), container.labels(), container.pid(), container.conmonPid(), container.hostname(), @@ -120,7 +121,7 @@ public class ContainerEngineMock implements ContainerEngine { @Override public void pullImage(TaskContext context, DockerImage image, RegistryCredentials registryCredentials) { String imageId = image.asString(); - ImageDownload imageDownload = images.computeIfAbsent(imageId, (ignored) -> new ImageDownload(new Image(imageId, Optional.empty(), List.of(imageId)))); + ImageDownload imageDownload = images.computeIfAbsent(imageId, (ignored) -> new ImageDownload(new Image(imageId, List.of(imageId)))); if (!asyncImageDownload) { imageDownload.complete(); } @@ -160,6 +161,7 @@ public class ContainerEngineMock implements ContainerEngine { public Container createContainer(NodeAgentContext context, PartialContainer.State state, ContainerResources containerResources) { return new Container(new ContainerId("id-of-" + context.containerName()), context.containerName(), + Instant.EPOCH, state, "image-id", context.node().wantedDockerImage().get(), diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/ContainerOperationsTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/ContainerOperationsTest.java index e6c63220d35..d0c54dd4e04 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/ContainerOperationsTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/ContainerOperationsTest.java @@ -7,6 +7,7 @@ import com.yahoo.vespa.test.file.TestFileSystem; import org.junit.Test; import java.nio.file.FileSystem; +import java.time.Instant; import java.util.List; import java.util.Map; import java.util.Set; @@ -58,7 +59,7 @@ public class ContainerOperationsTest { } private Container createContainer(String name, boolean managed) { - return new Container(new ContainerId("id-of-" + name), new ContainerName(name), PartialContainer.State.running, + return new Container(new ContainerId("id-of-" + name), new ContainerName(name), Instant.EPOCH, PartialContainer.State.running, "image-id", DockerImage.EMPTY, Map.of(), 42, 43, name, ContainerResources.UNLIMITED, List.of(), managed); } diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/image/ContainerImagePrunerTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/image/ContainerImagePrunerTest.java index 43d3fe94552..a7307b5d1de 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/image/ContainerImagePrunerTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/image/ContainerImagePrunerTest.java @@ -10,17 +10,13 @@ import com.yahoo.vespa.hosted.node.admin.container.ContainerEngineMock; import com.yahoo.vespa.hosted.node.admin.container.ContainerId; import com.yahoo.vespa.hosted.node.admin.container.ContainerName; import com.yahoo.vespa.hosted.node.admin.container.ContainerResources; -import com.yahoo.vespa.hosted.node.admin.container.image.ContainerImagePruner; -import com.yahoo.vespa.hosted.node.admin.container.image.Image; import org.junit.Test; import java.time.Duration; -import java.util.Arrays; -import java.util.Collections; +import java.time.Instant; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Optional; import static org.junit.Assert.assertTrue; @@ -54,70 +50,31 @@ public class ContainerImagePrunerTest { .expectDeletedImages(); } - @Test public void multipleUnusedImagesAreIdentified() { tester.withExistingImages(image("image-1"), image("image-2")) .expectDeletedImages("image-1", "image-2"); } - - @Test - public void multipleUnusedLeavesAreIdentified() { - tester.withExistingImages(image("parent-image"), - image("image-1", "parent-image"), - image("image-2", "parent-image")) - .expectDeletedImages("image-1", "image-2", "parent-image"); - } - - - @Test - public void unusedLeafWithUsedSiblingIsIdentified() { - tester.withExistingImages(image("parent-image"), - image("image-1", "parent-image", "latest"), - image("image-2", "parent-image", "1.24")) - .withExistingContainers(container("vespa-node-1", "image-1")) - .expectDeletedImages("1.24"); // Deleting the only tag will delete the image - } - - @Test public void unusedImagesWithMultipleTags() { - tester.withExistingImages(image("parent-image"), - image("image-1", "parent-image", "vespa-6", "vespa-6.28", "vespa:latest")) - .expectDeletedImages("vespa-6", "vespa-6.28", "vespa:latest", "parent-image"); + tester.withExistingImages(image("image-1", "vespa-6", "vespa-6.28", "vespa:latest")) + .expectDeletedImages("vespa-6", "vespa-6.28", "vespa:latest"); } - @Test public void unusedImagesWithMultipleUntagged() { - tester.withExistingImages(image("image1", null, "<none>:<none>"), - image("image2", null, "<none>:<none>")) + tester.withExistingImages(image("image1", "<none>:<none>"), + image("image2", "<none>:<none>")) .expectDeletedImages("image1", "image2"); } - @Test public void taggedImageWithNoContainersIsUnused() { - tester.withExistingImages(image("image-1", null, "vespa-6")) + tester.withExistingImages(image("image-1", "vespa-6")) .expectDeletedImages("vespa-6"); } - - @Test - public void unusedImagesWithSimpleImageGc() { - tester.withExistingImages(image("parent-image")) - .expectDeletedImagesAfterMinutes(30) - .withExistingImages(image("parent-image"), - image("image-1", "parent-image")) - .expectDeletedImagesAfterMinutes(0) - .expectDeletedImagesAfterMinutes(30) - // At this point, parent-image has been unused for 1h, but image-1 depends on parent-image and it has - // only been unused for 30m, so we cannot delete parent-image yet. 30 mins later both can be removed - .expectDeletedImagesAfterMinutes(30, "image-1", "parent-image"); - } - - @Test public void reDownloadingImageIsNotImmediatelyDeleted() { tester.withExistingImages(image("image")) @@ -127,54 +84,43 @@ public class ContainerImagePrunerTest { .expectDeletedImages("image"); // 1h after re-download it is deleted again } - @Test public void reDownloadingImageIsNotImmediatelyDeletedWhenDeletingByTag() { - tester.withExistingImages(image("image", null, "image-1", "my-tag")) - .expectDeletedImages("image-1", "my-tag") // After 1h we delete image + tester.withExistingImages(image("image", "my-tag")) + .expectDeletedImages("my-tag") // After 1h we delete image .expectDeletedImagesAfterMinutes(0) // image is immediately re-downloaded, but is not deleted .expectDeletedImagesAfterMinutes(10) - .expectDeletedImages("image-1", "my-tag"); // 1h after re-download it is deleted again + .expectDeletedImages("my-tag"); // 1h after re-download it is deleted again } /** Same scenario as in {@link #multipleUnusedImagesAreIdentified()} */ @Test public void doesNotDeleteExcludedByIdImages() { - tester.withExistingImages(image("parent-image"), - image("image-1", "parent-image"), - image("image-2", "parent-image")) - // Normally, image-1 and parent-image should also be deleted, but because we exclude image-1 - // we cannot delete parent-image, so only image-2 is deleted + tester.withExistingImages(image("image-1"), image("image-2")) + // Normally, image-1 should also be deleted, but because we exclude image-1 only image-2 is deleted .expectDeletedImages(List.of("image-1"), "image-2"); } /** Same as in {@link #doesNotDeleteExcludedByIdImages()} but with tags */ @Test public void doesNotDeleteExcludedByTagImages() { - tester.withExistingImages(image("parent-image", "rhel-6"), - image("image-1", "parent-image", "vespa:6.288.16"), - image("image-2", "parent-image", "vespa:6.289.94")) + tester.withExistingImages(image("image-1", "vespa:6.288.16"), image("image-2", "vespa:6.289.94")) .expectDeletedImages(List.of("vespa:6.288.16"), "vespa:6.289.94"); } @Test - public void exludingNotDownloadedImageIsNoop() { - tester.withExistingImages(image("parent-image", "rhel-6"), - image("image-1", "parent-image", "vespa:6.288.16"), - image("image-2", "parent-image", "vespa:6.289.94")) + public void excludingNotDownloadedImageIsNoop() { + tester.withExistingImages(image("image-1", "vespa:6.288.16"), + image("image-2", "vespa:6.289.94")) .expectDeletedImages(List.of("vespa:6.300.1"), "vespa:6.288.16", "vespa:6.289.94", "rhel-6"); } - private static Image image(String id) { - return image(id, null); - } - - private static Image image(String id, String parentId, String... tags) { - return new Image(id, Optional.ofNullable(parentId), List.of(tags)); + private static Image image(String id, String... tags) { + return new Image(id, List.of(tags)); } private static Container container(String name, String imageId) { - return new Container(new ContainerId("id-of-" + name), new ContainerName(name), + return new Container(new ContainerId("id-of-" + name), new ContainerName(name), Instant.EPOCH, Container.State.running, imageId, DockerImage.EMPTY, Map.of(), 42, 43, name + ".example.com", ContainerResources.UNLIMITED, List.of(), true); @@ -209,7 +155,7 @@ public class ContainerImagePrunerTest { } private Tester expectDeletedImagesAfterMinutes(int minutesAfter, String... imageIds) { - return expectDeletedImagesAfterMinutes(minutesAfter, Collections.emptyList(), imageIds); + return expectDeletedImagesAfterMinutes(minutesAfter, List.of(), imageIds); } private Tester expectDeletedImagesAfterMinutes(int minutesAfter, List<String> excludedRefs, String... imageIds) { @@ -223,7 +169,7 @@ public class ContainerImagePrunerTest { pruner.removeUnusedImages(context, excludedRefs, Duration.ofHours(1).minusSeconds(1)); - Arrays.stream(imageIds) + List.of(imageIds) .forEach(imageId -> { int newValue = removalCountByImageId.getOrDefault(imageId, 0) + 1; removalCountByImageId.put(imageId, newValue); diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java index 1c6831e6379..13f101be6e8 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java @@ -187,7 +187,7 @@ public class NodeAgentImplTest { inOrder.verify(containerOperations, times(1)).resumeNode(eq(context)); inOrder.verify(healthChecker, times(1)).verifyHealth(eq(context)); inOrder.verify(nodeRepository).updateNodeAttributes( - hostName, new NodeAttributes().withDockerImage(dockerImage).withVespaVersion(dockerImage.tagAsVersion())); + hostName, new NodeAttributes().withDockerImage(dockerImage).withVespaVersion(dockerImage.tagAsVersion()).withRebootGeneration(0)); inOrder.verify(orchestrator, never()).resume(hostName); } @@ -285,7 +285,7 @@ public class NodeAgentImplTest { inOrder.verify(containerOperations).removeContainer(eq(secondContext), any()); inOrder.verify(containerOperations, never()).updateContainer(any(), any(), any()); inOrder.verify(containerOperations, never()).restartVespa(any()); - inOrder.verify(nodeRepository).updateNodeAttributes(eq(hostName), eq(new NodeAttributes().withRestartGeneration(2))); + inOrder.verify(nodeRepository).updateNodeAttributes(eq(hostName), eq(new NodeAttributes().withRestartGeneration(2).withRebootGeneration(0))); nodeAgent.doConverge(secondContext); inOrder.verify(orchestrator).resume(any(String.class)); @@ -610,7 +610,7 @@ public class NodeAgentImplTest { inOrder.verify(aclMaintainer, times(1)).converge(eq(context)); inOrder.verify(containerOperations, times(1)).resumeNode(eq(context)); inOrder.verify(nodeRepository).updateNodeAttributes( - hostName, new NodeAttributes().withDockerImage(dockerImage).withVespaVersion(dockerImage.tagAsVersion())); + hostName, new NodeAttributes().withDockerImage(dockerImage).withVespaVersion(dockerImage.tagAsVersion()).withRebootGeneration(0)); inOrder.verify(orchestrator).resume(hostName); } @@ -765,6 +765,7 @@ public class NodeAgentImplTest { Optional.of(new Container( containerId, ContainerName.fromHostname(hostName), + clock.instant(), isRunning ? Container.State.running : Container.State.exited, "image-id-1", dockerImage, diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java index 812befda432..e7a20ff54b3 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java @@ -432,10 +432,12 @@ public final class Node implements Nodelike { /** Returns a copy of this node with the current reboot generation set to the given number at the given instant */ public Node withCurrentRebootGeneration(long generation, Instant instant) { + if (generation < status.reboot().current()) + throw new IllegalArgumentException("Cannot set reboot generation to " + generation + + ": lower than current generation: " + status.reboot().current()); + Status newStatus = status().withReboot(status().reboot().withCurrent(generation)); - History newHistory = history(); - if (generation > status().reboot().current()) - newHistory = history.with(new History.Event(History.Event.Type.rebooted, Agent.system, instant)); + History newHistory = history.with(new History.Event(History.Event.Type.rebooted, Agent.system, instant)); return this.with(newStatus).with(newHistory); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/SharedLoadBalancerService.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/SharedLoadBalancerService.java index 44fc8cf53b5..7dbebd1fc47 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/SharedLoadBalancerService.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/SharedLoadBalancerService.java @@ -17,8 +17,11 @@ import java.util.Set; import java.util.stream.Collectors; /** - * This code mimics provisioning load balancers that already exist in the routing layer. - * It will just map the load balancer request to the proxy nodes available in the node repository. + * This implementation of {@link LoadBalancerService} returns the load balancer(s) that exists by default in the shared + * routing layer. + * + * Since such load balancers always exist, we can return the hostname of the routing layer VIP and the networks of the + * proxy nodes directly. Nothing has to be provisioned. * * @author ogronnesby */ @@ -27,32 +30,26 @@ public class SharedLoadBalancerService implements LoadBalancerService { private static final Comparator<Node> hostnameComparator = Comparator.comparing(Node::hostname); private final NodeRepository nodeRepository; + private final String vipHostname; - public SharedLoadBalancerService(NodeRepository nodeRepository) { + public SharedLoadBalancerService(NodeRepository nodeRepository, String vipHostname) { this.nodeRepository = Objects.requireNonNull(nodeRepository); + this.vipHostname = Objects.requireNonNull(vipHostname); } @Override public LoadBalancerInstance create(LoadBalancerSpec spec, boolean force) { NodeList proxyNodes = nodeRepository.nodes().list().nodeType(NodeType.proxy).sortedBy(hostnameComparator); - - if (proxyNodes.size() == 0) { - throw new IllegalStateException("Missing proxy nodes in node repository"); - } - - Node firstProxyNode = proxyNodes.first().get(); + if (proxyNodes.isEmpty()) throw new IllegalStateException("No proxy nodes found in node-repository"); Set<String> networks = proxyNodes.stream() .flatMap(node -> node.ipConfig().primary().stream()) .map(SharedLoadBalancerService::withPrefixLength) .collect(Collectors.toSet()); - - return new LoadBalancerInstance( - HostName.from(firstProxyNode.hostname()), - Optional.empty(), - Set.of(4080, 4443), - networks, - spec.reals() - ); + return new LoadBalancerInstance(HostName.from(vipHostname), + Optional.empty(), + Set.of(4080, 4443), + networks, + spec.reals()); } @Override diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeResourceLimits.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeResourceLimits.java index 0ff315b7a2a..d5dbe08dca9 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeResourceLimits.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeResourceLimits.java @@ -73,7 +73,7 @@ public class NodeResourceLimits { private double minAdvertisedMemoryGb(ClusterSpec.Type clusterType) { if (zone().system() == SystemName.dev) return 1; // Allow small containers in dev system if (clusterType == ClusterSpec.Type.admin) return 1; - return 2; + return 4; } private double minAdvertisedDiskGb(NodeResources requested, boolean exclusive) { diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java index 4483b09d04b..a3c7b7d2d2b 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java @@ -470,9 +470,9 @@ public class AutoscalingTest { tester.deploy(application1, cluster1, 6, 1, hostResources.withVcpu(hostResources.vcpu() / 2)); tester.clock().advance(Duration.ofDays(2)); tester.addQueryRateMeasurements(application1, cluster1.id(), 100, t -> t == 0 ? 20.0 : 10.0); // Query traffic only - tester.addMemMeasurements(0.01f, 0.95f, 120, application1); + tester.addMemMeasurements(0.02f, 0.95f, 120, application1); tester.assertResources("Scaling down", - 7, 1, 2.5, 2.0, 79.2, + 6, 1, 2.9, 4.0, 95.0, tester.autoscale(application1, cluster1.id(), min, max).target()); } @@ -500,7 +500,7 @@ public class AutoscalingTest { tester.clock().advance(Duration.ofMinutes(-10 * 5)); tester.addQueryRateMeasurements(application1, cluster1.id(), 10, t -> t == 0 ? 20.0 : 10.0); // Query traffic only tester.assertResources("Scaling down", - 8, 1, 1.0, 2.2, 67.9, + 6, 1, 1.4, 4.0, 95.0, tester.autoscale(application1, cluster1.id(), min, max).target()); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/lb/SharedLoadBalancerServiceTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/lb/SharedLoadBalancerServiceTest.java index bcde89152bb..28b40bb7642 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/lb/SharedLoadBalancerServiceTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/lb/SharedLoadBalancerServiceTest.java @@ -19,7 +19,7 @@ import static org.junit.Assert.assertEquals; public class SharedLoadBalancerServiceTest { private final ProvisioningTester tester = new ProvisioningTester.Builder().build(); - private final SharedLoadBalancerService loadBalancerService = new SharedLoadBalancerService(tester.nodeRepository()); + private final SharedLoadBalancerService loadBalancerService = new SharedLoadBalancerService(tester.nodeRepository(), "vip.example.com"); private final ApplicationId applicationId = ApplicationId.from("tenant1", "application1", "default"); private final ClusterSpec.Id clusterId = ClusterSpec.Id.from("qrs1"); private final Set<Real> reals = Set.of( @@ -32,7 +32,7 @@ public class SharedLoadBalancerServiceTest { tester.makeReadyNodes(2, "default", NodeType.proxy); var lb = loadBalancerService.create(new LoadBalancerSpec(applicationId, clusterId, reals), false); - assertEquals(HostName.from("host-1.yahoo.com"), lb.hostname()); + assertEquals(HostName.from("vip.example.com"), lb.hostname()); assertEquals(Optional.empty(), lb.dnsZone()); assertEquals(Set.of("127.0.0.1/32", "127.0.0.2/32", "::1/128", "::2/128"), lb.networks()); assertEquals(Set.of(4080, 4443), lb.ports()); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainerTest.java index 623e9ea51a7..03b41412896 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainerTest.java @@ -85,7 +85,7 @@ public class ScalingSuggestionsMaintainerTest { addMeasurements(0.10f, 0.10f, 0.10f, 0, 500, app1, tester.nodeRepository()); maintainer.maintain(); assertEquals("Peak suggestion has been outdated", - "7 nodes with [vcpu: 1.2, memory: 2.0 Gb, disk 10.0 Gb, bandwidth: 0.1 Gbps]", + "5 nodes with [vcpu: 1.8, memory: 4.0 Gb, disk 10.0 Gb, bandwidth: 0.1 Gbps]", suggestionOf(app1, cluster1, tester).get().resources().toString()); assertTrue(shouldSuggest(app1, cluster1, tester)); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java index 475e4913381..20546cc5bd9 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java @@ -534,10 +534,10 @@ public class ProvisioningTest { tester.makeReadyHosts(10, defaultResources).activateTenantHosts(); try { prepare(application, 2, 2, 3, 3, - new NodeResources(2, 1, 10, 2), tester); + new NodeResources(2, 2, 10, 2), tester); } catch (IllegalArgumentException e) { - assertEquals("container cluster 'container0': Min memoryGb size is 1.00 Gb but must be at least 2.00 Gb", e.getMessage()); + assertEquals("container cluster 'container0': Min memoryGb size is 2.00 Gb but must be at least 4.00 Gb", e.getMessage()); } } diff --git a/parent/pom.xml b/parent/pom.xml index 4edb9278c83..aea586f3949 100644 --- a/parent/pom.xml +++ b/parent/pom.xml @@ -78,8 +78,7 @@ <jdkToolchain> <version>11</version> </jdkToolchain> - <source>11</source> - <target>11</target> + <release>11</release> <showWarnings>true</showWarnings> <optimize>true</optimize> <showDeprecation>false</showDeprecation> diff --git a/searchcore/src/tests/proton/bucketdb/bucketdb/bucketdb_test.cpp b/searchcore/src/tests/proton/bucketdb/bucketdb/bucketdb_test.cpp index 94300d4abac..8f10f4b8045 100644 --- a/searchcore/src/tests/proton/bucketdb/bucketdb/bucketdb_test.cpp +++ b/searchcore/src/tests/proton/bucketdb/bucketdb/bucketdb_test.cpp @@ -1,8 +1,11 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/document/base/documentid.h> #include <vespa/searchcore/proton/bucketdb/bucket_db_explorer.h> #include <vespa/searchcore/proton/bucketdb/bucketdb.h> +#include <vespa/searchcore/proton/bucketdb/remove_batch_entry.h> #include <vespa/vespalib/data/slime/slime.h> #include <vespa/vespalib/stllike/asciistream.h> +#include <vespa/vespalib/util/stringfmt.h> #include <vespa/vespalib/testkit/testapp.h> #include <vespa/log/log.h> @@ -29,6 +32,24 @@ constexpr uint32_t DOCSIZE_2(10000u); typedef BucketInfo::ReadyState RS; typedef SubDbType SDT; +namespace { + +constexpr uint32_t bucket_bits = 16; + +uint32_t num_buckets() { return (1u << bucket_bits); } + +BucketId make_bucket_id(uint32_t n) { + return BucketId(bucket_bits, n & (num_buckets() - 1)); +} + +GlobalId make_gid(uint32_t n, uint32_t i) +{ + DocumentId id(vespalib::make_string("id::test:n=%u:%u", n & (num_buckets() - 1), i)); + return id.getGlobalId(); +} + +} + void assertDocCount(uint32_t ready, uint32_t notReady, @@ -70,12 +91,20 @@ struct Fixture Fixture() : _db() {} + void add(const GlobalId &gid, const Timestamp ×tamp, uint32_t docSize, SubDbType subDbType) { + BucketId bucket(bucket_bits, gid.convertToBucketId().getRawId()); + _db.add(gid, bucket, timestamp, docSize, subDbType); + } const BucketState &add(const Timestamp ×tamp, uint32_t docSize, SubDbType subDbType) { return _db.add(GID_1, BUCKET_1, timestamp, docSize, subDbType); } const BucketState &add(const Timestamp ×tamp, SubDbType subDbType) { return add(timestamp, DOCSIZE_1, subDbType); } + void remove(const GlobalId& gid, const Timestamp ×tamp, uint32_t docSize, SubDbType subDbType) { + BucketId bucket(bucket_bits, gid.convertToBucketId().getRawId()); + _db.remove(gid, bucket, timestamp, docSize, subDbType); + } BucketState remove(const Timestamp ×tamp, uint32_t docSize, SubDbType subDbType) { _db.remove(GID_1, BUCKET_1, timestamp, docSize, subDbType); return get(); @@ -83,8 +112,14 @@ struct Fixture BucketState remove(const Timestamp ×tamp, SubDbType subDbType) { return remove(timestamp, DOCSIZE_1, subDbType); } + void remove_batch(const std::vector<RemoveBatchEntry> &removed, SubDbType sub_db_type) { + _db.remove_batch(removed, sub_db_type); + } + BucketState get(BucketId bucket_id) const { + return _db.get(bucket_id); + } BucketState get() const { - return _db.get(BUCKET_1); + return get(BUCKET_1); } BucketChecksum getChecksum(const Timestamp ×tamp, uint32_t docSize, SubDbType subDbType) { BucketDB db; @@ -181,6 +216,36 @@ TEST_F("require that bucket checksum ignores document sizes", Fixture) EXPECT_EQUAL(state1.getChecksum(), state2.getChecksum()); } +TEST_F("require that remove batch works", Fixture) +{ + f.add(make_gid(4, 1), Timestamp(10), 100, SDT::READY); + f.add(make_gid(4, 2), Timestamp(11), 104, SDT::READY); + f.add(make_gid(4, 3), Timestamp(12), 102, SDT::READY); + f.add(make_gid(5, 4), Timestamp(13), 200, SDT::READY); + f.add(make_gid(5, 5), Timestamp(14), 270, SDT::READY); + f.add(make_gid(5, 6), Timestamp(15), 1000, SDT::READY); + auto state1 = f.get(make_bucket_id(4)); + EXPECT_EQUAL(306u, state1.getReadyDocSizes()); + EXPECT_EQUAL(3u, state1.getReadyCount()); + auto state2 = f.get(make_bucket_id(5)); + EXPECT_EQUAL(1470u, state2.getReadyDocSizes()); + EXPECT_EQUAL(3u, state2.getReadyCount()); + std::vector<RemoveBatchEntry> removed; + removed.emplace_back(make_gid(4, 1), make_bucket_id(4), Timestamp(10), 100); + removed.emplace_back(make_gid(4, 3), make_bucket_id(4), Timestamp(12), 102); + removed.emplace_back(make_gid(5, 5), make_bucket_id(5), Timestamp(14), 270); + f.remove_batch(removed, SDT::READY); + auto state3 = f.get(make_bucket_id(4)); + EXPECT_EQUAL(104u, state3.getReadyDocSizes()); + EXPECT_EQUAL(1u, state3.getReadyCount()); + auto state4 = f.get(make_bucket_id(5)); + EXPECT_EQUAL(1200u, state4.getReadyDocSizes()); + EXPECT_EQUAL(2u, state4.getReadyCount()); + f.remove(make_gid(4, 2), Timestamp(11), 104, SDT::READY); + f.remove(make_gid(5, 4), Timestamp(13), 200, SDT::READY); + f.remove(make_gid(5, 6), Timestamp(15), 1000, SDT::READY); +} + TEST("require that bucket db can be explored") { BucketDBOwner db; diff --git a/searchcore/src/tests/proton/docsummary/docsummary.cpp b/searchcore/src/tests/proton/docsummary/docsummary.cpp index 27574cb68df..1808aed5d0b 100644 --- a/searchcore/src/tests/proton/docsummary/docsummary.cpp +++ b/searchcore/src/tests/proton/docsummary/docsummary.cpp @@ -250,6 +250,7 @@ public: PutRes putRes(dms.put(docId.getGlobalId(), BucketFactory::getBucketId(docId), Timestamp(0u), docSize, lid, 0u)); LOG_ASSERT(putRes.ok()); + dms.commit(CommitParam(0u)); uint64_t serialNum = _ddb->getFeedHandler().inc_serial_num(); _aw->put(serialNum, doc, lid, std::shared_ptr<IDestructorCallback>()); _aw->forceCommit(serialNum, std::shared_ptr<IDestructorCallback>()); diff --git a/searchcore/src/tests/proton/documentdb/feedview/feedview_test.cpp b/searchcore/src/tests/proton/documentdb/feedview/feedview_test.cpp index 1b4848f9c8b..97faa81b48a 100644 --- a/searchcore/src/tests/proton/documentdb/feedview/feedview_test.cpp +++ b/searchcore/src/tests/proton/documentdb/feedview/feedview_test.cpp @@ -144,7 +144,7 @@ struct MyIndexWriter : public test::MockIndexWriter _wantedLidLimit(0), _tracer(tracer) {} - void put(SerialNum serialNum, const document::Document &doc, const DocumentIdT lid) override { + void put(SerialNum serialNum, const document::Document &doc, const DocumentIdT lid, OnWriteDoneType) override { (void) doc; _tracer.tracePut(indexAdapterTypeName, serialNum, lid); } diff --git a/searchcore/src/tests/proton/documentmetastore/documentmetastore_test.cpp b/searchcore/src/tests/proton/documentmetastore/documentmetastore_test.cpp index 4b4c287b97c..d201b02a61d 100644 --- a/searchcore/src/tests/proton/documentmetastore/documentmetastore_test.cpp +++ b/searchcore/src/tests/proton/documentmetastore/documentmetastore_test.cpp @@ -109,6 +109,7 @@ assertPut(const BucketId &bucketId, Result inspect = dms.inspect(gid, 0u); uint32_t docSize = 1; PutRes putRes = dms.put(gid, bucketId, timestamp, docSize, inspect.getLid(), 0u); + dms.commit(); EXPECT_TRUE(putRes.ok()); EXPECT_EQ(lid, putRes.getLid()); } @@ -197,7 +198,7 @@ assertSearchResult(const SimpleResult &exp, const DocumentMetaStore &dms, bool strict, uint32_t docIdLimit = 100) { AttributeVector::SearchContext::UP sc = - dms.getSearch(QueryTermSimple::UP(new QueryTermSimple(term, termType)), SearchContextParams()); + dms.getSearch(std::make_unique<QueryTermSimple>(term, termType), SearchContextParams()); TermFieldMatchData tfmd; SearchIterator::UP sb = sc->createIterator(&tfmd, strict); SimpleResult act; @@ -249,6 +250,7 @@ addGid(DocumentMetaStore &dms, const GlobalId &gid, const BucketId &bid, Timesta Result inspect = dms.inspect(gid, 0u); PutRes putRes; EXPECT_TRUE((putRes = dms.put(gid, bid, timestamp, docSize, inspect.getLid(), 0u)).ok()); + dms.commit(); return putRes.getLid(); } @@ -265,6 +267,7 @@ putGid(DocumentMetaStore &dms, const GlobalId &gid, uint32_t lid, Timestamp time BucketId bid(minNumBits, gid.convertToBucketId().getRawId()); uint32_t docSize = 1; EXPECT_TRUE(dms.put(gid, bid, timestamp, docSize, lid, 0u).ok()); + dms.commit(); } TEST(DocumentMetaStoreTest, control_meta_data_sizeof) { @@ -285,6 +288,7 @@ TEST(DocumentMetaStoreTest, control_meta_data_sizeof) { assertPut(bucketId2, time2, 2, gid2, dms); EXPECT_EQ(bucketId2, dms.getBucketOf(guard, 2)); EXPECT_TRUE(dms.remove(1, 0u)); + dms.commit(); EXPECT_EQ(BucketId(), dms.getBucketOf(guard, 1)); EXPECT_EQ(bucketId2, dms.getBucketOf(guard, 2)); } @@ -352,8 +356,7 @@ TEST(DocumentMetaStore, gids_can_be_cleared) TEST(DocumentMetaStore, generation_handling_is_working) { - AttributeVector::SP av(new DocumentMetaStore(createBucketDB())); - DocumentMetaStore * dms = static_cast<DocumentMetaStore *>(av.get()); + auto dms = std::make_shared<DocumentMetaStore>(createBucketDB()); dms->constructFreeList(); const GenerationHandler & gh = dms->getGenerationHandler(); EXPECT_EQ(1u, gh.getCurrentGeneration()); @@ -361,10 +364,10 @@ TEST(DocumentMetaStore, generation_handling_is_working) EXPECT_EQ(2u, gh.getCurrentGeneration()); EXPECT_EQ(0u, gh.getGenerationRefCount()); { - AttributeGuard g1(av); + AttributeGuard g1(dms); EXPECT_EQ(1u, gh.getGenerationRefCount()); { - AttributeGuard g2(av); + AttributeGuard g2(dms); EXPECT_EQ(2u, gh.getGenerationRefCount()); } EXPECT_EQ(1u, gh.getGenerationRefCount()); @@ -372,7 +375,7 @@ TEST(DocumentMetaStore, generation_handling_is_working) EXPECT_EQ(0u, gh.getGenerationRefCount()); dms->remove(1, 0u); dms->removeComplete(1); - EXPECT_EQ(4u, gh.getCurrentGeneration()); + EXPECT_EQ(3u, gh.getCurrentGeneration()); } TEST(DocumentMetaStoreTest, basic_free_list_is_working) @@ -494,8 +497,7 @@ TEST(DocumentMetaStoreTest, lid_state_vector_resizing_is_working) TEST(DocumentMetaStoreTest, lid_and_gid_space_is_reused) { - AttributeVector::SP av(new DocumentMetaStore(createBucketDB())); - DocumentMetaStore * dms = static_cast<DocumentMetaStore *>(av.get()); + auto dms = std::make_shared<DocumentMetaStore>(createBucketDB()); dms->constructFreeList(); EXPECT_EQ(1u, dms->getNumDocs()); EXPECT_EQ(0u, dms->getNumUsedLids()); @@ -515,7 +517,7 @@ TEST(DocumentMetaStoreTest, lid_and_gid_space_is_reused) EXPECT_EQ(2u, dms->getNumUsedLids()); // reuse assertGid(gid3, 2, *dms); { - AttributeGuard g1(av); // guard on gen 5 + AttributeGuard g1(dms); // guard on gen 5 dms->remove(2, 0u); dms->removeComplete(2); EXPECT_EQ(3u, dms->getNumDocs()); @@ -704,13 +706,16 @@ TEST(DocumentMetaStoreTest, can_put_and_remove_before_free_list_construct) { DocumentMetaStore dms(createBucketDB()); EXPECT_TRUE(dms.put(gid4, bucketId4, time4, docSize4, 4, 0u).ok()); + dms.commit(); assertLid(4, gid4, dms); assertGid(gid4, 4, dms); EXPECT_EQ(1u, dms.getNumUsedLids()); EXPECT_EQ(5u, dms.getNumDocs()); EXPECT_TRUE(dms.put(gid1, bucketId1, time1, docSize1, 1, 0u).ok()); + dms.commit(); // already there, nothing changes EXPECT_TRUE(dms.put(gid1, bucketId1, time1, docSize1, 1, 0u).ok()); + dms.commit(); assertLid(1, gid1, dms); assertGid(gid1, 1, dms); EXPECT_EQ(2u, dms.getNumUsedLids()); @@ -725,6 +730,7 @@ TEST(DocumentMetaStoreTest, can_put_and_remove_before_free_list_construct) EXPECT_EQ(2u, dms.getNumUsedLids()); EXPECT_EQ(5u, dms.getNumDocs()); EXPECT_TRUE(dms.remove(4, 0u)); // -> goes to free list. cleared and re-applied in constructFreeList(). + dms.commit(); uint32_t lid; GlobalId gid; EXPECT_TRUE(!dms.getLid(gid4, lid)); @@ -1189,6 +1195,7 @@ struct SplitAndJoinFixture : public SplitAndJoinEmptyFixture { docSize, gids[i].lid, 0u).ok()); } + dms.commit(); } void insertGids2() { uint32_t docSize = 1; @@ -1197,6 +1204,7 @@ struct SplitAndJoinFixture : public SplitAndJoinEmptyFixture { docSize, gids[i].lid, 0u).ok()); } + dms.commit(); } void @@ -1208,6 +1216,7 @@ struct SplitAndJoinFixture : public SplitAndJoinEmptyFixture { BucketId b(g.bid3 == alt ? g.bid2 : g.bid1); EXPECT_TRUE(dms.put(g.gid, b, Timestamp(0), docSize, g.lid, 0u).ok()); } + dms.commit(); } void @@ -1219,6 +1228,7 @@ struct SplitAndJoinFixture : public SplitAndJoinEmptyFixture { BucketId b(g.bid3 == alt ? g.bid1 : g.bid2); EXPECT_TRUE(dms.put(g.gid, b, Timestamp(0), docSize, g.lid, 0u).ok()); } + dms.commit(); } }; @@ -1786,6 +1796,7 @@ TEST(DocumentMetaStoreTest, move_works) EXPECT_EQ(gid1, gid); EXPECT_EQ(2u, lid); EXPECT_TRUE(dms.remove(1, 0u)); + dms.commit(); EXPECT_FALSE(dms.getGid(1u, gid)); EXPECT_FALSE(dms.getGidEvenIfMoved(1u, gid)); EXPECT_TRUE(dms.getGid(2u, gid)); @@ -1795,6 +1806,7 @@ TEST(DocumentMetaStoreTest, move_works) EXPECT_TRUE(dms.getGid(2u, gid)); EXPECT_EQ(1u, dms.getNumUsedLids()); dms.move(2u, 1u, 0u); + dms.commit(); EXPECT_TRUE(dms.getGid(1u, gid)); EXPECT_FALSE(dms.getGid(2u, gid)); EXPECT_TRUE(dms.getGidEvenIfMoved(2u, gid)); @@ -1864,7 +1876,7 @@ TEST(DocumentMetaStoreTest, shrink_works) TEST(DocumentMetaStoreTest, shrink_via_flush_target_works) { - DocumentMetaStore::SP dms(new DocumentMetaStore(createBucketDB())); + auto dms = std::make_shared<DocumentMetaStore>(createBucketDB()); dms->constructFreeList(); TuneFileAttributes tuneFileAttributes; DummyFileHeaderContext fileHeaderContext; @@ -1888,7 +1900,7 @@ TEST(DocumentMetaStoreTest, shrink_via_flush_target_works) assertLidSpace(10, shrinkTarget, shrinkTarget - 1, true, false, *dms); EXPECT_EQ(ft->getApproxMemoryGain().getBefore(), ft->getApproxMemoryGain().getAfter()); - AttributeGuard::UP g(new AttributeGuard(dms)); + auto g = std::make_shared<AttributeGuard>(dms); dms->holdUnblockShrinkLidSpace(); assertLidSpace(10, shrinkTarget, shrinkTarget - 1, true, false, *dms); @@ -2092,6 +2104,7 @@ TEST(DocumentMetaStoreTest, call_to_remove_is_notified) addLid(dms, 1); dms.remove(1, 0u); + dms.commit(); EXPECT_EQ(1, listener->remove_cnt); } diff --git a/searchcore/src/tests/proton/feed_and_search/feed_and_search.cpp b/searchcore/src/tests/proton/feed_and_search/feed_and_search.cpp index 8314fa6bfb8..855b31310a3 100644 --- a/searchcore/src/tests/proton/feed_and_search/feed_and_search.cpp +++ b/searchcore/src/tests/proton/feed_and_search/feed_and_search.cpp @@ -168,10 +168,10 @@ void Test::requireThatMemoryIndexCanBeDumpedAndSearched() { DocBuilder doc_builder(schema); Document::UP doc = buildDocument(doc_builder, doc_id1, word1); - memory_index.insertDocument(doc_id1, *doc.get()); + memory_index.insertDocument(doc_id1, *doc, {}); - doc = buildDocument(doc_builder, doc_id2, word2); - memory_index.insertDocument(doc_id2, *doc.get()); + auto doc2 = buildDocument(doc_builder, doc_id2, word2); + memory_index.insertDocument(doc_id2, *doc2, {}); commit_memory_index_and_wait(memory_index); testSearch(memory_index, word1, doc_id1); diff --git a/searchcore/src/tests/proton/index/fusionrunner_test.cpp b/searchcore/src/tests/proton/index/fusionrunner_test.cpp index 736bc4bae96..ae85211fe24 100644 --- a/searchcore/src/tests/proton/index/fusionrunner_test.cpp +++ b/searchcore/src/tests/proton/index/fusionrunner_test.cpp @@ -157,7 +157,7 @@ Document::UP buildDocument(DocBuilder & doc_builder, int id, const string &word) void addDocument(DocBuilder & doc_builder, MemoryIndex &index, ISourceSelector &selector, uint8_t index_id, uint32_t docid, const string &word) { Document::UP doc = buildDocument(doc_builder, docid, word); - index.insertDocument(docid, *doc); + index.insertDocument(docid, *doc, {}); vespalib::Gate gate; index.commit(std::make_shared<vespalib::GateCallback>(gate)); selector.setSource(docid, index_id); diff --git a/searchcore/src/tests/proton/index/index_writer/index_writer_test.cpp b/searchcore/src/tests/proton/index/index_writer/index_writer_test.cpp index ac17c17892c..62a691d72e6 100644 --- a/searchcore/src/tests/proton/index/index_writer/index_writer_test.cpp +++ b/searchcore/src/tests/proton/index/index_writer/index_writer_test.cpp @@ -49,7 +49,7 @@ struct MyIndexManager : public test::MockIndexManager return toString(removes[lid]); } // Implements IIndexManager - void putDocument(uint32_t lid, const Document &, SerialNum serialNum) override { + void putDocument(uint32_t lid, const Document &, SerialNum serialNum, OnWriteDoneType) override { puts[lid].push_back(serialNum); } void removeDocuments(LidVector lids, SerialNum serialNum) override { @@ -94,7 +94,7 @@ struct Fixture return builder.endDocument(); } void put(SerialNum serialNum, const search::DocumentIdT lid) { - iw.put(serialNum, *dummyDoc, lid); + iw.put(serialNum, *dummyDoc, lid, {}); iw.commit(serialNum, std::shared_ptr<IDestructorCallback>()); } void remove(SerialNum serialNum, const search::DocumentIdT lid) { diff --git a/searchcore/src/tests/proton/index/indexmanager_test.cpp b/searchcore/src/tests/proton/index/indexmanager_test.cpp index 8e41323e461..4c442d38443 100644 --- a/searchcore/src/tests/proton/index/indexmanager_test.cpp +++ b/searchcore/src/tests/proton/index/indexmanager_test.cpp @@ -192,7 +192,7 @@ IndexManagerTest::addDocument(uint32_t id) Document::UP doc = buildDocument(_builder, id, "foo"); SerialNum serialNum = ++_serial_num; vespalib::Gate gate; - runAsIndex([&]() { _index_manager->putDocument(id, *doc, serialNum); + runAsIndex([&]() { _index_manager->putDocument(id, *doc, serialNum, {}); _index_manager->commit(serialNum, std::make_shared<vespalib::GateCallback>(gate)); }); gate.await(); @@ -416,7 +416,7 @@ TEST_F(IndexManagerTest, require_that_flush_stats_are_calculated) EXPECT_EQ(0u, _index_manager->getMaintainer().getFlushStats().cpu_time_required); Document::UP doc = addDocument(docid); - inverter.invertDocument(docid, *doc); + inverter.invertDocument(docid, *doc, {}); push_documents_and_wait(inverter); index_size = fic.getMemoryUsage().allocatedBytes() - fixed_index_size; @@ -431,9 +431,9 @@ TEST_F(IndexManagerTest, require_that_flush_stats_are_calculated) _index_manager->getMaintainer().getFlushStats().cpu_time_required); doc = addDocument(docid + 10); - inverter.invertDocument(docid + 10, *doc); - doc = addDocument(docid + 100); - inverter.invertDocument(docid + 100, *doc); + inverter.invertDocument(docid + 10, *doc, {}); + auto doc100 = addDocument(docid + 100); + inverter.invertDocument(docid + 100, *doc100, {}); push_documents_and_wait(inverter); index_size = fic.getMemoryUsage().allocatedBytes() - fixed_index_size; /// Must account for both docid 0 being reserved and the extra after. diff --git a/searchcore/src/tests/proton/matching/partial_result/partial_result_test.cpp b/searchcore/src/tests/proton/matching/partial_result/partial_result_test.cpp index 139288f6b6f..1fadd3993ff 100644 --- a/searchcore/src/tests/proton/matching/partial_result/partial_result_test.cpp +++ b/searchcore/src/tests/proton/matching/partial_result/partial_result_test.cpp @@ -23,7 +23,7 @@ void checkMerge(const std::vector<double> &a, const std::vector<double> &b, EXPECT_EQUAL(a.size() + b.size(), res_a.totalHits()); ASSERT_EQUAL(expect.size(), res_a.size()); for (size_t i = 0; i < expect.size(); ++i) { - EXPECT_EQUAL(expect[i], res_a.hit(i)._rankValue); + EXPECT_EQUAL(expect[i], res_a.hit(i).getRank()); } } @@ -70,10 +70,10 @@ TEST("require that partial results can be created without sort data") { res.totalHits(1000); EXPECT_EQUAL(1000u, res.totalHits()); ASSERT_EQUAL(2u, res.size()); - EXPECT_EQUAL(1u, res.hit(0)._docId); - EXPECT_EQUAL(10.0, res.hit(0)._rankValue); - EXPECT_EQUAL(2u, res.hit(1)._docId); - EXPECT_EQUAL(5.0, res.hit(1)._rankValue); + EXPECT_EQUAL(1u, res.hit(0).getDocId()); + EXPECT_EQUAL(10.0, res.hit(0).getRank()); + EXPECT_EQUAL(2u, res.hit(1).getDocId()); + EXPECT_EQUAL(5.0, res.hit(1).getRank()); } TEST("require that partial results can be created with sort data") { @@ -90,12 +90,12 @@ TEST("require that partial results can be created with sort data") { res.totalHits(1000); EXPECT_EQUAL(1000u, res.totalHits()); ASSERT_EQUAL(2u, res.size()); - EXPECT_EQUAL(1u, res.hit(0)._docId); - EXPECT_EQUAL(10.0, res.hit(0)._rankValue); + EXPECT_EQUAL(1u, res.hit(0).getDocId()); + EXPECT_EQUAL(10.0, res.hit(0).getRank()); EXPECT_EQUAL(str1.data(), res.sortData(0).first); EXPECT_EQUAL(str1.size(), res.sortData(0).second); - EXPECT_EQUAL(2u, res.hit(1)._docId); - EXPECT_EQUAL(5.0, res.hit(1)._rankValue); + EXPECT_EQUAL(2u, res.hit(1).getDocId()); + EXPECT_EQUAL(5.0, res.hit(1).getRank()); EXPECT_EQUAL(str2.data(), res.sortData(1).first); EXPECT_EQUAL(str2.size(), res.sortData(1).second); } @@ -133,10 +133,10 @@ TEST("require that lower docid is preferred when sorting on rank") { res_c.add(search::RankedHit(1, 1.0)); res_a.merge(res_b); ASSERT_EQUAL(1u, res_a.size()); - EXPECT_EQUAL(2u, res_a.hit(0)._docId); + EXPECT_EQUAL(2u, res_a.hit(0).getDocId()); res_a.merge(res_c); ASSERT_EQUAL(1u, res_a.size()); - EXPECT_EQUAL(1u, res_a.hit(0)._docId); + EXPECT_EQUAL(1u, res_a.hit(0).getDocId()); } TEST("require that lower docid is preferred when using sortspec") { @@ -149,10 +149,10 @@ TEST("require that lower docid is preferred when using sortspec") { res_c.add(search::RankedHit(1, 1.0), PartialResult::SortRef(foo.data(), foo.size())); res_a.merge(res_b); ASSERT_EQUAL(1u, res_a.size()); - EXPECT_EQUAL(2u, res_a.hit(0)._docId); + EXPECT_EQUAL(2u, res_a.hit(0).getDocId()); res_a.merge(res_c); ASSERT_EQUAL(1u, res_a.size()); - EXPECT_EQUAL(1u, res_a.hit(0)._docId); + EXPECT_EQUAL(1u, res_a.hit(0).getDocId()); } TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchcore/src/tests/proton/server/documentretriever_test.cpp b/searchcore/src/tests/proton/server/documentretriever_test.cpp index c0035f28fd2..aed9f44799a 100644 --- a/searchcore/src/tests/proton/server/documentretriever_test.cpp +++ b/searchcore/src/tests/proton/server/documentretriever_test.cpp @@ -343,6 +343,7 @@ struct Fixture { Result inspect = meta_store.get().inspect(gid, 0u); uint32_t docSize = 1; Result putRes(meta_store.get().put(gid, bucket_id, timestamp, docSize, inspect.getLid(), 0u)); + meta_store.get().commit(search::CommitParam(0)); lid = putRes.getLid(); ASSERT_TRUE(putRes.ok()); schema::CollectionType ct = schema::CollectionType::SINGLE; diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.cpp index 57ab149404d..587ec8dda1f 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.cpp @@ -63,7 +63,7 @@ AttributeWriter::WriteField::WriteField(AttributeVector &attribute) AttributeWriter::WriteField::~WriteField() = default; void -AttributeWriter::WriteField::buildFieldPath(const DocumentType &docType) +AttributeWriter::WriteField::buildFieldPath(const DocumentType &docType) const { const vespalib::string &name = _attribute.getName(); FieldPath fp; @@ -80,6 +80,8 @@ AttributeWriter::WriteField::buildFieldPath(const DocumentType &docType) AttributeWriter::WriteContext::WriteContext(ExecutorId executorId) noexcept : _executorId(executorId), _fields(), + _data_type(nullptr), + _two_phase_put_field_path(), _hasStructFieldAttribute(false), _use_two_phase_put(false) { @@ -108,10 +110,18 @@ AttributeWriter::WriteContext::add(AttributeVector &attr) } void -AttributeWriter::WriteContext::buildFieldPaths(const DocumentType &docType) -{ - for (auto &field : _fields) { - field.buildFieldPath(docType); +AttributeWriter::WriteContext::consider_build_field_paths(const Document& doc) const +{ + auto data_type = doc.getDataType(); + if (_data_type != data_type) { + _data_type = data_type; + auto& doc_type = doc.getType(); + for (auto &field : _fields) { + field.buildFieldPath(doc_type); + } + if (_use_two_phase_put) { + _two_phase_put_field_path = std::make_shared<const FieldPath>(_fields[0].getFieldPath()); + } } } @@ -154,18 +164,43 @@ applyPutToAttribute(SerialNum serialNum, const FieldValue::UP &fieldValue, Docum attr.commitIfChangeVectorTooLarge(); } +class FieldValueAndPrepareResult { + std::unique_ptr<const FieldValue> _field_value; + std::unique_ptr<PrepareResult> _prepare_result; +public: + FieldValueAndPrepareResult(std::unique_ptr<const FieldValue> field_value, + std::unique_ptr<PrepareResult> prepare_result) + : _field_value(std::move(field_value)), + _prepare_result(std::move(prepare_result)) + { + } + FieldValueAndPrepareResult() + : _field_value(), + _prepare_result() + { + } + ~FieldValueAndPrepareResult(); + FieldValueAndPrepareResult(FieldValueAndPrepareResult&&); + const std::unique_ptr<const FieldValue>& get_field_value() const noexcept { return _field_value; } + std::unique_ptr<PrepareResult> steal_prepare_result() { return std::move(_prepare_result); } +}; + +FieldValueAndPrepareResult::~FieldValueAndPrepareResult() = default; + +FieldValueAndPrepareResult::FieldValueAndPrepareResult(FieldValueAndPrepareResult&&) = default; + void complete_put_to_attribute(SerialNum serial_num, uint32_t docid, AttributeVector& attr, - const FieldValue::SP& field_value, - std::future<std::unique_ptr<PrepareResult>>& result_future, + std::future<FieldValueAndPrepareResult> result_future, AttributeWriter::OnWriteDoneType) { ensureLidSpace(serial_num, docid, attr); - if (field_value.get()) { - auto result = result_future.get(); - AttributeUpdater::complete_set_value(attr, docid, *field_value, std::move(result)); + auto result = result_future.get(); + auto& field_value = result.get_field_value(); + if (field_value) { + AttributeUpdater::complete_set_value(attr, docid, *field_value, result.steal_prepare_result()); } else { attr.clearDoc(docid); } @@ -303,31 +338,21 @@ class PutTask : public vespalib::Executor::Task const uint32_t _lid; const bool _allAttributes; std::remove_reference_t<AttributeWriter::OnWriteDoneType> _onWriteDone; - std::shared_ptr<DocumentFieldExtractor> _fieldExtractor; - std::vector<FieldValue::UP> _fieldValues; + const Document& _doc; public: - PutTask(const AttributeWriter::WriteContext &wc, SerialNum serialNum, std::shared_ptr<DocumentFieldExtractor> fieldExtractor, uint32_t lid, bool allAttributes, AttributeWriter::OnWriteDoneType onWriteDone); + PutTask(const AttributeWriter::WriteContext &wc, SerialNum serialNum, const Document& doc, uint32_t lid, bool allAttributes, AttributeWriter::OnWriteDoneType onWriteDone); ~PutTask() override; void run() override; }; -PutTask::PutTask(const AttributeWriter::WriteContext &wc, SerialNum serialNum, std::shared_ptr<DocumentFieldExtractor> fieldExtractor, uint32_t lid, bool allAttributes, AttributeWriter::OnWriteDoneType onWriteDone) +PutTask::PutTask(const AttributeWriter::WriteContext &wc, SerialNum serialNum, const Document& doc, uint32_t lid, bool allAttributes, AttributeWriter::OnWriteDoneType onWriteDone) : _wc(wc), _serialNum(serialNum), _lid(lid), _allAttributes(allAttributes), _onWriteDone(onWriteDone), - _fieldExtractor(std::move(fieldExtractor)), - _fieldValues() + _doc(doc) { - const auto &fields = _wc.getFields(); - _fieldValues.reserve(fields.size()); - for (const auto &field : fields) { - if (_allAttributes || field.isStructFieldAttribute()) { - FieldValue::UP fv = _fieldExtractor->getFieldValue(field.getFieldPath()); - _fieldValues.emplace_back(std::move(fv)); - } - } } PutTask::~PutTask() = default; @@ -335,33 +360,35 @@ PutTask::~PutTask() = default; void PutTask::run() { - uint32_t fieldId = 0; + _wc.consider_build_field_paths(_doc); + DocumentFieldExtractor field_extractor(_doc); const auto &fields = _wc.getFields(); for (auto field : fields) { if (_allAttributes || field.isStructFieldAttribute()) { AttributeVector &attr = field.getAttribute(); if (attr.getStatus().getLastSyncToken() < _serialNum) { - applyPutToAttribute(_serialNum, _fieldValues[fieldId], _lid, attr, _onWriteDone); + auto fv = field_extractor.getFieldValue(field.getFieldPath()); + applyPutToAttribute(_serialNum, fv, _lid, attr, _onWriteDone); } - ++fieldId; } } } - class PreparePutTask : public vespalib::Executor::Task { private: const SerialNum _serial_num; const uint32_t _docid; AttributeVector& _attr; - FieldValue::SP _field_value; - std::promise<std::unique_ptr<PrepareResult>> _result_promise; + std::shared_ptr<const FieldPath> _field_path; + const Document* const _doc; + std::unique_ptr<FieldValue> _field_value; + std::promise<FieldValueAndPrepareResult> _result_promise; public: PreparePutTask(SerialNum serial_num, uint32_t docid, - const AttributeWriter::WriteField& field, - std::shared_ptr<DocumentFieldExtractor> field_extractor); + const AttributeWriter::WriteContext& wc, + const Document& doc); PreparePutTask(SerialNum serial_num, uint32_t docid, AttributeVector& attr, @@ -371,25 +398,23 @@ public: SerialNum serial_num() const { return _serial_num; } uint32_t docid() const { return _docid; } AttributeVector& attr() { return _attr; } - FieldValue::SP field_value() { return _field_value; } - std::future<std::unique_ptr<PrepareResult>> result_future() { + std::future<FieldValueAndPrepareResult> result_future() { return _result_promise.get_future(); } }; PreparePutTask::PreparePutTask(SerialNum serial_num, uint32_t docid, - const AttributeWriter::WriteField& field, - std::shared_ptr<DocumentFieldExtractor> field_extractor) + const AttributeWriter::WriteContext& wc, + const Document& doc) : _serial_num(serial_num), _docid(docid), - _attr(field.getAttribute()), + _attr(wc.getFields()[0].getAttribute()), + _field_path(wc.get_two_phase_put_field_path()), + _doc(&doc), _field_value(), _result_promise() { - // Note: No need to store the field extractor as we are not extracting struct fields. - auto value = field_extractor->getFieldValue(field.getFieldPath()); - _field_value.reset(value.release()); } PreparePutTask::PreparePutTask(SerialNum serial_num, @@ -399,6 +424,8 @@ PreparePutTask::PreparePutTask(SerialNum serial_num, : _serial_num(serial_num), _docid(docid), _attr(attr), + _field_path(), + _doc(nullptr), _field_value(field_value.clone()), _result_promise() { @@ -410,8 +437,15 @@ void PreparePutTask::run() { if (_attr.getStatus().getLastSyncToken() < _serial_num) { + if (_field_path) { + DocumentFieldExtractor field_extractor(*_doc); + _field_value = field_extractor.getFieldValue(*_field_path); + } if (_field_value.get()) { - _result_promise.set_value(AttributeUpdater::prepare_set_value(_attr, _docid, *_field_value)); + auto& fv = *_field_value; + _result_promise.set_value(FieldValueAndPrepareResult(std::move(_field_value), AttributeUpdater::prepare_set_value(_attr, _docid, fv))); + } else { + _result_promise.set_value(FieldValueAndPrepareResult()); } } } @@ -422,7 +456,7 @@ private: const uint32_t _docid; AttributeVector& _attr; FieldValue::SP _field_value; - std::future<std::unique_ptr<PrepareResult>> _result_future; + std::future<FieldValueAndPrepareResult> _result_future; std::remove_reference_t<AttributeWriter::OnWriteDoneType> _on_write_done; public: @@ -437,7 +471,6 @@ CompletePutTask::CompletePutTask(PreparePutTask& prepare_task, : _serial_num(prepare_task.serial_num()), _docid(prepare_task.docid()), _attr(prepare_task.attr()), - _field_value(prepare_task.field_value()), _result_future(prepare_task.result_future()), _on_write_done(on_write_done) { @@ -449,7 +482,7 @@ void CompletePutTask::run() { if (_attr.getStatus().getLastSyncToken() < _serial_num) { - complete_put_to_attribute(_serial_num, _docid, _attr, _field_value, _result_future, _on_write_done); + complete_put_to_attribute(_serial_num, _docid, _attr, std::move(_result_future), _on_write_done); } } @@ -587,33 +620,20 @@ AttributeWriter::setupWriteContexts() } void -AttributeWriter::buildFieldPaths(const DocumentType & docType, const DataType *dataType) -{ - for (auto &wc : _writeContexts) { - wc.buildFieldPaths(docType); - } - _dataType = dataType; -} - -void AttributeWriter::internalPut(SerialNum serialNum, const Document &doc, DocumentIdT lid, bool allAttributes, OnWriteDoneType onWriteDone) { - const DataType *dataType(doc.getDataType()); - if (_dataType != dataType) { - buildFieldPaths(doc.getType(), dataType); - } - auto extractor = std::make_shared<DocumentFieldExtractor>(doc); for (const auto &wc : _writeContexts) { if (wc.use_two_phase_put()) { assert(wc.getFields().size() == 1); - auto prepare_task = std::make_unique<PreparePutTask>(serialNum, lid, wc.getFields()[0], extractor); + wc.consider_build_field_paths(doc); + auto prepare_task = std::make_unique<PreparePutTask>(serialNum, lid, wc, doc); auto complete_task = std::make_unique<CompletePutTask>(*prepare_task, onWriteDone); _shared_executor.execute(std::move(prepare_task)); _attributeFieldWriter.executeTask(wc.getExecutorId(), std::move(complete_task)); } else { if (allAttributes || wc.hasStructFieldAttribute()) { - auto putTask = std::make_unique<PutTask>(wc, serialNum, extractor, lid, allAttributes, onWriteDone); + auto putTask = std::make_unique<PutTask>(wc, serialNum, doc, lid, allAttributes, onWriteDone); _attributeFieldWriter.executeTask(wc.getExecutorId(), std::move(putTask)); } } diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.h b/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.h index 7605b130689..726005ae04b 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.h +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.h @@ -32,7 +32,7 @@ public: * Represents an attribute vector for a field and details about how to write to it. */ class WriteField { - FieldPath _fieldPath; + mutable FieldPath _fieldPath; AttributeVector &_attribute; bool _structFieldAttribute; // in array/map of struct bool _use_two_phase_put; @@ -41,7 +41,7 @@ public: ~WriteField(); AttributeVector &getAttribute() const { return _attribute; } const FieldPath &getFieldPath() const { return _fieldPath; } - void buildFieldPath(const DocumentType &docType); + void buildFieldPath(const DocumentType &docType) const; bool isStructFieldAttribute() const { return _structFieldAttribute; } bool use_two_phase_put() const { return _use_two_phase_put; } }; @@ -52,6 +52,8 @@ public: class WriteContext { ExecutorId _executorId; std::vector<WriteField> _fields; + mutable const DataType* _data_type; + mutable std::shared_ptr<const FieldPath> _two_phase_put_field_path; bool _hasStructFieldAttribute; // When this is true, the context only contains a single field. bool _use_two_phase_put; @@ -60,12 +62,13 @@ public: WriteContext(WriteContext &&rhs) noexcept; ~WriteContext(); WriteContext &operator=(WriteContext &&rhs) noexcept; - void buildFieldPaths(const DocumentType &docType); + void consider_build_field_paths(const Document& doc) const; void add(AttributeVector &attr); ExecutorId getExecutorId() const { return _executorId; } const std::vector<WriteField> &getFields() const { return _fields; } bool hasStructFieldAttribute() const { return _hasStructFieldAttribute; } bool use_two_phase_put() const { return _use_two_phase_put; } + std::shared_ptr<const FieldPath> get_two_phase_put_field_path() const noexcept { return _two_phase_put_field_path; } }; struct AttributeWithInfo { diff --git a/searchcore/src/vespa/searchcore/proton/bucketdb/bucketdb.cpp b/searchcore/src/vespa/searchcore/proton/bucketdb/bucketdb.cpp index 998da6b5789..e1b631a2124 100644 --- a/searchcore/src/vespa/searchcore/proton/bucketdb/bucketdb.cpp +++ b/searchcore/src/vespa/searchcore/proton/bucketdb/bucketdb.cpp @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "bucketdb.h" +#include "remove_batch_entry.h" #include <cassert> #include <algorithm> @@ -9,6 +10,8 @@ using storage::spi::BucketChecksum; namespace proton { +using bucketdb::RemoveBatchEntry; + BucketDB::BucketDB() : _map(), _cachedBucketId(), @@ -65,6 +68,20 @@ BucketDB::remove(const GlobalId &gid, } void +BucketDB::remove_batch(const std::vector<RemoveBatchEntry> &removed, SubDbType sub_db_type) +{ + std::optional<BucketId> prev_bucket_id; + BucketState* state = nullptr; + for (auto &entry : removed) { + if (!prev_bucket_id.has_value() || prev_bucket_id.value() != entry.get_bucket_id()) { + state = &_map[entry.get_bucket_id()]; + prev_bucket_id = entry.get_bucket_id(); + } + state->remove(entry.get_gid(), entry.get_timestamp(), entry.get_doc_size(), sub_db_type); + } +} + +void BucketDB::modify(const GlobalId &gid, const BucketId &oldBucketId, const Timestamp &oldTimestamp, uint32_t oldDocSize, const BucketId &newBucketId, const Timestamp &newTimestamp, uint32_t newDocSize, diff --git a/searchcore/src/vespa/searchcore/proton/bucketdb/bucketdb.h b/searchcore/src/vespa/searchcore/proton/bucketdb/bucketdb.h index 1723609e48e..2ea7594bde1 100644 --- a/searchcore/src/vespa/searchcore/proton/bucketdb/bucketdb.h +++ b/searchcore/src/vespa/searchcore/proton/bucketdb/bucketdb.h @@ -7,6 +7,8 @@ #include <vespa/persistence/spi/result.h> #include <map> +namespace proton::bucketdb { class RemoveBatchEntry; } + namespace proton { class BucketDB @@ -42,6 +44,8 @@ public: const BucketId &bucketId, const Timestamp ×tamp, uint32_t docSize, SubDbType subDbType); + void remove_batch(const std::vector<bucketdb::RemoveBatchEntry> &removed, SubDbType sub_db_type); + void modify(const GlobalId &gid, const BucketId &oldBucketId, const Timestamp &oldTimestamp, uint32_t oldDocSize, const BucketId &newBucketId, const Timestamp &newTimestamp, uint32_t newDocSize, diff --git a/searchcore/src/vespa/searchcore/proton/bucketdb/remove_batch_entry.h b/searchcore/src/vespa/searchcore/proton/bucketdb/remove_batch_entry.h new file mode 100644 index 00000000000..1ab1adb1add --- /dev/null +++ b/searchcore/src/vespa/searchcore/proton/bucketdb/remove_batch_entry.h @@ -0,0 +1,36 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vespa/document/base/globalid.h> +#include <vespa/document/bucket/bucketid.h> +#include <persistence/spi/types.h> + + +namespace proton::bucketdb { + +/* + * Class containing meta data for a single document being removed from + * bucket db. + */ +class RemoveBatchEntry { + document::GlobalId _gid; + document::BucketId _bucket_id; + storage::spi::Timestamp _timestamp; + uint32_t _doc_size; +public: + RemoveBatchEntry(const document::GlobalId& gid, const document::BucketId& bucket_id, const storage::spi::Timestamp& timestamp, uint32_t doc_size) noexcept + : _gid(gid), + _bucket_id(bucket_id), + _timestamp(timestamp), + _doc_size(doc_size) + { + } + + const document::GlobalId& get_gid() const noexcept { return _gid; } + const document::BucketId& get_bucket_id() const noexcept { return _bucket_id; } + const storage::spi::Timestamp& get_timestamp() const noexcept { return _timestamp; } + uint32_t get_doc_size() const noexcept { return _doc_size; } +}; + +} diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.cpp b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.cpp index 78b0780cfa9..13d4a39c8b1 100644 --- a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.cpp +++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.cpp @@ -9,6 +9,7 @@ #include <vespa/persistence/spi/bucket_limits.h> #include <vespa/searchcore/proton/bucketdb/bucketsessionbase.h> #include <vespa/searchcore/proton/bucketdb/joinbucketssession.h> +#include <vespa/searchcore/proton/bucketdb/remove_batch_entry.h> #include <vespa/searchcore/proton/bucketdb/splitbucketsession.h> #include <vespa/searchlib/attribute/load_utils.h> #include <vespa/searchlib/attribute/readerbase.h> @@ -30,6 +31,7 @@ LOG_SETUP(".proton.documentmetastore"); using document::BucketId; using document::GlobalId; using proton::bucketdb::BucketState; +using proton::bucketdb::RemoveBatchEntry; using proton::documentmetastore::GidToLidMapKey; using search::AttributeVector; using search::FileReader; @@ -178,7 +180,7 @@ DocumentMetaStore::insert(GidToLidMapKey key, const RawDocumentMetaData &metaDat std::atomic_thread_fence(std::memory_order_release); _lidAlloc.registerLid(lid); updateUncommittedDocIdLimit(lid); - incGeneration(); + _changesSinceCommit++; const BucketState &state = _bucketDB->takeGuard()->add(metaData.getGid(), metaData.getBucketId().stripUnused(), @@ -208,8 +210,11 @@ DocumentMetaStore::onCommit() _gidToLidMap.compact_worst(); _gid_to_lid_map_write_itr_prepare_serial_num = 0u; _gid_to_lid_map_write_itr.begin(_gidToLidMap.getRoot()); - this->incGeneration(); - this->updateStat(true); + incGeneration(); + updateStat(true); + } else if (_changesSinceCommit > 0) { + incGeneration(); + _changesSinceCommit = 0; } } @@ -415,6 +420,7 @@ DocumentMetaStore::DocumentMetaStore(BucketDBOwnerSP bucketDB, _shrinkLidSpaceBlockers(0), _subDbType(subDbType), _trackDocumentSizes(true), + _changesSinceCommit(0), _op_listener(), _cached_gid_to_lid_map_memory_usage() { @@ -590,7 +596,7 @@ DocumentMetaStore::remove(DocId lid, uint64_t prepare_serial_num) RawDocumentMetaData meta = removeInternal(lid, prepare_serial_num); _bucketDB->takeGuard()->remove(meta.getGid(), meta.getBucketId().stripUnused(), meta.getTimestamp(), meta.getDocSize(), _subDbType); - incGeneration(); + _changesSinceCommit++; if (_op_listener) { _op_listener->notify_remove(); } @@ -629,27 +635,63 @@ DocumentMetaStore::move(DocId fromLid, DocId toLid, uint64_t prepare_serial_num) _gidToLidMap.thaw(itr); itr.writeKey(GidToLidMapKey(toLid, find_key.get_gid_key())); _lidAlloc.moveLidEnd(fromLid, toLid); - incGeneration(); + _changesSinceCommit++; +} + +void +DocumentMetaStore::remove_batch_internal_btree(std::vector<LidAndRawDocumentMetaData>& removed) +{ + // Sort removed array to same order as entries in gid to lid map b-tree + GlobalId::BucketOrderCmp cmp; + std::sort(removed.begin(), removed.end(), [cmp](auto& lhs, auto& rhs) { return cmp(lhs.second.getGid(), rhs.second.getGid()); }); + + _gid_to_lid_map_write_itr_prepare_serial_num = 0u; + auto& itr = _gid_to_lid_map_write_itr; + itr.begin(_gidToLidMap.getRoot()); + for (const auto& lid_and_meta : removed) { + auto lid = lid_and_meta.first; + auto& meta = lid_and_meta.second; + const GlobalId& gid = meta.getGid(); + KeyComp comp(gid, _metaDataStore); + GidToLidMapKey find_key(lid, gid); + if (itr.valid() && comp(itr.getKey(), find_key)) { + itr.binarySeek(find_key, comp); + } + if (!itr.valid() || comp(find_key, itr.getKey())) { + throw IllegalStateException(make_string( + "document meta data store corrupted," + " cannot remove" + " document with lid '%u' and gid '%s'", + lid, gid.toString().c_str())); + } + _gidToLidMap.remove(itr); + } } void DocumentMetaStore::removeBatch(const std::vector<DocId> &lidsToRemove, const uint32_t docIdLimit) { - std::vector<RawDocumentMetaData> removed; + std::vector<LidAndRawDocumentMetaData> removed; removed.reserve(lidsToRemove.size()); for (const auto &lid : lidsToRemove) { assert(lid > 0 && lid < docIdLimit); (void) docIdLimit; assert(validLid(lid)); - removed.push_back(removeInternal(lid, 0u)); + removed.emplace_back(lid, _metaDataStore[lid]); } + remove_batch_internal_btree(removed); + _lidAlloc.unregister_lids(lidsToRemove); { - bucketdb::Guard bucketGuard = _bucketDB->takeGuard(); - for (const auto &meta: removed) { - bucketGuard->remove(meta.getGid(), meta.getBucketId().stripUnused(), - meta.getTimestamp(), meta.getDocSize(), _subDbType); + std::vector<RemoveBatchEntry> bdb_removed; + bdb_removed.reserve(removed.size()); + for (const auto& lid_and_meta : removed) { + auto& meta = lid_and_meta.second; + bdb_removed.emplace_back(meta.getGid(), meta.getBucketId().stripUnused(), + meta.getTimestamp(), meta.getDocSize()); } + bucketdb::Guard bucketGuard = _bucketDB->takeGuard(); + bucketGuard->remove_batch(bdb_removed, _subDbType); } incGeneration(); if (_op_listener) { diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.h b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.h index b99136a6611..4b396d6a971 100644 --- a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.h +++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.h @@ -64,6 +64,7 @@ private: // using the metadata store. using TreeType = vespalib::btree::BTree<documentmetastore::GidToLidMapKey, vespalib::btree::BTreeNoLeafData, vespalib::btree::NoAggregated, const KeyComp &>; + using LidAndRawDocumentMetaData = std::pair<uint32_t, RawDocumentMetaData>; MetaDataStore _metaDataStore; TreeType _gidToLidMap; @@ -74,6 +75,7 @@ private: uint32_t _shrinkLidSpaceBlockers; const SubDbType _subDbType; bool _trackDocumentSizes; + size_t _changesSinceCommit; OperationListenerSP _op_listener; vespalib::MemoryUsage _cached_gid_to_lid_map_memory_usage; @@ -127,6 +129,7 @@ private: VESPA_DLL_LOCAL DocId readNextDoc(documentmetastore::Reader & reader, TreeType::Builder & treeBuilder); RawDocumentMetaData removeInternal(DocId lid, uint64_t cached_iterator_sequence_id); + void remove_batch_internal_btree(std::vector<LidAndRawDocumentMetaData>& removed); public: typedef TreeType::Iterator Iterator; diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastoreflushtarget.cpp b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastoreflushtarget.cpp index 6e18575a90a..9164797b86f 100644 --- a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastoreflushtarget.cpp +++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastoreflushtarget.cpp @@ -56,8 +56,9 @@ Flusher(DocumentMetaStoreFlushTarget &dmsft, _syncToken(syncToken), _flushDir("") { - DocumentMetaStore &dms = *_dmsft._dms; // Called by document db executor + DocumentMetaStore &dms = *_dmsft._dms; + dms.commit(CommitParam(syncToken)); _flushDir = writer.getSnapshotDir(syncToken); vespalib::string newBaseFileName(_flushDir + "/" + dms.getName()); _saver = dms.initSave(newBaseFileName); diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_allocator.cpp b/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_allocator.cpp index 8a6de305f0e..fc5ecfee48b 100644 --- a/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_allocator.cpp +++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_allocator.cpp @@ -32,7 +32,7 @@ LidAllocator::LidAllocator(uint32_t size, } -LidAllocator::~LidAllocator() {} +LidAllocator::~LidAllocator() = default; LidAllocator::DocId LidAllocator::getFreeLid(DocId lidLimit) @@ -79,16 +79,12 @@ LidAllocator::unregisterLid(DocId lid) } } -size_t -LidAllocator::getUsedLidsSize() const -{ - return _usedLids.byteSize(); -} - void -LidAllocator::trimHoldLists(generation_t firstUsed) +LidAllocator::unregister_lids(const std::vector<DocId>& lids) { - _holdLids.trimHoldLists(firstUsed, _freeLids); + for (auto lid : lids) { + unregisterLid(lid); + } } void @@ -268,10 +264,4 @@ LidAllocator::shrinkLidSpace(DocId committedDocIdLimit) ensureSpace(committedDocIdLimit, committedDocIdLimit); } -uint32_t -LidAllocator::getNumUsedLids() const -{ - return _usedLids.count(); -} - } diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_allocator.h b/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_allocator.h index 8c7e9bd0f9c..a8b6fb16f53 100644 --- a/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_allocator.h +++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_allocator.h @@ -38,8 +38,11 @@ public: void ensureSpace(uint32_t newSize, uint32_t newCapacity); void registerLid(DocId lid) { _usedLids.setBit(lid); } void unregisterLid(DocId lid); - size_t getUsedLidsSize() const; - void trimHoldLists(generation_t firstUsed); + void unregister_lids(const std::vector<DocId>& lids); + size_t getUsedLidsSize() const { return _usedLids.byteSize(); } + void trimHoldLists(generation_t firstUsed) { + _holdLids.trimHoldLists(firstUsed, _freeLids); + } void moveLidBegin(DocId fromLid, DocId toLid); void moveLidEnd(DocId fromLid, DocId toLid); void holdLid(DocId lid, DocId lidLimit, generation_t currentGeneration); @@ -51,7 +54,7 @@ public: void updateActiveLids(DocId lid, bool active); void clearDocs(DocId lidLow, DocId lidLimit); void shrinkLidSpace(DocId committedDocIdLimit); - uint32_t getNumUsedLids() const; + uint32_t getNumUsedLids() const { return _usedLids.count(); } uint32_t getNumActiveLids() const { return _numActiveLids; } diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/lidstatevector.cpp b/searchcore/src/vespa/searchcore/proton/documentmetastore/lidstatevector.cpp index e1b5660f483..49e8d3eb23a 100644 --- a/searchcore/src/vespa/searchcore/proton/documentmetastore/lidstatevector.cpp +++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/lidstatevector.cpp @@ -13,7 +13,6 @@ LidStateVector::LidStateVector(unsigned int newSize, unsigned int newCapacity, : _bv(newSize, newCapacity, generationHolder), _lowest(trackLowest ? newSize : 0u), _highest(0), - _count(0u), _trackLowest(trackLowest), _trackHighest(trackHighest) { @@ -29,16 +28,12 @@ LidStateVector::resizeVector(uint32_t newSize, uint32_t newCapacity) bool nolowest(_lowest == _bv.size()); if (_bv.size() > newSize) { _bv.shrink(newSize); - assert(_count >= internalCount()); - _count = internalCount(); } if (_bv.capacity() < newCapacity) { _bv.reserve(newCapacity); - assert(_count == internalCount()); } if (_bv.size() < newSize) { _bv.extend(newSize); - assert(_count == internalCount()); } if (_trackLowest) { if (nolowest) { @@ -57,7 +52,6 @@ LidStateVector::resizeVector(uint32_t newSize, uint32_t newCapacity) maybeUpdateHighest(); } - void LidStateVector::updateLowest() { @@ -70,7 +64,6 @@ LidStateVector::updateLowest() _lowest = lowest; } - void LidStateVector::updateHighest() { @@ -83,7 +76,6 @@ LidStateVector::updateHighest() _highest = highest; } - void LidStateVector::setBit(unsigned int idx) { @@ -96,59 +88,16 @@ LidStateVector::setBit(unsigned int idx) } assert(!_bv.testBit(idx)); _bv.setBitAndMaintainCount(idx); - ++_count; - assert(_count == internalCount()); } - void LidStateVector::clearBit(unsigned int idx) { assert(idx < _bv.size()); assert(_bv.testBit(idx)); _bv.clearBitAndMaintainCount(idx); - --_count; - assert(_count == internalCount()); maybeUpdateLowest(); maybeUpdateHighest(); } - -bool -LidStateVector::empty() const -{ - return _count == 0u; -} - - -unsigned int -LidStateVector::getLowest() const -{ - return _lowest; -} - - -unsigned int -LidStateVector::getHighest() const -{ - return _highest; -} - - -uint32_t -LidStateVector::internalCount() -{ - // Called by document db executor thread. - return _bv.countTrueBits(); -} - - -uint32_t -LidStateVector::count() const -{ - // Called by document db executor thread or metrics related threads - return _count; -} - - } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/lidstatevector.h b/searchcore/src/vespa/searchcore/proton/documentmetastore/lidstatevector.h index 13ce5e213f3..be47676716b 100644 --- a/searchcore/src/vespa/searchcore/proton/documentmetastore/lidstatevector.h +++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/lidstatevector.h @@ -11,9 +11,8 @@ class LidStateVector search::GrowableBitVector _bv; uint32_t _lowest; uint32_t _highest; - uint32_t _count; - bool _trackLowest; - bool _trackHighest; + bool _trackLowest; + bool _trackHighest; void updateLowest(); void updateHighest(); @@ -26,12 +25,6 @@ class LidStateVector if (_trackHighest && _highest != 0 && !_bv.testBit(_highest)) updateHighest(); } - - /** - * Get number of bits set in vector. Should only be called by - * write thread. - */ - uint32_t internalCount(); public: LidStateVector(unsigned int newSize, unsigned int newCapacity, @@ -48,15 +41,18 @@ public: unsigned int byteSize() const { return _bv.extraByteSize() + sizeof(LidStateVector); } - bool empty() const; - unsigned int getLowest() const; - unsigned int getHighest() const; + bool empty() const { return count() == 0u; } + unsigned int getLowest() const { return _lowest; } + unsigned int getHighest() const { return _highest; } /** * Get cached number of bits set in vector. Called by read or * write thread. Write thread must updated cached number as needed. */ - uint32_t count() const; + uint32_t count() const { + // Called by document db executor thread or metrics related threads + return _bv.countTrueBits(); + } unsigned int getNextTrueBit(unsigned int idx) const { return _bv.getNextTrueBit(idx); diff --git a/searchcore/src/vespa/searchcore/proton/index/i_index_writer.h b/searchcore/src/vespa/searchcore/proton/index/i_index_writer.h index 8a920d3d580..a96e344979e 100644 --- a/searchcore/src/vespa/searchcore/proton/index/i_index_writer.h +++ b/searchcore/src/vespa/searchcore/proton/index/i_index_writer.h @@ -24,7 +24,7 @@ public: virtual const std::shared_ptr<IIndexManager> &getIndexManager() const = 0; // feed interface - virtual void put(search::SerialNum serialNum, const document::Document &doc, const search::DocumentIdT lid) = 0; + virtual void put(search::SerialNum serialNum, const document::Document &doc, const search::DocumentIdT lid, OnWriteDoneType on_write_done) = 0; void remove(search::SerialNum serialNum, search::DocumentIdT lid) { LidVector lids; lids.push_back(lid); diff --git a/searchcore/src/vespa/searchcore/proton/index/index_writer.cpp b/searchcore/src/vespa/searchcore/proton/index/index_writer.cpp index f86ada8c45c..3512d2eebad 100644 --- a/searchcore/src/vespa/searchcore/proton/index/index_writer.cpp +++ b/searchcore/src/vespa/searchcore/proton/index/index_writer.cpp @@ -23,7 +23,7 @@ IndexWriter::ignoreOperation(search::SerialNum serialNum) const { } void -IndexWriter::put(search::SerialNum serialNum, const document::Document &doc, const search::DocumentIdT lid) +IndexWriter::put(search::SerialNum serialNum, const document::Document &doc, const search::DocumentIdT lid, OnWriteDoneType on_write_done) { if (ignoreOperation(serialNum)) { return; @@ -39,7 +39,7 @@ IndexWriter::put(search::SerialNum serialNum, const document::Document &doc, con serialNum, doc.getId().toString().c_str()+accum, lid, s1.size(), accum, std::min(accum+chunksize, s1.size()), s1.c_str()); } } - _mgr->putDocument(lid, doc, serialNum); + _mgr->putDocument(lid, doc, serialNum, on_write_done); } void diff --git a/searchcore/src/vespa/searchcore/proton/index/index_writer.h b/searchcore/src/vespa/searchcore/proton/index/index_writer.h index 28357d6fd55..3e0822205bc 100644 --- a/searchcore/src/vespa/searchcore/proton/index/index_writer.h +++ b/searchcore/src/vespa/searchcore/proton/index/index_writer.h @@ -18,7 +18,7 @@ public: ~IndexWriter() override; const IIndexManager::SP & getIndexManager() const override { return _mgr; } - void put(search::SerialNum serialNum, const document::Document &doc, const search::DocumentIdT lid) override; + void put(search::SerialNum serialNum, const document::Document &doc, const search::DocumentIdT lid, OnWriteDoneType on_write_done) override; void removeDocs(search::SerialNum serialNum, LidVector lids) override; void commit(search::SerialNum serialNum, OnWriteDoneType onWriteDone) override; diff --git a/searchcore/src/vespa/searchcore/proton/index/indexmanager.h b/searchcore/src/vespa/searchcore/proton/index/indexmanager.h index 8212978527a..4113af30b0d 100644 --- a/searchcore/src/vespa/searchcore/proton/index/indexmanager.h +++ b/searchcore/src/vespa/searchcore/proton/index/indexmanager.h @@ -86,8 +86,8 @@ public: /** * Implements searchcorespi::IIndexManager **/ - void putDocument(uint32_t lid, const Document &doc, SerialNum serialNum) override { - _maintainer.putDocument(lid, doc, serialNum); + void putDocument(uint32_t lid, const Document &doc, SerialNum serialNum, OnWriteDoneType on_write_done) override { + _maintainer.putDocument(lid, doc, serialNum, on_write_done); } void removeDocuments(LidVector lids, SerialNum serialNum) override { diff --git a/searchcore/src/vespa/searchcore/proton/index/memoryindexwrapper.h b/searchcore/src/vespa/searchcore/proton/index/memoryindexwrapper.h index e4eaad538cb..432262a0322 100644 --- a/searchcore/src/vespa/searchcore/proton/index/memoryindexwrapper.h +++ b/searchcore/src/vespa/searchcore/proton/index/memoryindexwrapper.h @@ -76,8 +76,8 @@ public: vespalib::MemoryUsage getMemoryUsage() const override { return _index.getMemoryUsage(); } - void insertDocument(uint32_t lid, const document::Document &doc) override { - _index.insertDocument(lid, doc); + void insertDocument(uint32_t lid, const document::Document &doc, OnWriteDoneType on_write_done) override { + _index.insertDocument(lid, doc, on_write_done); } void removeDocuments(LidVector lids) override { _index.removeDocuments(std::move(lids)); diff --git a/searchcore/src/vespa/searchcore/proton/matching/extract_features.cpp b/searchcore/src/vespa/searchcore/proton/matching/extract_features.cpp index ef03fac2f6a..b3d59f9ac5f 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/extract_features.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/extract_features.cpp @@ -24,6 +24,32 @@ using OrderedDocs = ExtractFeatures::OrderedDocs; namespace { +auto extract_names(const FeatureResolver &resolver) { + std::vector<vespalib::string> result; + result.reserve(resolver.num_features()); + for (size_t i = 0; i < resolver.num_features(); ++i) { + result.emplace_back(resolver.name_of(i)); + } + return result; +} + +void extract_values(const FeatureResolver &resolver, uint32_t docid, FeatureSet::Value *dst) { + for (uint32_t i = 0; i < resolver.num_features(); ++i) { + if (resolver.is_object(i)) { + auto obj = resolver.resolve(i).as_object(docid); + if (!obj.get().type().is_double()) { + vespalib::nbostream buf; + encode_value(obj.get(), buf); + dst[i].set_data(vespalib::Memory(buf.peek(), buf.size())); + } else { + dst[i].set_double(obj.get().as_double()); + } + } else { + dst[i].set_double(resolver.resolve(i).as_number(docid)); + } + } +} + struct MyChunk : Runnable { const std::pair<uint32_t,uint32_t> *begin; const std::pair<uint32_t,uint32_t> *end; @@ -32,41 +58,26 @@ struct MyChunk : Runnable { const std::pair<uint32_t,uint32_t> *end_in, FeatureValues &result_in) : begin(begin_in), end(end_in), result(result_in) {} - void calculate_features(SearchIterator &search, FeatureResolver &resolver) { - size_t num_features = result.names.size(); + void calculate_features(SearchIterator &search, const FeatureResolver &resolver) { assert(end > begin); - assert(num_features == resolver.num_features()); + assert(resolver.num_features() == result.names.size()); search.initRange(begin[0].first, end[-1].first + 1); for (auto pos = begin; pos != end; ++pos) { - uint32_t docid = pos->first; - search.unpack(docid); - auto * f = &result.values[pos->second * num_features]; - for (uint32_t i = 0; i < num_features; ++i) { - if (resolver.is_object(i)) { - auto obj = resolver.resolve(i).as_object(docid); - if (!obj.get().type().is_double()) { - vespalib::nbostream buf; - encode_value(obj.get(), buf); - f[i].set_data(vespalib::Memory(buf.peek(), buf.size())); - } else { - f[i].set_double(obj.get().as_double()); - } - } else { - f[i].set_double(resolver.resolve(i).as_number(docid)); - } - } + search.unpack(pos->first); + auto *dst = &result.values[pos->second * resolver.num_features()]; + extract_values(resolver, pos->first, dst); } } }; struct FirstChunk : MyChunk { SearchIterator &search; - FeatureResolver &resolver; + const FeatureResolver &resolver; FirstChunk(const std::pair<uint32_t,uint32_t> *begin_in, const std::pair<uint32_t,uint32_t> *end_in, FeatureValues &result_in, SearchIterator &search_in, - FeatureResolver &resolver_in) + const FeatureResolver &resolver_in) : MyChunk(begin_in, end_in, result_in), search(search_in), resolver(resolver_in) {} @@ -110,32 +121,14 @@ struct MyWork { FeatureSet::UP ExtractFeatures::get_feature_set(SearchIterator &search, RankProgram &rank_program, const std::vector<uint32_t> &docs) { - std::vector<vespalib::string> featureNames; FeatureResolver resolver(rank_program.get_seeds(false)); - featureNames.reserve(resolver.num_features()); - for (size_t i = 0; i < resolver.num_features(); ++i) { - featureNames.emplace_back(resolver.name_of(i)); - } - auto result = std::make_unique<FeatureSet>(featureNames, docs.size()); + auto result = std::make_unique<FeatureSet>(extract_names(resolver), docs.size()); if (!docs.empty()) { search.initRange(docs.front(), docs.back()+1); for (uint32_t docid: docs) { search.unpack(docid); - auto * f = result->getFeaturesByIndex(result->addDocId(docid)); - for (uint32_t i = 0; i < featureNames.size(); ++i) { - if (resolver.is_object(i)) { - auto obj = resolver.resolve(i).as_object(docid); - if (!obj.get().type().is_double()) { - vespalib::nbostream buf; - encode_value(obj.get(), buf); - f[i].set_data(vespalib::Memory(buf.peek(), buf.size())); - } else { - f[i].set_double(obj.get().as_double()); - } - } else { - f[i].set_double(resolver.resolve(i).as_number(docid)); - } - } + auto *dst = result->getFeaturesByIndex(result->addDocId(docid)); + extract_values(resolver, docid, dst); } } return result; @@ -148,10 +141,7 @@ ExtractFeatures::get_match_features(const MatchToolsFactory &mtf, const OrderedD auto tools = mtf.createMatchTools(); tools->setup_match_features(); FeatureResolver resolver(tools->rank_program().get_seeds(false)); - result.names.reserve(resolver.num_features()); - for (size_t i = 0; i < resolver.num_features(); ++i) { - result.names.emplace_back(resolver.name_of(i)); - } + result.names = extract_names(resolver); result.values.resize(result.names.size() * docs.size()); MyWork work(thread_bundle); size_t per_thread = docs.size() / work.num_threads; diff --git a/searchcore/src/vespa/searchcore/proton/matching/extract_features.h b/searchcore/src/vespa/searchcore/proton/matching/extract_features.h index 66e98d9db2d..3c0f60b5b3f 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/extract_features.h +++ b/searchcore/src/vespa/searchcore/proton/matching/extract_features.h @@ -5,7 +5,7 @@ #include <vespa/searchlib/common/featureset.h> #include <vector> -namespace vespalib { class ThreadBundle; }; +namespace vespalib { struct ThreadBundle; }; namespace search::queryeval { class SearchIterator; } namespace search::fef { class RankProgram; } diff --git a/searchcore/src/vespa/searchcore/proton/matching/match_master.cpp b/searchcore/src/vespa/searchcore/proton/matching/match_master.cpp index 827ff4b5aca..3ff0a7d1808 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/match_master.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/match_master.cpp @@ -13,7 +13,6 @@ #include <vespa/vespalib/data/slime/inserter.h> #include <vespa/vespalib/data/slime/inject.h> #include <vespa/vespalib/data/slime/cursor.h> -#include <vespa/vespalib/objects/nbostream.h> #include <vespa/log/log.h> LOG_SETUP(".proton.matching.match_master"); @@ -60,7 +59,8 @@ createScheduler(uint32_t numThreads, uint32_t numSearchPartitions, uint32_t numD return std::make_unique<TaskDocidRangeScheduler>(numThreads, numSearchPartitions, numDocs); } -auto make_reply(const MatchToolsFactory &mtf, ResultProcessor &processor, ThreadBundle &bundle, auto full_result) { +template <class FullResult> +auto make_reply(const MatchToolsFactory &mtf, ResultProcessor &processor, ThreadBundle &bundle, FullResult full_result) { if (mtf.has_match_features()) { auto docs = processor.extract_docid_ordering(*full_result); auto reply = processor.makeReply(std::move(std::move(full_result))); diff --git a/searchcore/src/vespa/searchcore/proton/matching/partial_result.cpp b/searchcore/src/vespa/searchcore/proton/matching/partial_result.cpp index 6ae97a125ad..432752d69d0 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/partial_result.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/partial_result.cpp @@ -7,10 +7,10 @@ namespace proton::matching { namespace { bool before(const search::RankedHit &a, const search::RankedHit &b) { - if (a._rankValue != b._rankValue) { - return (a._rankValue > b._rankValue); + if (a.getRank() != b.getRank()) { + return (a.getRank() > b.getRank()); } - return (a._docId < b._docId); + return (a.getDocId() < b.getDocId()); } void mergeHits(size_t maxHits, diff --git a/searchcore/src/vespa/searchcore/proton/matching/result_processor.cpp b/searchcore/src/vespa/searchcore/proton/matching/result_processor.cpp index da1e6a2d567..f332ca5ec26 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/result_processor.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/result_processor.cpp @@ -5,7 +5,6 @@ #include "sessionmanager.h" #include <vespa/searchcore/grouping/groupingmanager.h> #include <vespa/searchcore/grouping/groupingcontext.h> -#include <vespa/searchlib/common/docstamp.h> #include <vespa/searchlib/uca/ucaconverter.h> #include <vespa/searchlib/engine/searchreply.h> @@ -109,7 +108,7 @@ ResultProcessor::extract_docid_ordering(const PartialResult &result) const std::vector<std::pair<uint32_t,uint32_t>> list; list.reserve(est_size); for (size_t i = _offset; i < result.size(); ++i) { - list.emplace_back(result.hit(i)._docId, list.size()); + list.emplace_back(result.hit(i).getDocId(), list.size()); } std::sort(list.begin(), list.end(), [](const auto &a, const auto &b){ return (a.first < b.first); }); return list; @@ -143,11 +142,11 @@ ResultProcessor::makeReply(PartialResultUP full_result) for (size_t i = 0; i < hitcnt; ++i) { search::engine::SearchReply::Hit &dst = r.hits[i]; const search::RankedHit &src = result.hit(hitOffset + i); - uint32_t docId = src._docId; + uint32_t docId = src.getDocId(); if (metaStore.getGidEvenIfMoved(docId, gid)) { dst.gid = gid; } - dst.metric = src._rankValue; + dst.metric = src.getRank(); LOG(debug, "convertLidToGid: hit[%zu]: lid(%u) -> gid(%s)", i, docId, dst.gid.toString().c_str()); } if (result.hasSortData() && (hitcnt > 0)) { diff --git a/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.cpp b/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.cpp index 2e1fc74037c..04d91b9c028 100644 --- a/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.cpp +++ b/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.cpp @@ -364,7 +364,7 @@ PersistenceEngine::putAsync(const Bucket &bucket, Timestamp ts, storage::spi::Do return onComplete->onComplete(std::make_unique<Result>(Result::ErrorType::PERMANENT_ERROR, make_string("No handler for document type '%s'", docType.toString().c_str()))); } - auto transportContext = std::make_shared<AsyncTranportContext>(1, std::move(onComplete)); + auto transportContext = std::make_shared<AsyncTransportContext>(1, std::move(onComplete)); handler->handlePut(feedtoken::make(std::move(transportContext)), bucket, ts, std::move(doc)); } @@ -384,7 +384,7 @@ PersistenceEngine::removeAsync(const Bucket& b, Timestamp t, const DocumentId& d return onComplete->onComplete(std::make_unique<RemoveResult>(Result::ErrorType::PERMANENT_ERROR, make_string("No handler for document type '%s'", docType.toString().c_str()))); } - auto transportContext = std::make_shared<AsyncTranportContext>(1, std::move(onComplete)); + auto transportContext = std::make_shared<AsyncTransportContext>(1, std::move(onComplete)); handler->handleRemove(feedtoken::make(std::move(transportContext)), b, t, did); } @@ -436,7 +436,7 @@ PersistenceEngine::updateAsync(const Bucket& b, Timestamp t, DocumentUpdate::SP if (handler == nullptr) { return onComplete->onComplete(std::make_unique<UpdateResult>(Result::ErrorType::PERMANENT_ERROR, make_string("No handler for document type '%s'", docType.toString().c_str()))); } - auto transportContext = std::make_shared<AsyncTranportContext>(1, std::move(onComplete)); + auto transportContext = std::make_shared<AsyncTransportContext>(1, std::move(onComplete)); handler->handleUpdate(feedtoken::make(std::move(transportContext)), b, t, std::move(upd)); } @@ -555,7 +555,7 @@ PersistenceEngine::createBucketAsync(const Bucket &b, Context &, OperationComple LOG(spam, "createBucket(%s)", b.toString().c_str()); HandlerSnapshot snap = getHandlerSnapshot(rguard, b.getBucketSpace()); - auto transportContext = std::make_shared<AsyncTranportContext>(snap.size(), std::move(onComplete)); + auto transportContext = std::make_shared<AsyncTransportContext>(snap.size(), std::move(onComplete)); while (snap.handlers().valid()) { IPersistenceHandler *handler = snap.handlers().get(); snap.handlers().next(); @@ -575,7 +575,7 @@ PersistenceEngine::deleteBucketAsync(const Bucket& b, Context&, OperationComplet LOG(spam, "deleteBucket(%s)", b.toString().c_str()); HandlerSnapshot snap = getHandlerSnapshot(rguard, b.getBucketSpace()); - auto transportContext = std::make_shared<AsyncTranportContext>(snap.size(), std::move(onComplete)); + auto transportContext = std::make_shared<AsyncTransportContext>(snap.size(), std::move(onComplete)); while (snap.handlers().valid()) { IPersistenceHandler *handler = snap.handlers().get(); snap.handlers().next(); diff --git a/searchcore/src/vespa/searchcore/proton/persistenceengine/transport_latch.cpp b/searchcore/src/vespa/searchcore/proton/persistenceengine/transport_latch.cpp index 61c240f0d6a..8a0955b3147 100644 --- a/searchcore/src/vespa/searchcore/proton/persistenceengine/transport_latch.cpp +++ b/searchcore/src/vespa/searchcore/proton/persistenceengine/transport_latch.cpp @@ -67,7 +67,7 @@ TransportLatch::send(ResultUP result, bool documentWasFound) _latch.countDown(); } -AsyncTranportContext::AsyncTranportContext(uint32_t cnt, OperationComplete::UP onComplete) +AsyncTransportContext::AsyncTransportContext(uint32_t cnt, OperationComplete::UP onComplete) : TransportMerger(cnt > 1), _countDown(cnt), _onComplete(std::move(onComplete)) @@ -78,17 +78,17 @@ AsyncTranportContext::AsyncTranportContext(uint32_t cnt, OperationComplete::UP o } void -AsyncTranportContext::completeIfDone() { +AsyncTransportContext::completeIfDone() { _countDown--; if (_countDown == 0) { _onComplete->onComplete(std::move(_result)); } } -AsyncTranportContext::~AsyncTranportContext() = default; +AsyncTransportContext::~AsyncTransportContext() = default; void -AsyncTranportContext::send(ResultUP result, bool documentWasFound) +AsyncTransportContext::send(ResultUP result, bool documentWasFound) { mergeResult(std::move(result), documentWasFound); } diff --git a/searchcore/src/vespa/searchcore/proton/persistenceengine/transport_latch.h b/searchcore/src/vespa/searchcore/proton/persistenceengine/transport_latch.h index 084ad0e8d10..91cadfedcd5 100644 --- a/searchcore/src/vespa/searchcore/proton/persistenceengine/transport_latch.h +++ b/searchcore/src/vespa/searchcore/proton/persistenceengine/transport_latch.h @@ -63,7 +63,7 @@ public: * Implementation of FeedToken::ITransport for async handling of the async reply for an operation. * Uses an internal count to keep track the number of the outstanding replies. */ -class AsyncTranportContext : public TransportMerger { +class AsyncTransportContext : public TransportMerger { private: using Result = storage::spi::Result; using OperationComplete = storage::spi::OperationComplete; @@ -72,8 +72,8 @@ private: OperationComplete::UP _onComplete; void completeIfDone() override; public: - AsyncTranportContext(uint32_t cnt, OperationComplete::UP); - ~AsyncTranportContext() override; + AsyncTransportContext(uint32_t cnt, OperationComplete::UP); + ~AsyncTransportContext() override; void send(ResultUP result, bool documentWasFound) override; }; diff --git a/searchcore/src/vespa/searchcore/proton/server/searchable_feed_view.cpp b/searchcore/src/vespa/searchcore/proton/server/searchable_feed_view.cpp index d973777020d..323ca9add17 100644 --- a/searchcore/src/vespa/searchcore/proton/server/searchable_feed_view.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/searchable_feed_view.cpp @@ -61,7 +61,7 @@ SearchableFeedView::performIndexPut(SerialNum serialNum, search::DocumentIdT lid "database(%s): performIndexPut: serialNum(%" PRIu64 "), docId(%s), lid(%d)", _params._docTypeName.toString().c_str(), serialNum, doc.getId().toString().c_str(), lid); - _indexWriter->put(serialNum, doc, lid); + _indexWriter->put(serialNum, doc, lid, onWriteDone); } void diff --git a/searchcore/src/vespa/searchcore/proton/test/mock_index_manager.h b/searchcore/src/vespa/searchcore/proton/test/mock_index_manager.h index 7c4de5be2db..bf76dc3d59b 100644 --- a/searchcore/src/vespa/searchcore/proton/test/mock_index_manager.h +++ b/searchcore/src/vespa/searchcore/proton/test/mock_index_manager.h @@ -10,7 +10,7 @@ namespace proton::test { */ struct MockIndexManager : public searchcorespi::IIndexManager { - void putDocument(uint32_t, const Document &, SerialNum) override {} + void putDocument(uint32_t, const Document &, SerialNum, OnWriteDoneType) override {} void removeDocuments(LidVector, SerialNum) override {} void commit(SerialNum, OnWriteDoneType) override {} SerialNum getCurrentSerialNum() const override { return 0; } diff --git a/searchcore/src/vespa/searchcore/proton/test/mock_index_writer.h b/searchcore/src/vespa/searchcore/proton/test/mock_index_writer.h index 96af605b27c..bf571a74941 100644 --- a/searchcore/src/vespa/searchcore/proton/test/mock_index_writer.h +++ b/searchcore/src/vespa/searchcore/proton/test/mock_index_writer.h @@ -14,7 +14,7 @@ struct MockIndexWriter : public IIndexWriter MockIndexWriter() : _idxMgr() {} MockIndexWriter(const IIndexManager::SP &idxMgr) : _idxMgr(idxMgr) {} const IIndexManager::SP &getIndexManager() const override { return _idxMgr; } - void put(search::SerialNum, const document::Document &, const search::DocumentIdT) override {} + void put(search::SerialNum, const document::Document &, const search::DocumentIdT, OnWriteDoneType) override {} void removeDocs(search::SerialNum, LidVector) override {} void commit(search::SerialNum, OnWriteDoneType) override {} void heartBeat(search::SerialNum) override {} diff --git a/searchcorespi/src/vespa/searchcorespi/index/iindexmanager.h b/searchcorespi/src/vespa/searchcorespi/index/iindexmanager.h index 02adcbc11ce..a4173b41aa5 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/iindexmanager.h +++ b/searchcorespi/src/vespa/searchcorespi/index/iindexmanager.h @@ -93,8 +93,11 @@ public: * * @param serialNum The unique monotoninc increasing serial number * for this operation. + * + * @param on_write_done shared object that notifies write done when + * destructed. **/ - virtual void putDocument(uint32_t lid, const Document &doc, SerialNum serialNum) = 0; + virtual void putDocument(uint32_t lid, const Document &doc, SerialNum serialNum, OnWriteDoneType on_write_done) = 0; /** * Removes the given document from the index. This method is diff --git a/searchcorespi/src/vespa/searchcorespi/index/imemoryindex.h b/searchcorespi/src/vespa/searchcorespi/index/imemoryindex.h index bff929206a0..16feeb2ce24 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/imemoryindex.h +++ b/searchcorespi/src/vespa/searchcorespi/index/imemoryindex.h @@ -42,8 +42,9 @@ struct IMemoryIndex : public searchcorespi::IndexSearchable { * * @param lid the local document id. * @param doc the document to insert. + * @param on_write_done shared object that notifies write done when destructed. */ - virtual void insertDocument(uint32_t lid, const document::Document &doc) = 0; + virtual void insertDocument(uint32_t lid, const document::Document &doc, OnWriteDoneType on_write_done) = 0; /** * Removes the given document from this memory index. diff --git a/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.cpp b/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.cpp index 74848e93411..19469d59d1b 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.cpp +++ b/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.cpp @@ -16,6 +16,7 @@ #include <vespa/vespalib/io/fileutil.h> #include <vespa/vespalib/util/array.hpp> #include <vespa/vespalib/util/exceptions.h> +#include <vespa/vespalib/util/gate.h> #include <vespa/vespalib/util/lambdatask.h> #include <vespa/vespalib/util/destructor_callbacks.h> #include <vespa/vespalib/util/time.h> @@ -940,9 +941,8 @@ IndexMaintainer::initFlush(SerialNum serialNum, searchcorespi::FlushStats * stat IMemoryIndex::SP new_index(_operations.createMemoryIndex(getSchema(), *_current_index, _current_serial_num)); FlushArgs args; args.stats = stats; - scheduleCommit(); // Ensure that all index thread tasks accessing memory index have completed. - _ctx.getThreadingService().sync(); + commit_and_wait(); // Call reconfig closure for this change auto configure = makeLambdaConfigure([this, argsP=&args, indexP=&new_index]() { return doneInitFlush(argsP, indexP); @@ -1165,12 +1165,12 @@ IndexMaintainer::getNumFrozenMemoryIndexes(void) const } void -IndexMaintainer::putDocument(uint32_t lid, const Document &doc, SerialNum serialNum) +IndexMaintainer::putDocument(uint32_t lid, const Document &doc, SerialNum serialNum, OnWriteDoneType on_write_done) { assert(_ctx.getThreadingService().index().isCurrentThread()); LockGuard lock(_index_update_lock); try { - _current_index->insertDocument(lid, doc); + _current_index->insertDocument(lid, doc, on_write_done); } catch (const vespalib::IllegalStateException & e) { vespalib::string s = "Failed inserting document :\n" + doc.toXml(" ") + "\n"; LOG(error, "%s", s.c_str()); @@ -1197,20 +1197,22 @@ IndexMaintainer::removeDocuments(LidVector lids, SerialNum serialNum) } void -IndexMaintainer::scheduleCommit() +IndexMaintainer::commit_and_wait() { assert(_ctx.getThreadingService().master().isCurrentThread()); - _ctx.getThreadingService().index().execute(makeLambdaTask([this]() { commit(); })); + vespalib::Gate gate; + _ctx.getThreadingService().index().execute(makeLambdaTask([this, &gate]() { commit(gate); })); + // Ensure that all index thread tasks accessing memory index have completed. + gate.await(); } void -IndexMaintainer::commit() +IndexMaintainer::commit(vespalib::Gate& gate) { - // only triggered via scheduleCommit() + // only triggered via commit_and_wait() assert(_ctx.getThreadingService().index().isCurrentThread()); LockGuard lock(_index_update_lock); - _current_index->commit({}, _current_serial_num); - // caller calls _ctx.getThreadingService().sync() + _current_index->commit(std::make_shared<vespalib::GateCallback>(gate), _current_serial_num); } void @@ -1260,9 +1262,8 @@ IndexMaintainer::setSchema(const Schema & schema, SerialNum serialNum) SetSchemaArgs args; args._newSchema = schema; - scheduleCommit(); // Ensure that all index thread tasks accessing memory index have completed. - _ctx.getThreadingService().sync(); + commit_and_wait(); // Everything should be quiet now. doneSetSchema(args, new_index); // Source collection has now changed, caller must reconfigure further @@ -1287,8 +1288,7 @@ IndexMaintainer::pruneRemovedFields(const Schema &schema, SerialNum serialNum) new_source_list = std::make_shared<IndexCollection>(_selector, *_source_list); } if (reopenDiskIndexes(*new_source_list)) { - scheduleCommit(); - _ctx.getThreadingService().sync(); + commit_and_wait(); // Everything should be quiet now. LockGuard state_lock(_state_lock); LockGuard lock(_new_search_lock); diff --git a/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.h b/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.h index 816ec76cd1f..6e4eb32ee50 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.h +++ b/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.h @@ -22,6 +22,7 @@ namespace document { class Document; } namespace search::common { class FileHeaderContext; } +namespace vespalib { class Gate; } namespace searchcorespi::index { @@ -258,8 +259,8 @@ class IndexMaintainer : public IIndexManager, bool reconfigure(std::unique_ptr<Configure> configure); void warmupDone(ISearchableIndexCollection::SP current) override; bool makeSureAllRemainingWarmupIsDone(ISearchableIndexCollection::SP keepAlive); - void scheduleCommit(); - void commit(); + void commit_and_wait(); + void commit(vespalib::Gate& gate); void pruneRemovedFields(const Schema &schema, SerialNum serialNum); public: @@ -324,7 +325,7 @@ public: vespalib::system_time getLastFlushTime() const { return _lastFlushTime; } // Implements IIndexManager - void putDocument(uint32_t lid, const Document &doc, SerialNum serialNum) override; + void putDocument(uint32_t lid, const Document &doc, SerialNum serialNum, OnWriteDoneType on_write_done) override; void removeDocuments(LidVector lids, SerialNum serialNum) override; void commit(SerialNum serialNum, OnWriteDoneType onWriteDone) override; void heartBeat(search::SerialNum serialNum) override; diff --git a/searchlib/src/apps/tests/memoryindexstress_test.cpp b/searchlib/src/apps/tests/memoryindexstress_test.cpp index dd445745f18..54864702a47 100644 --- a/searchlib/src/apps/tests/memoryindexstress_test.cpp +++ b/searchlib/src/apps/tests/memoryindexstress_test.cpp @@ -48,6 +48,7 @@ using search::query::Node; using search::query::SimplePhrase; using search::query::SimpleStringTerm; using search::index::test::MockFieldLengthInspector; +using vespalib::IDestructorCallback; using vespalib::asciistream; using vespalib::makeLambdaTask; @@ -190,6 +191,16 @@ Node::UP makePhrase(const std::string &term1, const std::string &term2) { return node; } +class HoldDoc : public IDestructorCallback { + std::unique_ptr<Document> _doc; +public: + HoldDoc(std::unique_ptr<Document> doc) noexcept + : _doc(std::move(doc)) + { + } + ~HoldDoc() override = default; +}; + } // namespace struct Fixture { @@ -224,7 +235,8 @@ struct Fixture { gate.await(); } void put(uint32_t id, Document::UP doc) { - index.insertDocument(id, *doc); + auto& docref = *doc; + index.insertDocument(id, docref, std::make_shared<HoldDoc>(std::move(doc))); } void remove(uint32_t id) { std::vector<uint32_t> lids; diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/RankingExpression.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/RankingExpression.java index 78d09b27b54..5a73d89cf2c 100755 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/RankingExpression.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/RankingExpression.java @@ -32,7 +32,7 @@ import java.util.Map; * <p>The identity of a ranking expression is decided by both its name and expression tree. Two expressions which * looks the same in string form are the same.</p> * - * <h3>Simple usage</h3> + * <h2>Simple usage</h2> <pre><code> try { MapContext context = new MapContext(); diff --git a/searchlib/src/tests/attribute/searchcontext/searchcontext_test.cpp b/searchlib/src/tests/attribute/searchcontext/searchcontext_test.cpp index 4076194542a..de54386e4af 100644 --- a/searchlib/src/tests/attribute/searchcontext/searchcontext_test.cpp +++ b/searchlib/src/tests/attribute/searchcontext/searchcontext_test.cpp @@ -487,7 +487,7 @@ SearchContextTest::checkResultSet(const ResultSet & rs, const DocSet & expected, ASSERT_TRUE(array != nullptr); uint32_t i = 0; for (auto iter = expected.begin(); iter != expected.end(); ++iter, ++i) { - EXPECT_TRUE(array[i]._docId == *iter); + EXPECT_TRUE(array[i].getDocId() == *iter); } } } @@ -1517,10 +1517,10 @@ SearchContextTest::requireThatSearchIsWorkingAfterClearDoc(const vespalib::strin EXPECT_EQUAL(4u, rs->getNumHits()); ASSERT_TRUE(4u == rs->getNumHits()); const RankedHit * array = rs->getArray(); - EXPECT_EQUAL(1u, array[0]._docId); - EXPECT_EQUAL(2u, array[1]._docId); - EXPECT_EQUAL(3u, array[2]._docId); - EXPECT_EQUAL(4u, array[3]._docId); + EXPECT_EQUAL(1u, array[0].getDocId()); + EXPECT_EQUAL(2u, array[1].getDocId()); + EXPECT_EQUAL(3u, array[2].getDocId()); + EXPECT_EQUAL(4u, array[3].getDocId()); } a->clearDoc(1); a->clearDoc(3); @@ -1529,8 +1529,8 @@ SearchContextTest::requireThatSearchIsWorkingAfterClearDoc(const vespalib::strin ResultSetPtr rs = performSearch(v, term); EXPECT_EQUAL(2u, rs->getNumHits()); const RankedHit * array = rs->getArray(); - EXPECT_EQUAL(2u, array[0]._docId); - EXPECT_EQUAL(4u, array[1]._docId); + EXPECT_EQUAL(2u, array[0].getDocId()); + EXPECT_EQUAL(4u, array[1].getDocId()); } } @@ -1578,9 +1578,9 @@ SearchContextTest::requireThatSearchIsWorkingAfterLoadAndClearDoc(const vespalib const RankedHit * array = rs->getArray(); for (uint32_t i = 0; i < 14; ++i) { if (i < 5) { - EXPECT_EQUAL(i + 1, array[i]._docId); + EXPECT_EQUAL(i + 1, array[i].getDocId()); } else - EXPECT_EQUAL(i + 2, array[i]._docId); + EXPECT_EQUAL(i + 2, array[i].getDocId()); } } ValueType buf; @@ -1682,15 +1682,15 @@ SearchContextTest::requireThatFlagAttributeIsWorkingWhenNewDocsAreAdded() { ResultSetPtr rs = performSearch(fa, "<24"); EXPECT_EQUAL(2u, rs->getNumHits()); - EXPECT_EQUAL(1u, rs->getArray()[0]._docId); - EXPECT_EQUAL(2u, rs->getArray()[1]._docId); + EXPECT_EQUAL(1u, rs->getArray()[0].getDocId()); + EXPECT_EQUAL(2u, rs->getArray()[1].getDocId()); } { ResultSetPtr rs = performSearch(fa, "24"); EXPECT_EQUAL(3u, rs->getNumHits()); - EXPECT_EQUAL(1u, rs->getArray()[0]._docId); - EXPECT_EQUAL(2u, rs->getArray()[1]._docId); - EXPECT_EQUAL(4u, rs->getArray()[2]._docId); + EXPECT_EQUAL(1u, rs->getArray()[0].getDocId()); + EXPECT_EQUAL(2u, rs->getArray()[1].getDocId()); + EXPECT_EQUAL(4u, rs->getArray()[2].getDocId()); } } { @@ -1717,15 +1717,15 @@ SearchContextTest::requireThatFlagAttributeIsWorkingWhenNewDocsAreAdded() EXPECT_EQUAL(exp50.size(), rs1->getNumHits()); EXPECT_EQUAL(exp50.size(), rs2->getNumHits()); for (size_t j = 0; j < exp50.size(); ++j) { - EXPECT_EQUAL(exp50[j], rs1->getArray()[j]._docId); - EXPECT_EQUAL(exp50[j], rs2->getArray()[j]._docId); + EXPECT_EQUAL(exp50[j], rs1->getArray()[j].getDocId()); + EXPECT_EQUAL(exp50[j], rs2->getArray()[j].getDocId()); } } { ResultSetPtr rs = performSearch(fa, "60"); EXPECT_EQUAL(exp60.size(), rs->getNumHits()); for (size_t j = 0; j < exp60.size(); ++j) { - EXPECT_EQUAL(exp60[j], rs->getArray()[j]._docId); + EXPECT_EQUAL(exp60[j], rs->getArray()[j].getDocId()); } } } diff --git a/searchlib/src/tests/diskindex/fusion/fusion_test.cpp b/searchlib/src/tests/diskindex/fusion/fusion_test.cpp index 3889de5b4c4..1c86981372d 100644 --- a/searchlib/src/tests/diskindex/fusion/fusion_test.cpp +++ b/searchlib/src/tests/diskindex/fusion/fusion_test.cpp @@ -332,7 +332,7 @@ FusionTest::requireThatFusionIsWorking(const vespalib::string &prefix, bool dire Document::UP doc; doc = make_doc10(b); - inv.invertDocument(10, *doc); + inv.invertDocument(10, *doc, {}); myPushDocument(inv); b.startDocument("id:ns:searchdocument::11"). @@ -340,7 +340,7 @@ FusionTest::requireThatFusionIsWorking(const vespalib::string &prefix, bool dire startElement(-27).addStr("zz").endElement(). endField(); doc = b.endDocument(); - inv.invertDocument(11, *doc); + inv.invertDocument(11, *doc, {}); myPushDocument(inv); b.startDocument("id:ns:searchdocument::12"). @@ -348,7 +348,7 @@ FusionTest::requireThatFusionIsWorking(const vespalib::string &prefix, bool dire startElement(0).addStr("zz0").endElement(). endField(); doc = b.endDocument(); - inv.invertDocument(12, *doc); + inv.invertDocument(12, *doc, {}); myPushDocument(inv); IndexBuilder ib(schema); @@ -466,7 +466,8 @@ FusionTest::make_simple_index(const vespalib::string &dump_dir, const IFieldLeng DocumentInverterContext inv_context(_schema, *invertThreads, *pushThreads, fic); DocumentInverter inv(inv_context); - inv.invertDocument(10, *make_doc10(b)); + auto doc10 = make_doc10(b); + inv.invertDocument(10, *doc10, {}); myPushDocument(inv); IndexBuilder ib(_schema); diff --git a/searchlib/src/tests/grouping/grouping_test.cpp b/searchlib/src/tests/grouping/grouping_test.cpp index ef4930de8ce..2eab66cb3b7 100644 --- a/searchlib/src/tests/grouping/grouping_test.cpp +++ b/searchlib/src/tests/grouping/grouping_test.cpp @@ -105,7 +105,7 @@ public: hit._rankValue = rank; _hits.push_back(hit); for (uint32_t pos = (_hits.size() - 1); - pos > 0 && (_hits[pos]._rankValue > _hits[pos - 1]._rankValue); + pos > 0 && (_hits[pos].getRank() > _hits[pos - 1].getRank()); --pos) { std::swap(_hits[pos], _hits[pos - 1]); diff --git a/searchlib/src/tests/groupingengine/groupingengine_benchmark.cpp b/searchlib/src/tests/groupingengine/groupingengine_benchmark.cpp index 66fa359f1a3..e82079073e7 100644 --- a/searchlib/src/tests/groupingengine/groupingengine_benchmark.cpp +++ b/searchlib/src/tests/groupingengine/groupingengine_benchmark.cpp @@ -88,7 +88,7 @@ public: hit._rankValue = rank; _hits.push_back(hit); for (uint32_t pos = (_hits.size() - 1); - pos > 0 && (_hits[pos]._rankValue > _hits[pos - 1]._rankValue); + pos > 0 && (_hits[pos].getRank() > _hits[pos - 1].getRank()); --pos) { std::swap(_hits[pos], _hits[pos - 1]); diff --git a/searchlib/src/tests/groupingengine/groupingengine_test.cpp b/searchlib/src/tests/groupingengine/groupingengine_test.cpp index a0179c36c23..d54b68388e4 100644 --- a/searchlib/src/tests/groupingengine/groupingengine_test.cpp +++ b/searchlib/src/tests/groupingengine/groupingengine_test.cpp @@ -87,7 +87,7 @@ public: hit._rankValue = rank; _hits.push_back(hit); for (uint32_t pos = (_hits.size() - 1); - pos > 0 && (_hits[pos]._rankValue > _hits[pos - 1]._rankValue); + pos > 0 && (_hits[pos].getRank() > _hits[pos - 1].getRank()); --pos) { std::swap(_hits[pos], _hits[pos - 1]); diff --git a/searchlib/src/tests/hitcollector/hitcollector_test.cpp b/searchlib/src/tests/hitcollector/hitcollector_test.cpp index 617e0e85824..ed68c47ea23 100644 --- a/searchlib/src/tests/hitcollector/hitcollector_test.cpp +++ b/searchlib/src/tests/hitcollector/hitcollector_test.cpp @@ -70,8 +70,8 @@ void checkResult(const ResultSet & rs, const std::vector<RankedHit> & exp) ASSERT_EQUAL(rs.getArrayUsed(), exp.size()); for (uint32_t i = 0; i < exp.size(); ++i) { - EXPECT_EQUAL(rh[i]._docId, exp[i]._docId); - EXPECT_EQUAL(rh[i]._rankValue + 1.0, exp[i]._rankValue + 1.0); + EXPECT_EQUAL(rh[i].getDocId(), exp[i].getDocId()); + EXPECT_EQUAL(rh[i].getRank() + 1.0, exp[i].getRank() + 1.0); } } else { ASSERT_TRUE(rs.getArray() == nullptr); diff --git a/searchlib/src/tests/memoryindex/document_inverter/document_inverter_test.cpp b/searchlib/src/tests/memoryindex/document_inverter/document_inverter_test.cpp index dec21c64456..3f8a04d9460 100644 --- a/searchlib/src/tests/memoryindex/document_inverter/document_inverter_test.cpp +++ b/searchlib/src/tests/memoryindex/document_inverter/document_inverter_test.cpp @@ -140,7 +140,8 @@ struct DocumentInverterTest : public ::testing::Test { TEST_F(DocumentInverterTest, require_that_fresh_insert_works) { - _inv.invertDocument(10, *makeDoc10(_b)); + auto doc10 = makeDoc10(_b); + _inv.invertDocument(10, *doc10, {}); pushDocuments(); EXPECT_EQ("f=0,w=a,a=10," "w=b,a=10," @@ -151,8 +152,10 @@ TEST_F(DocumentInverterTest, require_that_fresh_insert_works) TEST_F(DocumentInverterTest, require_that_multiple_docs_work) { - _inv.invertDocument(10, *makeDoc10(_b)); - _inv.invertDocument(11, *makeDoc11(_b)); + auto doc10 = makeDoc10(_b); + auto doc11 = makeDoc11(_b); + _inv.invertDocument(10, *doc10, {}); + _inv.invertDocument(11, *doc11, {}); pushDocuments(); EXPECT_EQ("f=0,w=a,a=10,a=11," "w=b,a=10,a=11," @@ -181,8 +184,10 @@ TEST_F(DocumentInverterTest, require_that_remove_works) TEST_F(DocumentInverterTest, require_that_reput_works) { - _inv.invertDocument(10, *makeDoc10(_b)); - _inv.invertDocument(10, *makeDoc11(_b)); + auto doc10 = makeDoc10(_b); + auto doc11 = makeDoc11(_b); + _inv.invertDocument(10, *doc10, {}); + _inv.invertDocument(10, *doc11, {}); pushDocuments(); EXPECT_EQ("f=0,w=a,a=10," "w=b,a=10," @@ -201,8 +206,8 @@ TEST_F(DocumentInverterTest, require_that_abort_pending_doc_works) auto doc13 = makeDoc13(_b); auto doc14 = makeDoc14(_b); - _inv.invertDocument(10, *doc10); - _inv.invertDocument(11, *doc11); + _inv.invertDocument(10, *doc10, {}); + _inv.invertDocument(11, *doc11, {}); _inv.removeDocument(10); pushDocuments(); EXPECT_EQ("f=0,w=a,a=11," @@ -213,11 +218,11 @@ TEST_F(DocumentInverterTest, require_that_abort_pending_doc_works) "w=g,a=11", _inserter_backend.toStr()); - _inv.invertDocument(10, *doc10); - _inv.invertDocument(11, *doc11); - _inv.invertDocument(12, *doc12); - _inv.invertDocument(13, *doc13); - _inv.invertDocument(14, *doc14); + _inv.invertDocument(10, *doc10, {}); + _inv.invertDocument(11, *doc11, {}); + _inv.invertDocument(12, *doc12, {}); + _inv.invertDocument(13, *doc13, {}); + _inv.invertDocument(14, *doc14, {}); _inv.removeDocument(11); _inv.removeDocument(13); _inserter_backend.reset(); @@ -232,11 +237,11 @@ TEST_F(DocumentInverterTest, require_that_abort_pending_doc_works) "w=j,a=14", _inserter_backend.toStr()); - _inv.invertDocument(10, *doc10); - _inv.invertDocument(11, *doc11); - _inv.invertDocument(12, *doc12); - _inv.invertDocument(13, *doc13); - _inv.invertDocument(14, *doc14); + _inv.invertDocument(10, *doc10, {}); + _inv.invertDocument(11, *doc11, {}); + _inv.invertDocument(12, *doc12, {}); + _inv.invertDocument(13, *doc13, {}); + _inv.invertDocument(14, *doc14, {}); _inv.removeDocument(11); _inv.removeDocument(12); _inv.removeDocument(13); @@ -256,7 +261,8 @@ TEST_F(DocumentInverterTest, require_that_mix_of_add_and_remove_works) _inv.getInverter(0)->remove("c", 9); _inv.getInverter(0)->remove("d", 10); _inv.getInverter(0)->remove("z", 12); - _inv.invertDocument(10, *makeDoc10(_b)); + auto doc10 = makeDoc10(_b); + _inv.invertDocument(10, *doc10, {}); pushDocuments(); EXPECT_EQ("f=0,w=a,a=10,r=11," "w=b,a=10," @@ -268,7 +274,8 @@ TEST_F(DocumentInverterTest, require_that_mix_of_add_and_remove_works) TEST_F(DocumentInverterTest, require_that_empty_document_can_be_inverted) { - _inv.invertDocument(15, *makeDoc15(_b)); + auto doc15 = makeDoc15(_b); + _inv.invertDocument(15, *doc15, {}); pushDocuments(); EXPECT_EQ("", _inserter_backend.toStr()); diff --git a/searchlib/src/tests/memoryindex/field_index/field_index_test.cpp b/searchlib/src/tests/memoryindex/field_index/field_index_test.cpp index 7b52eec78a6..ca30fe2d35e 100644 --- a/searchlib/src/tests/memoryindex/field_index/field_index_test.cpp +++ b/searchlib/src/tests/memoryindex/field_index/field_index_test.cpp @@ -956,7 +956,7 @@ TEST_F(BasicInverterTest, require_that_inversion_is_working) addStr("a").addStr("b").addStr("c").addStr("d"). endField(); doc = _b.endDocument(); - _inv.invertDocument(10, *doc); + _inv.invertDocument(10, *doc, {}); myPushDocument(_inv); _b.startDocument("id:ns:searchdocument::20"); @@ -964,7 +964,7 @@ TEST_F(BasicInverterTest, require_that_inversion_is_working) addStr("a").addStr("a").addStr("b").addStr("c").addStr("d"). endField(); doc = _b.endDocument(); - _inv.invertDocument(20, *doc); + _inv.invertDocument(20, *doc, {}); myPushDocument(_inv); _b.startDocument("id:ns:searchdocument::30"); @@ -993,7 +993,7 @@ TEST_F(BasicInverterTest, require_that_inversion_is_working) endElement(). endField(); doc = _b.endDocument(); - _inv.invertDocument(30, *doc); + _inv.invertDocument(30, *doc, {}); myPushDocument(_inv); _b.startDocument("id:ns:searchdocument::40"); @@ -1002,7 +1002,7 @@ TEST_F(BasicInverterTest, require_that_inversion_is_working) addStr("e").addStr("f"). endField(); doc = _b.endDocument(); - _inv.invertDocument(40, *doc); + _inv.invertDocument(40, *doc, {}); myPushDocument(_inv); _b.startDocument("id:ns:searchdocument::999"); @@ -1030,7 +1030,7 @@ TEST_F(BasicInverterTest, require_that_inversion_is_working) endField(); doc = _b.endDocument(); for (uint32_t docId = 10000; docId < 20000; ++docId) { - _inv.invertDocument(docId, *doc); + _inv.invertDocument(docId, *doc, {}); myPushDocument(_inv); } @@ -1144,13 +1144,13 @@ TEST_F(BasicInverterTest, require_that_inverter_handles_remove_via_document_remo _b.startIndexField("f0").addStr("a").addStr("b").endField(); _b.startIndexField("f1").addStr("a").addStr("c").endField(); Document::UP doc1 = _b.endDocument(); - _inv.invertDocument(1, *doc1.get()); + _inv.invertDocument(1, *doc1.get(), {}); myPushDocument(_inv); _b.startDocument("id:ns:searchdocument::2"); _b.startIndexField("f0").addStr("b").addStr("c").endField(); Document::UP doc2 = _b.endDocument(); - _inv.invertDocument(2, *doc2.get()); + _inv.invertDocument(2, *doc2.get(), {}); myPushDocument(_inv); EXPECT_TRUE(assertPostingList("[1]", find("a", 0))); @@ -1308,7 +1308,7 @@ TEST_F(UriInverterTest, require_that_uri_indexing_is_working) endElement(). endField(); doc = _b.endDocument(); - _inv.invertDocument(10, *doc); + _inv.invertDocument(10, *doc, {}); myPushDocument(_inv); SimpleMatchData match_data; @@ -1381,7 +1381,7 @@ TEST_F(CjkInverterTest, require_that_cjk_indexing_is_working) setAutoSpace(true). endField(); doc = _b.endDocument(); - _inv.invertDocument(10, *doc); + _inv.invertDocument(10, *doc, {}); myPushDocument(_inv); SimpleMatchData match_data; diff --git a/searchlib/src/tests/memoryindex/memory_index/memory_index_test.cpp b/searchlib/src/tests/memoryindex/memory_index/memory_index_test.cpp index 20cfb045081..c806a1b02ac 100644 --- a/searchlib/src/tests/memoryindex/memory_index/memory_index_test.cpp +++ b/searchlib/src/tests/memoryindex/memory_index/memory_index_test.cpp @@ -107,7 +107,7 @@ struct Index { Document::UP commit() { closeField(); Document::UP d = builder.endDocument(); - index.insertDocument(docid, *d); + index.insertDocument(docid, *d, {}); internalSyncCommit(); return d; } diff --git a/searchlib/src/tests/sortresults/sorttest.cpp b/searchlib/src/tests/sortresults/sorttest.cpp index cd892800ca5..bbd6d0b72ce 100644 --- a/searchlib/src/tests/sortresults/sorttest.cpp +++ b/searchlib/src/tests/sortresults/sorttest.cpp @@ -41,17 +41,17 @@ test_sort(unsigned int caseNum, unsigned int n, unsigned int ntop) } FastS_SortResults(array, n, ntop); - minmax = array[ntop - 1]._rankValue; + minmax = array[ntop - 1].getRank(); for(i = 0; i < n; i++) { if (i < ntop && i > 0 - && array[i]._rankValue > array[i - 1]._rankValue) { + && array[i].getRank() > array[i - 1].getRank()) { printf("ERROR: rank(%d) > rank(%d)\n", i, i - 1); ok = false; break; } if (i >= ntop && - array[i]._rankValue > minmax) { + array[i].getRank() > minmax) { printf("ERROR: rank(%d) > rank(%d)\n", i, ntop - 1); ok = false; diff --git a/searchlib/src/tests/sortspec/multilevelsort.cpp b/searchlib/src/tests/sortspec/multilevelsort.cpp index f438fce0e7f..576e1d1336c 100644 --- a/searchlib/src/tests/sortspec/multilevelsort.cpp +++ b/searchlib/src/tests/sortspec/multilevelsort.cpp @@ -275,21 +275,21 @@ MultilevelSortTest::sortAndCheck(const std::vector<Spec> &spec, uint32_t num, for (uint32_t j = 0; j < spec.size(); ++j) { int cmp = 0; if (spec[j]._type == RANK) { - if (hits[i]._rankValue < hits[i+1]._rankValue) { + if (hits[i].getRank() < hits[i+1].getRank()) { cmp = -1; - } else if (hits[i]._rankValue > hits[i+1]._rankValue) { + } else if (hits[i].getRank() > hits[i+1].getRank()) { cmp = 1; } } else if (spec[j]._type == DOCID) { - if (hits[i]._docId < hits[i+1]._docId) { + if (hits[i].getDocId() < hits[i+1].getDocId()) { cmp = -1; - } else if (hits[i]._docId > hits[i+1]._docId) { + } else if (hits[i].getDocId() > hits[i+1].getDocId()) { cmp = 1; } } else { AttributeVector *av = vec[spec[j]._name].get(); cmp = compare(av, spec[j]._type, - hits[i]._docId, hits[i+1]._docId); + hits[i].getDocId(), hits[i+1].getDocId()); } if (spec[j]._asc) { EXPECT_TRUE(cmp <= 0); diff --git a/searchlib/src/vespa/searchlib/aggregation/grouping.cpp b/searchlib/src/vespa/searchlib/aggregation/grouping.cpp index 68098d6c35a..f373b5fc0b3 100644 --- a/searchlib/src/vespa/searchlib/aggregation/grouping.cpp +++ b/searchlib/src/vespa/searchlib/aggregation/grouping.cpp @@ -205,13 +205,13 @@ void Grouping::postProcess() void Grouping::aggregateWithoutClock(const RankedHit * rankedHit, unsigned int len) { for(unsigned int i(0); i < len; i++) { - aggregate(rankedHit[i]._docId, rankedHit[i]._rankValue); + aggregate(rankedHit[i].getDocId(), rankedHit[i].getRank()); } } void Grouping::aggregateWithClock(const RankedHit * rankedHit, unsigned int len) { for(unsigned int i(0); (i < len) && !hasExpired(); i++) { - aggregate(rankedHit[i]._docId, rankedHit[i]._rankValue); + aggregate(rankedHit[i].getDocId(), rankedHit[i].getRank()); } } diff --git a/searchlib/src/vespa/searchlib/attribute/loadedenumvalue.h b/searchlib/src/vespa/searchlib/attribute/loadedenumvalue.h index b31e726b103..fdf9ab624ad 100644 --- a/searchlib/src/vespa/searchlib/attribute/loadedenumvalue.h +++ b/searchlib/src/vespa/searchlib/attribute/loadedenumvalue.h @@ -29,7 +29,7 @@ public: uint64_t operator()(const LoadedEnumAttribute &v) { - return (static_cast<uint64_t>(v._enum) << 32) | v._docId; + return (static_cast<uint64_t>(v._enum) << 32) | v.getDocId(); } }; diff --git a/searchlib/src/vespa/searchlib/attribute/loadedvalue.h b/searchlib/src/vespa/searchlib/attribute/loadedvalue.h index 701ccdc902c..b8f938838d2 100644 --- a/searchlib/src/vespa/searchlib/attribute/loadedvalue.h +++ b/searchlib/src/vespa/searchlib/attribute/loadedvalue.h @@ -103,9 +103,10 @@ public: T _value; uint32_t _eidx; }; + uint32_t _docId; uint32_t _idx; - vespalib::datastore::EntryRef _pidx; + vespalib::datastore::EntryRef _pidx; private: int32_t _weight; Value _value; diff --git a/searchlib/src/vespa/searchlib/common/CMakeLists.txt b/searchlib/src/vespa/searchlib/common/CMakeLists.txt index 6495fa8561d..73c8999520b 100644 --- a/searchlib/src/vespa/searchlib/common/CMakeLists.txt +++ b/searchlib/src/vespa/searchlib/common/CMakeLists.txt @@ -25,6 +25,7 @@ vespa_add_library(searchlib_common OBJECT packets.cpp partialbitvector.cpp resultset.cpp + schedule_sequenced_task_callback.cpp serialnumfileheadercontext.cpp sort.cpp sortdata.cpp diff --git a/searchlib/src/vespa/searchlib/common/allocatedbitvector.cpp b/searchlib/src/vespa/searchlib/common/allocatedbitvector.cpp index 80c6549499d..d428ce52953 100644 --- a/searchlib/src/vespa/searchlib/common/allocatedbitvector.cpp +++ b/searchlib/src/vespa/searchlib/common/allocatedbitvector.cpp @@ -157,8 +157,9 @@ AllocatedBitVector::grow(Index newSize, Index newCapacity) setSize(newSize); clearIntervalNoInvalidation(clearRange); } else { - clearInterval(newSize, size()); + clearIntervalNoInvalidation(Range(newSize, size())); setSize(newSize); + updateCount(); } } return ret; diff --git a/searchlib/src/vespa/searchlib/common/rankedhit.h b/searchlib/src/vespa/searchlib/common/rankedhit.h index d5c32a263aa..2d1ec0920dc 100644 --- a/searchlib/src/vespa/searchlib/common/rankedhit.h +++ b/searchlib/src/vespa/searchlib/common/rankedhit.h @@ -11,8 +11,7 @@ namespace search { struct RankedHit { RankedHit() noexcept : _docId(0), _rankValue(zero_rank_value) { } RankedHit(unsigned int docId, HitRank rank = zero_rank_value) noexcept : _docId(docId), _rankValue(rank) { } - unsigned int getDocId() const { return _docId & 0x7fffffff; } - bool hasMore() const { return _docId & 0x80000000; } + unsigned int getDocId() const { return _docId; } HitRank getRank() const { return _rankValue; } //:private unsigned int _docId; diff --git a/searchlib/src/vespa/searchlib/common/schedule_sequenced_task_callback.cpp b/searchlib/src/vespa/searchlib/common/schedule_sequenced_task_callback.cpp new file mode 100644 index 00000000000..6001770286e --- /dev/null +++ b/searchlib/src/vespa/searchlib/common/schedule_sequenced_task_callback.cpp @@ -0,0 +1,22 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "schedule_sequenced_task_callback.h" + +namespace search { + +ScheduleSequencedTaskCallback::ScheduleSequencedTaskCallback(vespalib::ISequencedTaskExecutor& executor, + vespalib::ISequencedTaskExecutor::ExecutorId id, + std::unique_ptr<vespalib::Executor::Task> task) noexcept + : _executor(executor), + _id(id), + _task(std::move(task)) +{ +} + + +ScheduleSequencedTaskCallback::~ScheduleSequencedTaskCallback() +{ + _executor.executeTask(_id, std::move(_task)); +} + +} diff --git a/searchlib/src/vespa/searchlib/common/schedule_sequenced_task_callback.h b/searchlib/src/vespa/searchlib/common/schedule_sequenced_task_callback.h new file mode 100644 index 00000000000..602ef1354e0 --- /dev/null +++ b/searchlib/src/vespa/searchlib/common/schedule_sequenced_task_callback.h @@ -0,0 +1,28 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include "vespa/vespalib/util/idestructorcallback.h" +#include <vespa/vespalib/util/isequencedtaskexecutor.h> + +namespace search { + +/** + * Class that schedules a sequenced task when instance is + * destroyed. Typically a shared pointer to an instance is passed + * around to multiple worker threads that performs portions of a + * larger task before dropping the shared pointer, triggering the + * callback when all worker threads have completed. + */ +class ScheduleSequencedTaskCallback : public vespalib::IDestructorCallback +{ + vespalib::ISequencedTaskExecutor& _executor; + vespalib::ISequencedTaskExecutor::ExecutorId _id; + std::unique_ptr<vespalib::Executor::Task> _task; +public: + ScheduleSequencedTaskCallback(vespalib::ISequencedTaskExecutor& executor, + vespalib::ISequencedTaskExecutor::ExecutorId id, + std::unique_ptr<vespalib::Executor::Task> task) noexcept; + ~ScheduleSequencedTaskCallback() override; +}; + +} // namespace search diff --git a/searchlib/src/vespa/searchlib/common/sortresults.cpp b/searchlib/src/vespa/searchlib/common/sortresults.cpp index 7a54de708d0..7510ae162ce 100644 --- a/searchlib/src/vespa/searchlib/common/sortresults.cpp +++ b/searchlib/src/vespa/searchlib/common/sortresults.cpp @@ -51,7 +51,7 @@ FastS_insertion_sort(RankedHit a[], uint32_t n) for (i=1; i<n ; i++) { swap = a[i]; j = i; - while (R(swap._rankValue) > R(a[j-1]._rankValue)) { + while (R(swap.getRank()) > R(a[j-1].getRank())) { a[j] = a[j-1]; if (!(--j)) break;; } @@ -74,13 +74,13 @@ FastS_radixsort(RankedHit a[], uint32_t n, uint32_t ntop) memset(cnt, 0, 256*sizeof(uint32_t)); // Count occurrences [NB: will fail with n < 3] for(i = 0; i < n - 3; i += 4) { - cnt[(R(a[i]._rankValue) >> SHIFT) & 0xFF]++; - cnt[(R(a[i + 1]._rankValue) >> SHIFT) & 0xFF]++; - cnt[(R(a[i + 2]._rankValue) >> SHIFT) & 0xFF]++; - cnt[(R(a[i + 3]._rankValue) >> SHIFT) & 0xFF]++; + cnt[(R(a[i].getRank()) >> SHIFT) & 0xFF]++; + cnt[(R(a[i + 1].getRank()) >> SHIFT) & 0xFF]++; + cnt[(R(a[i + 2].getRank()) >> SHIFT) & 0xFF]++; + cnt[(R(a[i + 3].getRank()) >> SHIFT) & 0xFF]++; } for(; i < n; i++) - cnt[(R(a[i]._rankValue) >> SHIFT) & 0xFF]++; + cnt[(R(a[i].getRank()) >> SHIFT) & 0xFF]++; // Accumulate cnt positions sorted = (cnt[0]==n); @@ -109,14 +109,14 @@ FastS_radixsort(RankedHit a[], uint32_t n, uint32_t ntop) // Grab first element to move j = ptr[i]; swap = a[j]; - k = (R(swap._rankValue) >> SHIFT) & 0xFF; + k = (R(swap.getRank()) >> SHIFT) & 0xFF; // Swap into correct class until cycle completed if (i!=k) { do { temp = a[ptr[k]]; a[ptr[k]++] = swap; - k = (R((swap = temp)._rankValue) >> SHIFT) & 0xFF; + k = (R((swap = temp).getRank()) >> SHIFT) & 0xFF; remain--; } while (i!=k); // Place last element in cycle @@ -265,11 +265,11 @@ FastS_SortSpec::initSortData(const RankedHit *hits, uint32_t n) written = sizeof(hits->_docId) + sizeof(_partitionId); break; case ASC_RANK: - serializeForSort<convertForSort<search::HitRank, true> >(hits[i]._rankValue, mySortData); + serializeForSort<convertForSort<search::HitRank, true> >(hits[i].getRank(), mySortData); written = sizeof(hits->_rankValue); break; case DESC_RANK: - serializeForSort<convertForSort<search::HitRank, false> >(hits[i]._rankValue, mySortData); + serializeForSort<convertForSort<search::HitRank, false> >(hits[i].getRank(), mySortData); written = sizeof(hits->_rankValue); break; case ASC_VECTOR: diff --git a/searchlib/src/vespa/searchlib/fef/termfieldmatchdata.cpp b/searchlib/src/vespa/searchlib/fef/termfieldmatchdata.cpp index 77c2aa3072a..ea278ebf607 100644 --- a/searchlib/src/vespa/searchlib/fef/termfieldmatchdata.cpp +++ b/searchlib/src/vespa/searchlib/fef/termfieldmatchdata.cpp @@ -18,7 +18,7 @@ TermFieldMatchData::TermFieldMatchData() : } TermFieldMatchData::TermFieldMatchData(const TermFieldMatchData & rhs) : - _docId(rhs._docId), + _docId(rhs.getDocId()), _fieldId(rhs._fieldId), _flags(rhs._flags), _sz(0), diff --git a/searchlib/src/vespa/searchlib/memoryindex/CMakeLists.txt b/searchlib/src/vespa/searchlib/memoryindex/CMakeLists.txt index e845201f5c6..34ac7d8e905 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/CMakeLists.txt +++ b/searchlib/src/vespa/searchlib/memoryindex/CMakeLists.txt @@ -1,6 +1,7 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_add_library(searchlib_memoryindex OBJECT SOURCES + bundled_fields_context.cpp compact_words_store.cpp document_inverter.cpp document_inverter_collection.cpp @@ -11,9 +12,14 @@ vespa_add_library(searchlib_memoryindex OBJECT field_index_collection.cpp field_index_remover.cpp field_inverter.cpp + invert_context.cpp + invert_task.cpp memory_index.cpp ordered_field_index_inserter.cpp posting_iterator.cpp + push_context.cpp + push_task.cpp + remove_task.cpp url_field_inverter.cpp word_store.cpp DEPENDS diff --git a/searchlib/src/vespa/searchlib/memoryindex/bundled_fields_context.cpp b/searchlib/src/vespa/searchlib/memoryindex/bundled_fields_context.cpp new file mode 100644 index 00000000000..af7e19ee20d --- /dev/null +++ b/searchlib/src/vespa/searchlib/memoryindex/bundled_fields_context.cpp @@ -0,0 +1,28 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "bundled_fields_context.h" + +namespace search::memoryindex { + +BundledFieldsContext::BundledFieldsContext(vespalib::ISequencedTaskExecutor::ExecutorId id) + : _id(id), + _fields(), + _uri_fields() +{ +} + +BundledFieldsContext::~BundledFieldsContext() = default; + +void +BundledFieldsContext::add_field(uint32_t field_id) +{ + _fields.emplace_back(field_id); +} + +void +BundledFieldsContext::add_uri_field(uint32_t uri_field_id) +{ + _uri_fields.emplace_back(uri_field_id); +} + +} diff --git a/searchlib/src/vespa/searchlib/memoryindex/bundled_fields_context.h b/searchlib/src/vespa/searchlib/memoryindex/bundled_fields_context.h new file mode 100644 index 00000000000..fb1a68d7273 --- /dev/null +++ b/searchlib/src/vespa/searchlib/memoryindex/bundled_fields_context.h @@ -0,0 +1,31 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vespa/vespalib/util/isequencedtaskexecutor.h> + +namespace search::memoryindex { + +/* + * Base class for PushContext and InvertContext, with mapping to + * the fields and uri fields handled by this context. Fields using + * the same thread appear in the same context. + */ +class BundledFieldsContext +{ + vespalib::ISequencedTaskExecutor::ExecutorId _id; + std::vector<uint32_t> _fields; + std::vector<uint32_t> _uri_fields; +protected: + BundledFieldsContext(vespalib::ISequencedTaskExecutor::ExecutorId id); + ~BundledFieldsContext(); +public: + void add_field(uint32_t field_id); + void add_uri_field(uint32_t uri_field_id); + void set_id(vespalib::ISequencedTaskExecutor::ExecutorId id) { _id = id; } + vespalib::ISequencedTaskExecutor::ExecutorId get_id() const noexcept { return _id; } + const std::vector<uint32_t>& get_fields() const noexcept { return _fields; } + const std::vector<uint32_t>& get_uri_fields() const noexcept { return _uri_fields; } +}; + +} diff --git a/searchlib/src/vespa/searchlib/memoryindex/document_inverter.cpp b/searchlib/src/vespa/searchlib/memoryindex/document_inverter.cpp index 879942ea5d7..fdb2de8fb59 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/document_inverter.cpp +++ b/searchlib/src/vespa/searchlib/memoryindex/document_inverter.cpp @@ -4,7 +4,11 @@ #include "document_inverter_context.h" #include "i_field_index_collection.h" #include "field_inverter.h" +#include "invert_task.h" +#include "push_task.h" +#include "remove_task.h" #include "url_field_inverter.h" +#include <vespa/searchlib/common/schedule_sequenced_task_callback.h> #include <vespa/vespalib/util/isequencedtaskexecutor.h> #include <vespa/vespalib/util/retain_guard.h> @@ -12,6 +16,7 @@ namespace search::memoryindex { using document::Document; using index::Schema; +using search::ScheduleSequencedTaskCallback; using search::index::FieldLengthCalculator; using vespalib::ISequencedTaskExecutor; using vespalib::RetainGuard; @@ -49,33 +54,18 @@ DocumentInverter::DocumentInverter(DocumentInverterContext& context) DocumentInverter::~DocumentInverter() { - _context.get_invert_threads().sync_all(); - _context.get_push_threads().sync_all(); + wait_for_zero_ref_count(); } void -DocumentInverter::invertDocument(uint32_t docId, const Document &doc) +DocumentInverter::invertDocument(uint32_t docId, const Document &doc, OnWriteDoneType on_write_done) { - // Might want to batch inverters as we do for attributes - _context.set_data_type(doc); - auto& schema_index_fields = _context.get_schema_index_fields(); auto& invert_threads = _context.get_invert_threads(); - for (uint32_t fieldId : schema_index_fields._textFields) { - auto fv = _context.get_field_value(doc, fieldId); - FieldInverter *inverter = _inverters[fieldId].get(); - invert_threads.execute(fieldId,[inverter, docId, fv(std::move(fv))]() { - inverter->invertField(docId, fv); - }); - } - uint32_t urlId = 0; - for (const auto & fi : schema_index_fields._uriFields) { - uint32_t fieldId = fi._all; - auto fv = _context.get_field_value(doc, fieldId); - UrlFieldInverter *inverter = _urlInverters[urlId].get(); - invert_threads.execute(fieldId,[inverter, docId, fv(std::move(fv))]() { - inverter->invertField(docId, fv); - }); - ++urlId; + auto& invert_contexts = _context.get_invert_contexts(); + for (auto& invert_context : invert_contexts) { + auto id = invert_context.get_id(); + auto task = std::make_unique<InvertTask>(_context, invert_context, _inverters, _urlInverters, docId, doc, on_write_done); + invert_threads.executeTask(id, std::move(task)); } } @@ -88,73 +78,36 @@ DocumentInverter::removeDocument(uint32_t docId) { void DocumentInverter::removeDocuments(LidVector lids) { - // Might want to batch inverters as we do for attributes - auto& schema_index_fields = _context.get_schema_index_fields(); auto& invert_threads = _context.get_invert_threads(); - for (uint32_t fieldId : schema_index_fields._textFields) { - FieldInverter *inverter = _inverters[fieldId].get(); - invert_threads.execute(fieldId, [inverter, lids]() { - for (uint32_t lid : lids) { - inverter->removeDocument(lid); - } - }); - } - uint32_t urlId = 0; - for (const auto & fi : schema_index_fields._uriFields) { - uint32_t fieldId = fi._all; - UrlFieldInverter *inverter = _urlInverters[urlId].get(); - invert_threads.execute(fieldId, [inverter, lids]() { - for (uint32_t lid : lids) { - inverter->removeDocument(lid); - } - }); - ++urlId; + auto& invert_contexts = _context.get_invert_contexts(); + for (auto& invert_context : invert_contexts) { + auto id = invert_context.get_id(); + auto task = std::make_unique<RemoveTask>(invert_context, _inverters, _urlInverters, lids); + invert_threads.executeTask(id, std::move(task)); } } -namespace { - -template <typename Inverter> -void push_documents_helper(ISequencedTaskExecutor& invert_threads, - ISequencedTaskExecutor& push_threads, - Inverter &inverter, - uint32_t field_id, - std::shared_ptr<vespalib::IDestructorCallback> on_write_done, - std::shared_ptr<RetainGuard> retain) -{ - auto invert_id = invert_threads.getExecutorId(field_id); - auto push_id = push_threads.getExecutorId(field_id); - invert_threads.execute(invert_id, - [&push_threads, push_id, &inverter, retain(std::move(retain)), on_write_done(std::move(on_write_done))] () mutable - { - push_threads.execute(push_id, - [&inverter, retain(std::move(retain)), on_write_done(std::move(on_write_done))]() - { - inverter.applyRemoves(); - inverter.pushDocuments(); - }); - }); -} - -} - void -DocumentInverter::pushDocuments(const std::shared_ptr<vespalib::IDestructorCallback> &onWriteDone) +DocumentInverter::pushDocuments(OnWriteDoneType on_write_done) { auto retain = std::make_shared<RetainGuard>(_ref_count); - auto& schema_index_fields = _context.get_schema_index_fields(); - auto& invert_threads = _context.get_invert_threads(); + using PushTasks = std::vector<std::shared_ptr<ScheduleSequencedTaskCallback>>; + PushTasks all_push_tasks; auto& push_threads = _context.get_push_threads(); - for (uint32_t field_id : schema_index_fields._textFields) { - auto& inverter = *_inverters[field_id]; - push_documents_helper(invert_threads, push_threads, inverter, field_id, onWriteDone, retain); + auto& push_contexts = _context.get_push_contexts(); + for (auto& push_context : push_contexts) { + auto task = std::make_unique<PushTask>(push_context, _inverters, _urlInverters, on_write_done, retain); + all_push_tasks.emplace_back(std::make_shared<ScheduleSequencedTaskCallback>(push_threads, push_context.get_id(), std::move(task))); } - uint32_t uri_field_id = 0; - for (const auto& uri_field : schema_index_fields._uriFields) { - uint32_t field_id = uri_field._all; - auto& inverter = *_urlInverters[uri_field_id]; - push_documents_helper(invert_threads, push_threads, inverter, field_id, onWriteDone, retain); - ++uri_field_id; + auto& invert_threads = _context.get_invert_threads(); + auto& invert_contexts = _context.get_invert_contexts(); + for (auto& invert_context : invert_contexts) { + PushTasks push_tasks; + for (auto& pusher : invert_context.get_pushers()) { + assert(pusher < all_push_tasks.size()); + push_tasks.emplace_back(all_push_tasks[pusher]); + } + invert_threads.execute(invert_context.get_id(), [push_tasks(std::move(push_tasks))]() { }); } } diff --git a/searchlib/src/vespa/searchlib/memoryindex/document_inverter.h b/searchlib/src/vespa/searchlib/memoryindex/document_inverter.h index dc4fba5d6b0..d89bdad5bb8 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/document_inverter.h +++ b/searchlib/src/vespa/searchlib/memoryindex/document_inverter.h @@ -16,9 +16,7 @@ namespace document { class FieldValue; } -namespace vespalib { - class IDestructorCallback; -} +namespace vespalib { class IDestructorCallback; } namespace search::memoryindex { @@ -40,6 +38,7 @@ private: DocumentInverterContext& _context; using LidVector = std::vector<uint32_t>; + using OnWriteDoneType = const std::shared_ptr<vespalib::IDestructorCallback> &; std::vector<std::unique_ptr<FieldInverter>> _inverters; std::vector<std::unique_ptr<UrlFieldInverter>> _urlInverters; @@ -61,13 +60,13 @@ public: * This function is async: * For each field inverter a task for pushing the inverted documents to the corresponding field index * is added to the 'push threads' executor, then this function returns. - * All tasks hold a reference to the 'onWriteDone' callback, so when the last task is completed, + * All tasks hold a reference to the 'on_write_done' callback, so when the last task is completed, * the callback is destructed. * * NOTE: The caller of this function should sync the 'invert threads' executor first, * to ensure that inverting is completed before pushing starts. */ - void pushDocuments(const std::shared_ptr<vespalib::IDestructorCallback> &onWriteDone); + void pushDocuments(OnWriteDoneType on_write_done); /** * Invert (add) the given document. @@ -76,7 +75,7 @@ public: * For each text and uri field in the document a task for inverting and adding that * field (using a field inverter) is added to the 'invert threads' executor, then this function returns. **/ - void invertDocument(uint32_t docId, const document::Document &doc); + void invertDocument(uint32_t docId, const document::Document &doc, OnWriteDoneType on_write_done); /** * Remove the given document. diff --git a/searchlib/src/vespa/searchlib/memoryindex/document_inverter_context.cpp b/searchlib/src/vespa/searchlib/memoryindex/document_inverter_context.cpp index 8fea82229c8..20fd333442b 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/document_inverter_context.cpp +++ b/searchlib/src/vespa/searchlib/memoryindex/document_inverter_context.cpp @@ -1,50 +1,122 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "document_inverter_context.h" -#include <vespa/document/datatype/documenttype.h> -#include <vespa/document/fieldvalue/document.h> #include <cassert> - -#include <vespa/log/log.h> -LOG_SETUP(".memoryindex.document_inverter_context"); +#include <optional> namespace search::memoryindex { -using document::DataType; -using document::Document; -using document::DocumentType; -using document::Field; using vespalib::ISequencedTaskExecutor; +using index::SchemaIndexFields; -void -DocumentInverterContext::add_field(const DocumentType& doc_type, uint32_t fieldId) +namespace { + +template <typename Context> +void make_contexts(const SchemaIndexFields& schema_index_fields, ISequencedTaskExecutor& executor, std::vector<Context>& contexts) { - assert(fieldId < _indexed_fields.size()); - std::unique_ptr<Field> fp; - if ( ! doc_type.hasField(_schema.getIndexField(fieldId).getName())) { - LOG(error, - "Mismatch between documentdefinition and schema. " - "No field named '%s' from schema in document type '%s'", - _schema.getIndexField(fieldId).getName().c_str(), - doc_type.getName().c_str()); - } else { - fp = std::make_unique<Field>(doc_type.getField(_schema.getIndexField(fieldId).getName())); + using ExecutorId = ISequencedTaskExecutor::ExecutorId; + using IdMapping = std::vector<std::tuple<ExecutorId, bool, uint32_t>>; + IdMapping map; + for (uint32_t field_id : schema_index_fields._textFields) { + // TODO: Add bias when sharing sequenced task executor between document types + map.emplace_back(executor.getExecutorId(field_id), false, field_id); + } + uint32_t uri_field_id = 0; + for (auto& uri_field : schema_index_fields._uriFields) { + // TODO: Add bias when sharing sequenced task executor between document types + map.emplace_back(executor.getExecutorId(uri_field._all), true, uri_field_id); + ++uri_field_id; + } + std::sort(map.begin(), map.end()); + std::optional<ExecutorId> prev_id; + for (auto& entry : map) { + if (!prev_id.has_value() || prev_id.value() != std::get<0>(entry)) { + contexts.emplace_back(std::get<0>(entry)); + prev_id = std::get<0>(entry); + } + if (std::get<1>(entry)) { + contexts.back().add_uri_field(std::get<2>(entry)); + } else { + contexts.back().add_field(std::get<2>(entry)); + } } - _indexed_fields[fieldId] = std::move(fp); } -void -DocumentInverterContext::build_fields(const DocumentType& doc_type, const DataType *data_type) +void switch_to_alternate_ids(ISequencedTaskExecutor& executor, std::vector<PushContext>& contexts, uint32_t bias) { - _indexed_fields.clear(); - _indexed_fields.resize(_schema.getNumIndexFields()); - for (const auto & fi : _schema_index_fields._textFields) { - add_field(doc_type, fi); + for (auto& context : contexts) { + context.set_id(executor.get_alternate_executor_id(context.get_id(), bias)); } - for (const auto & fi : _schema_index_fields._uriFields) { - add_field(doc_type, fi._all); +} + +class PusherMapping { + std::vector<std::optional<uint32_t>> _pushers; +public: + PusherMapping(size_t size); + ~PusherMapping(); + + void add_mapping(const std::vector<uint32_t>& fields, uint32_t pusher_id) { + for (auto field_id : fields) { + assert(field_id < _pushers.size()); + auto& opt_pusher = _pushers[field_id]; + assert(!opt_pusher.has_value()); + opt_pusher = pusher_id; + } + } + + void use_mapping(const std::vector<uint32_t>& fields, std::vector<uint32_t>& pushers) { + for (auto field_id : fields) { + assert(field_id < _pushers.size()); + auto& opt_pusher = _pushers[field_id]; + assert(opt_pusher.has_value()); + pushers.emplace_back(opt_pusher.value()); + } } - _data_type = data_type; +}; + +PusherMapping::PusherMapping(size_t size) + : _pushers(size) +{ +} + +PusherMapping::~PusherMapping() = default; + +/* + * Connect contexts for inverting to contexts for pushing. If we use + * different sequenced task executors or adds different biases to the + * getExecutorId() argument (to enable double buffering) then contexts + * for inverting and contexts for pushing will bundle different sets + * of fields, preventing a 1:1 mapping. If we use the same sequenced + * task executor and drop double buffering then we can simplify this + * to a 1:1 mapping. + */ +void connect_contexts(std::vector<InvertContext>& invert_contexts, + const std::vector<PushContext>& push_contexts, + uint32_t num_fields, + uint32_t num_uri_fields) +{ + PusherMapping field_to_pusher(num_fields); + PusherMapping uri_field_to_pusher(num_uri_fields); + uint32_t pusher_id = 0; + for (auto& push_context : push_contexts) { + field_to_pusher.add_mapping(push_context.get_fields(), pusher_id); + uri_field_to_pusher.add_mapping(push_context.get_uri_fields(), pusher_id); + ++pusher_id; + } + std::vector<uint32_t> pushers; + for (auto& invert_context : invert_contexts) { + pushers.clear(); + field_to_pusher.use_mapping(invert_context.get_fields(), pushers); + uri_field_to_pusher.use_mapping(invert_context.get_uri_fields(), pushers); + std::sort(pushers.begin(), pushers.end()); + auto last = std::unique(pushers.begin(), pushers.end()); + pushers.erase(last, pushers.end()); + for (auto pusher : pushers) { + invert_context.add_pusher(pusher); + } + } +} + } DocumentInverterContext::DocumentInverterContext(const index::Schema& schema, @@ -52,35 +124,29 @@ DocumentInverterContext::DocumentInverterContext(const index::Schema& schema, ISequencedTaskExecutor &push_threads, IFieldIndexCollection& field_indexes) : _schema(schema), - _indexed_fields(), - _data_type(nullptr), _schema_index_fields(), _invert_threads(invert_threads), _push_threads(push_threads), - _field_indexes(field_indexes) + _field_indexes(field_indexes), + _invert_contexts(), + _push_contexts() { _schema_index_fields.setup(schema); + setup_contexts(); } DocumentInverterContext::~DocumentInverterContext() = default; void -DocumentInverterContext::set_data_type(const Document& doc) -{ - const DataType *data_type(doc.getDataType()); - if (_indexed_fields.empty() || _data_type != data_type) { - build_fields(doc.getType(), data_type); - } -} - -std::unique_ptr<document::FieldValue> -DocumentInverterContext::get_field_value(const Document& doc, uint32_t field_id) const +DocumentInverterContext::setup_contexts() { - const Field *const field(_indexed_fields[field_id].get()); - if (field != nullptr) { - return doc.getValue(*field); + make_contexts(_schema_index_fields, _invert_threads, _invert_contexts); + make_contexts(_schema_index_fields, _push_threads, _push_contexts); + if (&_invert_threads == &_push_threads) { + uint32_t bias = _schema_index_fields._textFields.size() + _schema_index_fields._uriFields.size(); + switch_to_alternate_ids(_push_threads, _push_contexts, bias); } - return {}; + connect_contexts(_invert_contexts, _push_contexts, _schema.getNumIndexFields(), _schema_index_fields._uriFields.size()); } } diff --git a/searchlib/src/vespa/searchlib/memoryindex/document_inverter_context.h b/searchlib/src/vespa/searchlib/memoryindex/document_inverter_context.h index 7330f4376ea..552def934c2 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/document_inverter_context.h +++ b/searchlib/src/vespa/searchlib/memoryindex/document_inverter_context.h @@ -3,19 +3,11 @@ #pragma once #include <vespa/searchlib/index/schema_index_fields.h> +#include "invert_context.h" +#include "push_context.h" #include <memory> #include <vector> -namespace document { -class DataType; -class Document; -class DocumentType; -class Field; -class FieldValue; -} - -namespace vespalib { class ISequencedTaskExecutor; } - namespace search::memoryindex { class IFieldIndexCollection; @@ -25,29 +17,27 @@ class IFieldIndexCollection; * rarely (type dependent data, wiring). */ class DocumentInverterContext { - using IndexedFields = std::vector<std::unique_ptr<document::Field>>; const index::Schema& _schema; - IndexedFields _indexed_fields; - const document::DataType* _data_type; index::SchemaIndexFields _schema_index_fields; vespalib::ISequencedTaskExecutor& _invert_threads; vespalib::ISequencedTaskExecutor& _push_threads; IFieldIndexCollection& _field_indexes; - void add_field(const document::DocumentType& doc_type, uint32_t fieldId); - void build_fields(const document::DocumentType& doc_type, const document::DataType* data_type); + std::vector<InvertContext> _invert_contexts; + std::vector<PushContext> _push_contexts; + void setup_contexts(); public: DocumentInverterContext(const index::Schema &schema, vespalib::ISequencedTaskExecutor &invert_threads, vespalib::ISequencedTaskExecutor &push_threads, IFieldIndexCollection& field_indexes); ~DocumentInverterContext(); - void set_data_type(const document::Document& doc); const index::Schema& get_schema() const noexcept { return _schema; } const index::SchemaIndexFields& get_schema_index_fields() const noexcept { return _schema_index_fields; } vespalib::ISequencedTaskExecutor& get_invert_threads() noexcept { return _invert_threads; } vespalib::ISequencedTaskExecutor& get_push_threads() noexcept { return _push_threads; } IFieldIndexCollection& get_field_indexes() noexcept { return _field_indexes; } - std::unique_ptr<document::FieldValue> get_field_value(const document::Document& doc, uint32_t field_id) const; + const std::vector<InvertContext>& get_invert_contexts() const noexcept { return _invert_contexts; } + const std::vector<PushContext>& get_push_contexts() const noexcept { return _push_contexts; } }; } diff --git a/searchlib/src/vespa/searchlib/memoryindex/field_index_remover.h b/searchlib/src/vespa/searchlib/memoryindex/field_index_remover.h index f8328d15289..429eea038c9 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/field_index_remover.h +++ b/searchlib/src/vespa/searchlib/memoryindex/field_index_remover.h @@ -21,6 +21,7 @@ private: struct WordFieldDocTuple { vespalib::datastore::EntryRef _wordRef; uint32_t _docId; + WordFieldDocTuple() noexcept : _wordRef(0), _docId(0) diff --git a/searchlib/src/vespa/searchlib/memoryindex/invert_context.cpp b/searchlib/src/vespa/searchlib/memoryindex/invert_context.cpp new file mode 100644 index 00000000000..1e6506bc8d5 --- /dev/null +++ b/searchlib/src/vespa/searchlib/memoryindex/invert_context.cpp @@ -0,0 +1,81 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "invert_context.h" +#include "document_inverter_context.h" +#include <vespa/document/datatype/documenttype.h> +#include <vespa/document/fieldvalue/document.h> +#include <vespa/searchlib/index/schema_index_fields.h> + +#include <vespa/log/log.h> +LOG_SETUP(".memoryindex.invert_context"); + +namespace search::memoryindex { + +using document::Document; +using document::DocumentType; +using document::Field; + +namespace { + +std::unique_ptr<document::Field> +get_field(const DocumentType& doc_type, const vespalib::string& name) +{ + std::unique_ptr<Field> fp; + if ( ! doc_type.hasField(name)) { + LOG(error, + "Mismatch between documentdefinition and schema. " + "No field named '%s' from schema in document type '%s'", + name.c_str(), + doc_type.getName().c_str()); + } else { + fp = std::make_unique<Field>(doc_type.getField(name)); + } + return fp; +} + +} + + +InvertContext::InvertContext(vespalib::ISequencedTaskExecutor::ExecutorId id) + : BundledFieldsContext(id), + _pushers(), + _document_fields(), + _document_uri_fields(), + _data_type(nullptr) +{ +} + +InvertContext::~InvertContext() = default; + +InvertContext::InvertContext(InvertContext&&) = default; + +void +InvertContext::add_pusher(uint32_t pusher_id) +{ + _pushers.emplace_back(pusher_id); +} + +void +InvertContext::set_data_type(const DocumentInverterContext &doc_inv_context, const Document& doc) const +{ + auto data_type(doc.getDataType()); + if (_data_type == data_type) { + return; + } + auto& doc_type(doc.getType()); + _document_fields.clear(); + auto& schema = doc_inv_context.get_schema(); + for (auto field_id : get_fields()) { + auto& name = schema.getIndexField(field_id).getName(); + _document_fields.emplace_back(get_field(doc_type, name)); + } + _document_uri_fields.clear(); + auto& schema_index_fields = doc_inv_context.get_schema_index_fields(); + for (auto uri_field_id : get_uri_fields()) { + auto& name = schema.getIndexField(schema_index_fields._uriFields[uri_field_id]._all).getName(); + _document_uri_fields.emplace_back(get_field(doc_type, name)); + } + _data_type = data_type; +} + +} diff --git a/searchlib/src/vespa/searchlib/memoryindex/invert_context.h b/searchlib/src/vespa/searchlib/memoryindex/invert_context.h new file mode 100644 index 00000000000..059fdb25d06 --- /dev/null +++ b/searchlib/src/vespa/searchlib/memoryindex/invert_context.h @@ -0,0 +1,50 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "bundled_fields_context.h" + +namespace document { +class DataType; +class Document; +class Field; +} + +namespace search::index { +class Schema; +class SchemaIndexFields; +} + +namespace search::memoryindex { + +class DocumentInverterContext; + +/* + * Context used by an InvertTask to invert a set of document fields + * into corresponding field inverters or by a RemoveTask to remove + * documents from a set of field inverters. + * + * It is also used by DocumentInverter::pushDocuments() to execute + * PushTask at the proper time (i.e. when all related InvertTask / + * RemoveTask operations have completed). + */ +class InvertContext : public BundledFieldsContext +{ + using IndexedFields = std::vector<std::unique_ptr<const document::Field>>; + std::vector<uint32_t> _pushers; + vespalib::string _document_field_names; + mutable IndexedFields _document_fields; + mutable IndexedFields _document_uri_fields; + mutable const document::DataType* _data_type; +public: + void add_pusher(uint32_t pusher_id); + InvertContext(vespalib::ISequencedTaskExecutor::ExecutorId id); + ~InvertContext(); + InvertContext(InvertContext&&); + const std::vector<uint32_t>& get_pushers() const noexcept { return _pushers; } + void set_data_type(const DocumentInverterContext& doc_inv_context, const document::Document& doc) const; + const IndexedFields& get_document_fields() const noexcept { return _document_fields; } + const IndexedFields& get_document_uri_fields() const noexcept { return _document_uri_fields; } +}; + +} diff --git a/searchlib/src/vespa/searchlib/memoryindex/invert_task.cpp b/searchlib/src/vespa/searchlib/memoryindex/invert_task.cpp new file mode 100644 index 00000000000..fb6e1328b8b --- /dev/null +++ b/searchlib/src/vespa/searchlib/memoryindex/invert_task.cpp @@ -0,0 +1,56 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "invert_task.h" +#include "document_inverter_context.h" +#include "field_inverter.h" +#include "invert_context.h" +#include "url_field_inverter.h" + +namespace search::memoryindex { + +using document::Document; +using document::Field; + +namespace { + +std::unique_ptr<document::FieldValue> +get_field_value(const Document& doc, const std::unique_ptr<const Field>& field) +{ + if (field) { + return doc.getValue(*field); + } + return {}; +} + +} + +InvertTask::InvertTask(const DocumentInverterContext& inv_context, const InvertContext& context, const std::vector<std::unique_ptr<FieldInverter>>& inverters, const std::vector<std::unique_ptr<UrlFieldInverter>>& uri_inverters, uint32_t lid, const document::Document& doc, OnWriteDoneType on_write_done) + : _inv_context(inv_context), + _context(context), + _inverters(inverters), + _uri_inverters(uri_inverters), + _doc(doc), + _lid(lid), + _on_write_done(on_write_done) +{ +} + +InvertTask::~InvertTask() = default; + +void +InvertTask::run() +{ + _context.set_data_type(_inv_context, _doc); + auto document_field_itr = _context.get_document_fields().begin(); + for (auto field_id : _context.get_fields()) { + _inverters[field_id]->invertField(_lid, get_field_value(_doc, *document_field_itr)); + ++document_field_itr; + } + auto document_uri_field_itr = _context.get_document_uri_fields().begin(); + for (auto uri_field_id : _context.get_uri_fields()) { + _uri_inverters[uri_field_id]->invertField(_lid, get_field_value(_doc, *document_uri_field_itr)); + ++document_uri_field_itr; + } +} + +} diff --git a/searchlib/src/vespa/searchlib/memoryindex/invert_task.h b/searchlib/src/vespa/searchlib/memoryindex/invert_task.h new file mode 100644 index 00000000000..b898deb5c16 --- /dev/null +++ b/searchlib/src/vespa/searchlib/memoryindex/invert_task.h @@ -0,0 +1,38 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vespa/vespalib/util/executor.h> +#include <vector> + +namespace document { class Document; } +namespace vespalib { class IDestructorCallback; } + +namespace search::memoryindex { + +class DocumentInverterContext; +class FieldInverter; +class InvertContext; +class UrlFieldInverter; + +/* + * Task to invert a set of document fields into related field + * inverters and uri field inverters. + */ +class InvertTask : public vespalib::Executor::Task +{ + using OnWriteDoneType = const std::shared_ptr<vespalib::IDestructorCallback> &; + const DocumentInverterContext& _inv_context; + const InvertContext& _context; + const std::vector<std::unique_ptr<FieldInverter>>& _inverters; + const std::vector<std::unique_ptr<UrlFieldInverter>>& _uri_inverters; + const document::Document& _doc; + uint32_t _lid; + std::remove_reference_t<OnWriteDoneType> _on_write_done; +public: + InvertTask(const DocumentInverterContext& inv_context, const InvertContext& context, const std::vector<std::unique_ptr<FieldInverter>>& inverters, const std::vector<std::unique_ptr<UrlFieldInverter>>& uri_inverters, uint32_t lid, const document::Document& doc, OnWriteDoneType on_write_done); + ~InvertTask() override; + void run() override; +}; + +} diff --git a/searchlib/src/vespa/searchlib/memoryindex/memory_index.cpp b/searchlib/src/vespa/searchlib/memoryindex/memory_index.cpp index 15cdef2f664..330320d5047 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/memory_index.cpp +++ b/searchlib/src/vespa/searchlib/memoryindex/memory_index.cpp @@ -72,14 +72,10 @@ MemoryIndex::MemoryIndex(const Schema& schema, { } -MemoryIndex::~MemoryIndex() -{ - _invertThreads.sync_all(); - _pushThreads.sync_all(); -} +MemoryIndex::~MemoryIndex() = default; void -MemoryIndex::insertDocument(uint32_t docId, const document::Document &doc) +MemoryIndex::insertDocument(uint32_t docId, const document::Document &doc, OnWriteDoneType on_write_done) { if (_frozen) { LOG(warning, "Memory index frozen: ignoring insert of document '%s'(%u): '%s'", @@ -88,7 +84,7 @@ MemoryIndex::insertDocument(uint32_t docId, const document::Document &doc) } updateMaxDocId(docId); auto& inverter = _inverters->get_active_inverter(); - inverter.invertDocument(docId, doc); + inverter.invertDocument(docId, doc, on_write_done); if (_indexedDocs.insert(docId).second) { incNumDocs(); } @@ -113,10 +109,10 @@ MemoryIndex::removeDocuments(LidVector lids) } void -MemoryIndex::commit(const std::shared_ptr<vespalib::IDestructorCallback> &onWriteDone) +MemoryIndex::commit(OnWriteDoneType on_write_done) { auto& inverter = _inverters->get_active_inverter(); - inverter.pushDocuments(onWriteDone); + inverter.pushDocuments(on_write_done); _inverters->switch_active_inverter(); } diff --git a/searchlib/src/vespa/searchlib/memoryindex/memory_index.h b/searchlib/src/vespa/searchlib/memoryindex/memory_index.h index 760a4ecfb0f..dc1b5d8060d 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/memory_index.h +++ b/searchlib/src/vespa/searchlib/memoryindex/memory_index.h @@ -43,6 +43,7 @@ class MemoryIndex : public queryeval::Searchable { private: using ISequencedTaskExecutor = vespalib::ISequencedTaskExecutor; using LidVector = std::vector<uint32_t>; + using OnWriteDoneType = const std::shared_ptr<vespalib::IDestructorCallback> &; index::Schema _schema; ISequencedTaskExecutor &_invertThreads; ISequencedTaskExecutor &_pushThreads; @@ -107,7 +108,7 @@ public: * If the document is already in the index, the old version will be removed first. * This function is async. commit() must be called for changes to take effect. */ - void insertDocument(uint32_t docId, const document::Document &doc); + void insertDocument(uint32_t docId, const document::Document &doc, OnWriteDoneType on_write_done); /** * Remove a document from the underlying field indexes. @@ -119,11 +120,9 @@ public: /** * Commits the inserts and removes since the last commit, making them searchable. * - * When commit is completed, 'onWriteDone' goes out of scope, scheduling completion callback. - * - * Callers can call pushThreads.sync() to wait for push completion. + * When commit is completed, 'on_write_done' goes out of scope, scheduling completion callback. */ - void commit(const std::shared_ptr<vespalib::IDestructorCallback> &onWriteDone); + void commit(OnWriteDoneType on_write_done); /** * Freeze this index. diff --git a/searchlib/src/vespa/searchlib/memoryindex/push_context.cpp b/searchlib/src/vespa/searchlib/memoryindex/push_context.cpp new file mode 100644 index 00000000000..5a4a773a6f5 --- /dev/null +++ b/searchlib/src/vespa/searchlib/memoryindex/push_context.cpp @@ -0,0 +1,14 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "push_context.h" + +namespace search::memoryindex { + +PushContext::PushContext(vespalib::ISequencedTaskExecutor::ExecutorId id) + : BundledFieldsContext(id) +{ +} + +PushContext::~PushContext() = default; + +} diff --git a/searchlib/src/vespa/searchlib/memoryindex/push_context.h b/searchlib/src/vespa/searchlib/memoryindex/push_context.h new file mode 100644 index 00000000000..0e96346837e --- /dev/null +++ b/searchlib/src/vespa/searchlib/memoryindex/push_context.h @@ -0,0 +1,20 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "bundled_fields_context.h" + +namespace search::memoryindex { + +/* + * Context for pushing inverted data to memory index structure for a set + * of fields and uri fields. Currently used by PushTask. + */ +class PushContext : public BundledFieldsContext +{ +public: + PushContext(vespalib::ISequencedTaskExecutor::ExecutorId id); + ~PushContext(); +}; + +} diff --git a/searchlib/src/vespa/searchlib/memoryindex/push_task.cpp b/searchlib/src/vespa/searchlib/memoryindex/push_task.cpp new file mode 100644 index 00000000000..b68e23bfe02 --- /dev/null +++ b/searchlib/src/vespa/searchlib/memoryindex/push_task.cpp @@ -0,0 +1,44 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "push_task.h" +#include "push_context.h" +#include "field_inverter.h" +#include "url_field_inverter.h" + +namespace search::memoryindex { + +namespace { + +template <typename Inverter> +void push_inverter(Inverter& inverter) +{ + inverter.applyRemoves(); + inverter.pushDocuments(); +} + +} + + +PushTask::PushTask(const PushContext& context, const std::vector<std::unique_ptr<FieldInverter>>& inverters, const std::vector<std::unique_ptr<UrlFieldInverter>>& uri_inverters, OnWriteDoneType on_write_done, std::shared_ptr<vespalib::RetainGuard> retain) + : _context(context), + _inverters(inverters), + _uri_inverters(uri_inverters), + _on_write_done(on_write_done), + _retain(std::move(retain)) +{ +} + +PushTask::~PushTask() = default; + +void +PushTask::run() +{ + for (auto field_id : _context.get_fields()) { + push_inverter(*_inverters[field_id]); + } + for (auto uri_field_id : _context.get_uri_fields()) { + push_inverter(*_uri_inverters[uri_field_id]); + } +} + +} diff --git a/searchlib/src/vespa/searchlib/memoryindex/push_task.h b/searchlib/src/vespa/searchlib/memoryindex/push_task.h new file mode 100644 index 00000000000..002b9334b78 --- /dev/null +++ b/searchlib/src/vespa/searchlib/memoryindex/push_task.h @@ -0,0 +1,37 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vespa/vespalib/util/executor.h> +#include <vector> + +namespace vespalib { +class IDestructorCallback; +class RetainGuard; +} + +namespace search::memoryindex { + +class FieldInverter; +class PushContext; +class UrlFieldInverter; + +/* + * Task to push inverted data from a set of field inverters and uri + * field inverters to to memory index structure. + */ +class PushTask : public vespalib::Executor::Task +{ + using OnWriteDoneType = const std::shared_ptr<vespalib::IDestructorCallback> &; + const PushContext& _context; + const std::vector<std::unique_ptr<FieldInverter>>& _inverters; + const std::vector<std::unique_ptr<UrlFieldInverter>>& _uri_inverters; + std::remove_reference_t<OnWriteDoneType> _on_write_done; + std::shared_ptr<vespalib::RetainGuard> _retain; +public: + PushTask(const PushContext& context, const std::vector<std::unique_ptr<FieldInverter>>& inverters, const std::vector<std::unique_ptr<UrlFieldInverter>>& uri_inverters, OnWriteDoneType on_write_done, std::shared_ptr<vespalib::RetainGuard> retain); + ~PushTask() override; + void run() override; +}; + +} diff --git a/searchlib/src/vespa/searchlib/memoryindex/remove_task.cpp b/searchlib/src/vespa/searchlib/memoryindex/remove_task.cpp new file mode 100644 index 00000000000..d19abd50274 --- /dev/null +++ b/searchlib/src/vespa/searchlib/memoryindex/remove_task.cpp @@ -0,0 +1,44 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "remove_task.h" +#include "document_inverter_context.h" +#include "field_inverter.h" +#include "invert_context.h" +#include "url_field_inverter.h" + +namespace search::memoryindex { + +namespace { + +template <typename Inverter> +void remove_documents(Inverter& inverter, const std::vector<uint32_t>& lids) +{ + for (auto lid : lids) { + inverter.removeDocument(lid); + } +} + +} + +RemoveTask::RemoveTask(const InvertContext& context, const std::vector<std::unique_ptr<FieldInverter>>& inverters, const std::vector<std::unique_ptr<UrlFieldInverter>>& uri_inverters, const std::vector<uint32_t>& lids) + : _context(context), + _inverters(inverters), + _uri_inverters(uri_inverters), + _lids(lids) +{ +} + +RemoveTask::~RemoveTask() = default; + +void +RemoveTask::run() +{ + for (auto field_id : _context.get_fields()) { + remove_documents(*_inverters[field_id], _lids); + } + for (auto uri_field_id : _context.get_uri_fields()) { + remove_documents(*_uri_inverters[uri_field_id], _lids); + } +} + +} diff --git a/searchlib/src/vespa/searchlib/memoryindex/remove_task.h b/searchlib/src/vespa/searchlib/memoryindex/remove_task.h new file mode 100644 index 00000000000..5eba4390752 --- /dev/null +++ b/searchlib/src/vespa/searchlib/memoryindex/remove_task.h @@ -0,0 +1,30 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vespa/vespalib/util/executor.h> +#include <vector> + +namespace search::memoryindex { + +class FieldInverter; +class InvertContext; +class UrlFieldInverter; + +/* + * Task to remove a document from a set of field inverters and uri + * field inverters. + */ +class RemoveTask : public vespalib::Executor::Task +{ + const InvertContext& _context; + const std::vector<std::unique_ptr<FieldInverter>>& _inverters; + const std::vector<std::unique_ptr<UrlFieldInverter>>& _uri_inverters; + std::vector<uint32_t> _lids; +public: + RemoveTask(const InvertContext& context, const std::vector<std::unique_ptr<FieldInverter>>& inverters, const std::vector<std::unique_ptr<UrlFieldInverter>>& uri_inverters, const std::vector<uint32_t>& lids); + ~RemoveTask() override; + void run() override; +}; + +} diff --git a/searchlib/src/vespa/searchlib/queryeval/hitcollector.cpp b/searchlib/src/vespa/searchlib/queryeval/hitcollector.cpp index b2b1d49bae9..3293019e538 100644 --- a/searchlib/src/vespa/searchlib/queryeval/hitcollector.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/hitcollector.cpp @@ -195,7 +195,7 @@ mergeHitsIntoResultSet(const std::vector<HitCollector::Hit> &hits, ResultSet &re uint32_t rhCur(0); uint32_t rhEnd(result.getArrayUsed()); for (const auto &hit : hits) { - while (rhCur != rhEnd && result[rhCur]._docId != hit.first) { + while (rhCur != rhEnd && result[rhCur].getDocId() != hit.first) { // just set the iterators right ++rhCur; } diff --git a/searchlib/src/vespa/searchlib/test/fakedata/fakememtreeocc.h b/searchlib/src/vespa/searchlib/test/fakedata/fakememtreeocc.h index a94f6087a0d..d0a75930ed5 100644 --- a/searchlib/src/vespa/searchlib/test/fakedata/fakememtreeocc.h +++ b/searchlib/src/vespa/searchlib/test/fakedata/fakememtreeocc.h @@ -77,8 +77,8 @@ public: bool operator<(const PendingOp &rhs) const { if (_wordIdx != rhs._wordIdx) return _wordIdx < rhs._wordIdx; - if (_docId != rhs._docId) - return _docId < rhs._docId; + if (_docId != rhs.getDocId()) + return _docId < rhs.getDocId(); return _seq < rhs._seq; } }; diff --git a/security-utils/src/main/java/com/yahoo/security/tls/TlsContext.java b/security-utils/src/main/java/com/yahoo/security/tls/TlsContext.java index 7923249a16e..b222c8664cc 100644 --- a/security-utils/src/main/java/com/yahoo/security/tls/TlsContext.java +++ b/security-utils/src/main/java/com/yahoo/security/tls/TlsContext.java @@ -24,17 +24,18 @@ public interface TlsContext extends AutoCloseable { * For TLSv1.2 we only allow RSA and ECDSA with ephemeral key exchange and GCM. * For TLSv1.3 we allow the DEFAULT group ciphers. * Note that we _only_ allow AEAD ciphers for either TLS version. + * + * TODO(bjorncs) Add new ciphers once migrated to JDK-17 (also available in 11.0.13): + * - TLS_CHACHA20_POLY1305_SHA256, TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256, TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256 */ Set<String> ALLOWED_CIPHER_SUITES = Collections.unmodifiableSet(new HashSet<>(Arrays.asList( "TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384", "TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384", - "TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256", // Java 12 - "TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256", "TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256", "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256", "TLS_AES_128_GCM_SHA256", // TLSv1.3 - "TLS_AES_256_GCM_SHA384", // TLSv1.3 - "TLS_CHACHA20_POLY1305_SHA256"))); // TLSv1.3, Java 12 + "TLS_AES_256_GCM_SHA384" // TLSv1.3 + ))); // TODO Enable TLSv1.3 after upgrading to JDK 17 Set<String> ALLOWED_PROTOCOLS = Collections.singleton("TLSv1.2"); diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModelManager.java b/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModelManager.java index 70f93537387..7087995aa2e 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModelManager.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModelManager.java @@ -102,7 +102,7 @@ public class DuperModelManager implements DuperModelProvider, DuperModelInfraApi lockedRunnable(() -> { if (!superModelIsComplete) { superModelIsComplete = true; - logger.log(Level.INFO, "All bootstrap tenant applications have been activated"); + logger.log(Level.FINE, "All bootstrap tenant applications have been activated"); maybeSetDuperModelAsComplete(); } }); diff --git a/staging_vespalib/src/tests/sequencedtaskexecutor/sequencedtaskexecutor_test.cpp b/staging_vespalib/src/tests/sequencedtaskexecutor/sequencedtaskexecutor_test.cpp index 7c90f96afdd..647bcf9bcee 100644 --- a/staging_vespalib/src/tests/sequencedtaskexecutor/sequencedtaskexecutor_test.cpp +++ b/staging_vespalib/src/tests/sequencedtaskexecutor/sequencedtaskexecutor_test.cpp @@ -252,6 +252,30 @@ TEST("require that you distribute well") { EXPECT_EQUAL(97u, seq.getComponentEffectiveHashSize()); } +TEST("require that similar names get perfect distribution with 4 executors") { + auto four = SequencedTaskExecutor::create(sequenced_executor, 4); + EXPECT_EQUAL(0u, four->getExecutorIdFromName("f1").getId()); + EXPECT_EQUAL(1u, four->getExecutorIdFromName("f2").getId()); + EXPECT_EQUAL(2u, four->getExecutorIdFromName("f3").getId()); + EXPECT_EQUAL(3u, four->getExecutorIdFromName("f4").getId()); + EXPECT_EQUAL(0u, four->getExecutorIdFromName("f5").getId()); + EXPECT_EQUAL(1u, four->getExecutorIdFromName("f6").getId()); + EXPECT_EQUAL(2u, four->getExecutorIdFromName("f7").getId()); + EXPECT_EQUAL(3u, four->getExecutorIdFromName("f8").getId()); +} + +TEST("require that similar names gets 7/8 unique ids with 8 executors") { + auto four = SequencedTaskExecutor::create(sequenced_executor, 8); + EXPECT_EQUAL(0u, four->getExecutorIdFromName("f1").getId()); + EXPECT_EQUAL(1u, four->getExecutorIdFromName("f2").getId()); + EXPECT_EQUAL(2u, four->getExecutorIdFromName("f3").getId()); + EXPECT_EQUAL(3u, four->getExecutorIdFromName("f4").getId()); + EXPECT_EQUAL(4u, four->getExecutorIdFromName("f5").getId()); + EXPECT_EQUAL(5u, four->getExecutorIdFromName("f6").getId()); + EXPECT_EQUAL(2u, four->getExecutorIdFromName("f7").getId()); + EXPECT_EQUAL(6u, four->getExecutorIdFromName("f8").getId()); +} + TEST("Test creation of different types") { auto iseq = SequencedTaskExecutor::create(sequenced_executor, 1); diff --git a/staging_vespalib/src/vespa/vespalib/util/isequencedtaskexecutor.cpp b/staging_vespalib/src/vespa/vespalib/util/isequencedtaskexecutor.cpp index 41653889b8f..c54f182891c 100644 --- a/staging_vespalib/src/vespa/vespalib/util/isequencedtaskexecutor.cpp +++ b/staging_vespalib/src/vespa/vespalib/util/isequencedtaskexecutor.cpp @@ -18,4 +18,13 @@ ISequencedTaskExecutor::getExecutorIdFromName(vespalib::stringref componentId) c return getExecutorId(hashfun(componentId)); } +ISequencedTaskExecutor::ExecutorId +ISequencedTaskExecutor::get_alternate_executor_id(ExecutorId id, uint32_t bias) const +{ + if ((bias % _numExecutors) == 0) { + bias = 1; + } + return ExecutorId((id.getId() + bias) % _numExecutors); +} + } diff --git a/staging_vespalib/src/vespa/vespalib/util/isequencedtaskexecutor.h b/staging_vespalib/src/vespa/vespalib/util/isequencedtaskexecutor.h index 8268363d335..0e931838279 100644 --- a/staging_vespalib/src/vespa/vespalib/util/isequencedtaskexecutor.h +++ b/staging_vespalib/src/vespa/vespalib/util/isequencedtaskexecutor.h @@ -19,12 +19,12 @@ class ISequencedTaskExecutor public: class ExecutorId { public: - ExecutorId() : ExecutorId(0) { } - explicit ExecutorId(uint32_t id) : _id(id) { } - uint32_t getId() const { return _id; } - bool operator != (ExecutorId rhs) const { return _id != rhs._id; } - bool operator == (ExecutorId rhs) const { return _id == rhs._id; } - bool operator < (ExecutorId rhs) const { return _id < rhs._id; } + ExecutorId() noexcept : ExecutorId(0) { } + explicit ExecutorId(uint32_t id) noexcept : _id(id) { } + uint32_t getId() const noexcept { return _id; } + bool operator != (ExecutorId rhs) const noexcept { return _id != rhs._id; } + bool operator == (ExecutorId rhs) const noexcept { return _id == rhs._id; } + bool operator < (ExecutorId rhs) const noexcept { return _id < rhs._id; } private: uint32_t _id; }; @@ -43,6 +43,15 @@ public: ExecutorId getExecutorIdFromName(vespalib::stringref componentId) const; /** + * Returns an executor id that is NOT equal to the given executor id, + * using the given bias to offset the new id. + * + * This is relevant for pipelining operations on the same component, + * by doing pipeline steps in different executors. + */ + ExecutorId get_alternate_executor_id(ExecutorId id, uint32_t bias) const; + + /** * Schedule a task to run after all previously scheduled tasks with * same id. * diff --git a/staging_vespalib/src/vespa/vespalib/util/sequencedtaskexecutor.cpp b/staging_vespalib/src/vespa/vespalib/util/sequencedtaskexecutor.cpp index 038b9201724..f92e1655e7d 100644 --- a/staging_vespalib/src/vespa/vespalib/util/sequencedtaskexecutor.cpp +++ b/staging_vespalib/src/vespa/vespalib/util/sequencedtaskexecutor.cpp @@ -39,7 +39,7 @@ SequencedTaskExecutor::create(vespalib::Runnable::init_fun_t func, uint32_t thre executors->reserve(threads); for (uint32_t id = 0; id < threads; ++id) { if (optimize == OptimizeFor::THROUGHPUT) { - uint32_t watermark = kindOfWatermark == 0 ? taskLimit / 10 : kindOfWatermark; + uint32_t watermark = kindOfWatermark == 0 ? taskLimit / 2 : kindOfWatermark; executors->push_back(std::make_unique<SingleExecutor>(func, taskLimit, watermark, reactionTime)); } else { executors->push_back(std::make_unique<BlockingThreadStackExecutor>(1, stackSize, taskLimit, func)); diff --git a/statistics/src/main/java/com/yahoo/statistics/Axis.java b/statistics/src/main/java/com/yahoo/statistics/Axis.java index f0a5f017984..f0daff92d00 100644 --- a/statistics/src/main/java/com/yahoo/statistics/Axis.java +++ b/statistics/src/main/java/com/yahoo/statistics/Axis.java @@ -10,6 +10,7 @@ import java.util.Arrays; * * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> */ +@Deprecated class Axis { private final double[] limits; private final String name; diff --git a/statistics/src/main/java/com/yahoo/statistics/Bucket.java b/statistics/src/main/java/com/yahoo/statistics/Bucket.java index 6977afaac8c..fd69f202d2b 100644 --- a/statistics/src/main/java/com/yahoo/statistics/Bucket.java +++ b/statistics/src/main/java/com/yahoo/statistics/Bucket.java @@ -9,6 +9,7 @@ import java.util.List; * * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> */ +@Deprecated interface Bucket { void put(double[] value, int dim); void reset(); diff --git a/statistics/src/main/java/com/yahoo/statistics/Callback.java b/statistics/src/main/java/com/yahoo/statistics/Callback.java index 4bb332c99e4..fad86d38fd7 100644 --- a/statistics/src/main/java/com/yahoo/statistics/Callback.java +++ b/statistics/src/main/java/com/yahoo/statistics/Callback.java @@ -11,6 +11,7 @@ package com.yahoo.statistics; * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> * @since 5.1.4 */ +@Deprecated public interface Callback { /** * Invoked each logging cycle right before the events for a Handle are diff --git a/statistics/src/main/java/com/yahoo/statistics/Counter.java b/statistics/src/main/java/com/yahoo/statistics/Counter.java index 2e39cfa6e70..1425c2a4375 100644 --- a/statistics/src/main/java/com/yahoo/statistics/Counter.java +++ b/statistics/src/main/java/com/yahoo/statistics/Counter.java @@ -11,6 +11,7 @@ import com.yahoo.container.StatisticsConfig; * * @author Steinar Knutsen */ +@Deprecated public class Counter extends Handle { // The current value of this counter private AtomicLong current = new AtomicLong(0L); diff --git a/statistics/src/main/java/com/yahoo/statistics/CounterGroup.java b/statistics/src/main/java/com/yahoo/statistics/CounterGroup.java index 7e0258b3896..39d8ad19f29 100644 --- a/statistics/src/main/java/com/yahoo/statistics/CounterGroup.java +++ b/statistics/src/main/java/com/yahoo/statistics/CounterGroup.java @@ -15,6 +15,7 @@ import java.util.Iterator; * * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> */ +@Deprecated public class CounterGroup extends Group { private final boolean resetCounter; diff --git a/statistics/src/main/java/com/yahoo/statistics/CounterProxy.java b/statistics/src/main/java/com/yahoo/statistics/CounterProxy.java index 11db94673ea..da8a029982a 100644 --- a/statistics/src/main/java/com/yahoo/statistics/CounterProxy.java +++ b/statistics/src/main/java/com/yahoo/statistics/CounterProxy.java @@ -8,6 +8,7 @@ package com.yahoo.statistics; * * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> */ +@Deprecated class CounterProxy extends Proxy { private long raw; private boolean hasRaw = false; diff --git a/statistics/src/main/java/com/yahoo/statistics/Group.java b/statistics/src/main/java/com/yahoo/statistics/Group.java index 7b0c1823dbe..2935521b1e0 100644 --- a/statistics/src/main/java/com/yahoo/statistics/Group.java +++ b/statistics/src/main/java/com/yahoo/statistics/Group.java @@ -7,6 +7,7 @@ package com.yahoo.statistics; * * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> */ +@Deprecated abstract class Group extends Handle { Group(String name, Statistics manager, Callback parametrizedCallback) { super(name, manager, parametrizedCallback); diff --git a/statistics/src/main/java/com/yahoo/statistics/Handle.java b/statistics/src/main/java/com/yahoo/statistics/Handle.java index f67dafa9309..d1ad0e26f40 100644 --- a/statistics/src/main/java/com/yahoo/statistics/Handle.java +++ b/statistics/src/main/java/com/yahoo/statistics/Handle.java @@ -8,6 +8,7 @@ import java.util.TimerTask; * * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> */ +@Deprecated public abstract class Handle { private TimerTask task; diff --git a/statistics/src/main/java/com/yahoo/statistics/Histogram.java b/statistics/src/main/java/com/yahoo/statistics/Histogram.java index df529015d88..98330e80efc 100644 --- a/statistics/src/main/java/com/yahoo/statistics/Histogram.java +++ b/statistics/src/main/java/com/yahoo/statistics/Histogram.java @@ -8,12 +8,12 @@ import java.util.Iterator; import java.util.List; import java.util.ArrayList; - /** * A set of sums or other histograms. * * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> */ +@Deprecated public class Histogram implements Bucket { // The upper and lower limit for the bucket in another histogram // this histogram represents. The "outermost" histogram in a diff --git a/statistics/src/main/java/com/yahoo/statistics/HistogramType.java b/statistics/src/main/java/com/yahoo/statistics/HistogramType.java index fe6b9a3341b..cc7ca04dbbf 100644 --- a/statistics/src/main/java/com/yahoo/statistics/HistogramType.java +++ b/statistics/src/main/java/com/yahoo/statistics/HistogramType.java @@ -8,6 +8,7 @@ package com.yahoo.statistics; * * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> */ +@Deprecated public final class HistogramType { private final String representation; diff --git a/statistics/src/main/java/com/yahoo/statistics/Limits.java b/statistics/src/main/java/com/yahoo/statistics/Limits.java index 2c6d1cb3119..fc79fd00d19 100644 --- a/statistics/src/main/java/com/yahoo/statistics/Limits.java +++ b/statistics/src/main/java/com/yahoo/statistics/Limits.java @@ -12,6 +12,7 @@ import java.util.ArrayList; * * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> */ +@Deprecated public class Limits { private final List<Axis> axes = new ArrayList<>(1); private boolean frozen = false; diff --git a/statistics/src/main/java/com/yahoo/statistics/Proxy.java b/statistics/src/main/java/com/yahoo/statistics/Proxy.java index 5d656897f7b..c4146f4dbf8 100644 --- a/statistics/src/main/java/com/yahoo/statistics/Proxy.java +++ b/statistics/src/main/java/com/yahoo/statistics/Proxy.java @@ -8,6 +8,7 @@ package com.yahoo.statistics; * * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> */ +@Deprecated abstract class Proxy { private long timestamp; private String name; diff --git a/statistics/src/main/java/com/yahoo/statistics/SampleDirectory.java b/statistics/src/main/java/com/yahoo/statistics/SampleDirectory.java index 520fd88a042..bac08fe17b6 100644 --- a/statistics/src/main/java/com/yahoo/statistics/SampleDirectory.java +++ b/statistics/src/main/java/com/yahoo/statistics/SampleDirectory.java @@ -4,13 +4,12 @@ package com.yahoo.statistics; import java.util.ArrayList; import java.util.List; -import com.yahoo.statistics.SampleSet.Sampling; - /** * Book-keeping class to know which SampleSet instances have unlogged data. * * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> */ +@Deprecated final class SampleDirectory { private final Object directoryLock = new Object(); private List<SampleSet> directory = new ArrayList<>(200); @@ -27,11 +26,11 @@ final class SampleDirectory { * generation. This does the memory barrier two-step for the * client. */ - Sampling[] fetchValues() { - Sampling[] copyToReturn; + SampleSet.Sampling[] fetchValues() { + SampleSet.Sampling[] copyToReturn; synchronized (directoryLock) { List<SampleSet> tmpDir = directory; - copyToReturn = new Sampling[tmpDir.size()]; + copyToReturn = new SampleSet.Sampling[tmpDir.size()]; List<SampleSet> newDir = new ArrayList<>(200); for (int i = 0; i < copyToReturn.length; ++i) { copyToReturn[i] = tmpDir.get(i).getAndReset(); @@ -45,10 +44,10 @@ final class SampleDirectory { * Return a view of the current generation of data. This does the memory * barrier two-step for the client. */ - Sampling[] viewValues() { - Sampling[] copy; + SampleSet.Sampling[] viewValues() { + SampleSet.Sampling[] copy; synchronized (directoryLock) { - copy = new Sampling[directory.size()]; + copy = new SampleSet.Sampling[directory.size()]; for (int i = 0; i < copy.length; ++i) { copy[i] = directory.get(i).values; } diff --git a/statistics/src/main/java/com/yahoo/statistics/SampleSet.java b/statistics/src/main/java/com/yahoo/statistics/SampleSet.java index fa71c8842ed..768ae1433ea 100644 --- a/statistics/src/main/java/com/yahoo/statistics/SampleSet.java +++ b/statistics/src/main/java/com/yahoo/statistics/SampleSet.java @@ -8,6 +8,7 @@ package com.yahoo.statistics; * * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> */ +@Deprecated final class SampleSet { Sampling values; final Limits histogramLimits; diff --git a/statistics/src/main/java/com/yahoo/statistics/Statistics.java b/statistics/src/main/java/com/yahoo/statistics/Statistics.java index d261e2d9af3..92d320e0647 100644 --- a/statistics/src/main/java/com/yahoo/statistics/Statistics.java +++ b/statistics/src/main/java/com/yahoo/statistics/Statistics.java @@ -8,7 +8,9 @@ import com.yahoo.container.StatisticsConfig; * * @author steinar * @author Tony Vaagenes + * @deprecated Will be removed on Vespa 8. If required by a method, there exists an alternative to be used instead. */ +@Deprecated public interface Statistics { /** * Add a new handle to be scheduled for periodic logging. If a handle diff --git a/statistics/src/main/java/com/yahoo/statistics/StatisticsImpl.java b/statistics/src/main/java/com/yahoo/statistics/StatisticsImpl.java index 3fb0693648f..f3a0b23a551 100644 --- a/statistics/src/main/java/com/yahoo/statistics/StatisticsImpl.java +++ b/statistics/src/main/java/com/yahoo/statistics/StatisticsImpl.java @@ -21,6 +21,7 @@ import com.yahoo.container.StatisticsConfig; * * @author Steinar Knutsen */ +@SuppressWarnings("deprecation") public final class StatisticsImpl extends AbstractComponent implements Statistics { private final Timer worker; diff --git a/statistics/src/main/java/com/yahoo/statistics/Sum.java b/statistics/src/main/java/com/yahoo/statistics/Sum.java index 534f12f9b5c..5661e82cf1b 100644 --- a/statistics/src/main/java/com/yahoo/statistics/Sum.java +++ b/statistics/src/main/java/com/yahoo/statistics/Sum.java @@ -10,6 +10,7 @@ import java.util.List; * * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> */ +@Deprecated class Sum implements Bucket { private long sum = 0L; private double lower; diff --git a/statistics/src/main/java/com/yahoo/statistics/Value.java b/statistics/src/main/java/com/yahoo/statistics/Value.java index 98446a10135..9b41f526f13 100644 --- a/statistics/src/main/java/com/yahoo/statistics/Value.java +++ b/statistics/src/main/java/com/yahoo/statistics/Value.java @@ -8,7 +8,6 @@ import java.util.logging.Logger; import com.yahoo.container.StatisticsConfig; import com.yahoo.container.StatisticsConfig.Values.Operations; import java.util.logging.Level; -import com.yahoo.statistics.SampleSet.Sampling; /** * A statistical variable, typically representing a sampling of an @@ -16,6 +15,7 @@ import com.yahoo.statistics.SampleSet.Sampling; * * @author Steinar Knutsen */ +@Deprecated public class Value extends Handle { // For accumulated values, SampleSet instances are mem barriers between {n @@ -530,10 +530,10 @@ public class Value extends Handle { * Get mean value since last reset. */ public double getMean() { - Sampling[] values = directory.viewValues(); + SampleSet.Sampling[] values = directory.viewValues(); long insertions = 0L; double sum = 0.0d; - for (Sampling x : values) { + for (var x : values) { insertions += x.insertions; sum += x.sum; } @@ -547,10 +547,10 @@ public class Value extends Handle { * Get minimal value logged since last reset. */ public double getMin() { - Sampling[] values = directory.viewValues(); + SampleSet.Sampling[] values = directory.viewValues(); long insertions = 0L; double min = 0.0d; - for (Sampling x : values) { + for (var x : values) { if (x.insertions == 0) { continue; } @@ -568,10 +568,10 @@ public class Value extends Handle { * Get maximum value logged since last reset. */ public double getMax() { - Sampling[] values = directory.viewValues(); + SampleSet.Sampling[] values = directory.viewValues(); long insertions = 0L; double max = 0.0d; - for (Sampling x : values) { + for (var x : values) { if (x.insertions == 0) { continue; } @@ -589,9 +589,9 @@ public class Value extends Handle { if (histogram == null) { return null; } else { - Sampling[] values = directory.viewValues(); + SampleSet.Sampling[] values = directory.viewValues(); Histogram merged = new Histogram(histogram); - for (Sampling s : values) { + for (var s : values) { merged.merge(s.histogram); } return merged; @@ -706,11 +706,11 @@ public class Value extends Handle { lastRaw = lastValue; } if (logComposite()) { - Sampling[] lastInterval = directory.fetchValues(); + SampleSet.Sampling[] lastInterval = directory.fetchValues(); if (histogram != null) { mergedHistogram = new Histogram(histogram); } - for (Sampling threadData : lastInterval) { + for (var threadData : lastInterval) { if (threadData.insertions == 0) { continue; } diff --git a/statistics/src/main/java/com/yahoo/statistics/ValueGroup.java b/statistics/src/main/java/com/yahoo/statistics/ValueGroup.java index 8981a75b578..e8f26e625d4 100644 --- a/statistics/src/main/java/com/yahoo/statistics/ValueGroup.java +++ b/statistics/src/main/java/com/yahoo/statistics/ValueGroup.java @@ -3,7 +3,6 @@ package com.yahoo.statistics; import com.yahoo.log.event.Event; -import com.yahoo.statistics.Value.Parameters; import java.util.Iterator; import java.util.Map; @@ -14,6 +13,7 @@ import java.util.HashMap; * * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> */ +@Deprecated public class ValueGroup extends Group { // A map for names of subevents and Value instances private Map<String, Value> subEvents = new HashMap<>(); @@ -67,7 +67,7 @@ public class ValueGroup extends Group { } private Value getNewValue(String subName) { - Value v = Value.initializeUnregisteredValue(subName, new Parameters().setLogRaw(true)); + Value v = Value.initializeUnregisteredValue(subName, new Value.Parameters().setLogRaw(true)); subEvents.put(subName, v); return v; } diff --git a/statistics/src/main/java/com/yahoo/statistics/ValueProxy.java b/statistics/src/main/java/com/yahoo/statistics/ValueProxy.java index 77d2f45fcd6..3146dc7d2d7 100644 --- a/statistics/src/main/java/com/yahoo/statistics/ValueProxy.java +++ b/statistics/src/main/java/com/yahoo/statistics/ValueProxy.java @@ -8,6 +8,7 @@ package com.yahoo.statistics; * * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> */ +@Deprecated class ValueProxy extends Proxy { private double raw; private boolean hasRaw = false; diff --git a/statistics/src/test/java/com/yahoo/statistics/CounterGroupTestCase.java b/statistics/src/test/java/com/yahoo/statistics/CounterGroupTestCase.java index 57a321e126f..323a4fd23c6 100644 --- a/statistics/src/test/java/com/yahoo/statistics/CounterGroupTestCase.java +++ b/statistics/src/test/java/com/yahoo/statistics/CounterGroupTestCase.java @@ -18,6 +18,7 @@ import static org.junit.Assert.assertFalse; * * @author Steinar Knutsen */ +@SuppressWarnings("deprecation") public class CounterGroupTestCase { private volatile boolean gotRecord = false; diff --git a/statistics/src/test/java/com/yahoo/statistics/CounterTestCase.java b/statistics/src/test/java/com/yahoo/statistics/CounterTestCase.java index c4dd5dd9c67..8c17fd936a0 100644 --- a/statistics/src/test/java/com/yahoo/statistics/CounterTestCase.java +++ b/statistics/src/test/java/com/yahoo/statistics/CounterTestCase.java @@ -15,7 +15,7 @@ import static org.junit.Assert.assertFalse; * * @author Steinar Knutsen */ - +@SuppressWarnings("deprecation") public class CounterTestCase { @Test diff --git a/statistics/src/test/java/com/yahoo/statistics/HistogramTestCase.java b/statistics/src/test/java/com/yahoo/statistics/HistogramTestCase.java index a15d679c77e..517de9b4ef1 100644 --- a/statistics/src/test/java/com/yahoo/statistics/HistogramTestCase.java +++ b/statistics/src/test/java/com/yahoo/statistics/HistogramTestCase.java @@ -12,7 +12,7 @@ import static org.junit.Assert.fail; * * @author Steinar Knutsen */ - +@SuppressWarnings("deprecation") public class HistogramTestCase { @Test diff --git a/statistics/src/test/java/com/yahoo/statistics/ProxyTestCase.java b/statistics/src/test/java/com/yahoo/statistics/ProxyTestCase.java index dcfdac746ad..8eaee5d8ce8 100644 --- a/statistics/src/test/java/com/yahoo/statistics/ProxyTestCase.java +++ b/statistics/src/test/java/com/yahoo/statistics/ProxyTestCase.java @@ -12,6 +12,7 @@ import org.junit.Test; * * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> */ +@SuppressWarnings("deprecation") public class ProxyTestCase { private static final double MAX = 2.0d; private static final double MEAN = 1.0d; diff --git a/statistics/src/test/java/com/yahoo/statistics/StatisticsImplTestCase.java b/statistics/src/test/java/com/yahoo/statistics/StatisticsImplTestCase.java index f0f951610f5..1ae8725ffe2 100644 --- a/statistics/src/test/java/com/yahoo/statistics/StatisticsImplTestCase.java +++ b/statistics/src/test/java/com/yahoo/statistics/StatisticsImplTestCase.java @@ -16,6 +16,7 @@ import com.yahoo.container.StatisticsConfig; * * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> */ +@SuppressWarnings("deprecation") public class StatisticsImplTestCase { private static class TestHandle extends Handle { diff --git a/statistics/src/test/java/com/yahoo/statistics/ValueGroupTestCase.java b/statistics/src/test/java/com/yahoo/statistics/ValueGroupTestCase.java index 35db53b3316..f9c3ee79ad7 100644 --- a/statistics/src/test/java/com/yahoo/statistics/ValueGroupTestCase.java +++ b/statistics/src/test/java/com/yahoo/statistics/ValueGroupTestCase.java @@ -18,6 +18,7 @@ import static org.junit.Assert.assertTrue; * * @author Steinar Knutsen */ +@SuppressWarnings("deprecation") public class ValueGroupTestCase { private volatile boolean gotRecord = false; diff --git a/statistics/src/test/java/com/yahoo/statistics/ValueTestCase.java b/statistics/src/test/java/com/yahoo/statistics/ValueTestCase.java index 72825ffd6ad..91a02d90a6e 100644 --- a/statistics/src/test/java/com/yahoo/statistics/ValueTestCase.java +++ b/statistics/src/test/java/com/yahoo/statistics/ValueTestCase.java @@ -14,7 +14,6 @@ import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; -import com.yahoo.statistics.Value.Parameters; import org.junit.Test; /** @@ -22,6 +21,7 @@ import org.junit.Test; * * @author Steinar Knutsen */ +@SuppressWarnings("deprecation") public class ValueTestCase { private static final double delta = 0.0000000001; @@ -46,7 +46,7 @@ public class ValueTestCase { @Test public void testMean() { - Value v = new Value("thingie", Statistics.nullImplementation, new Parameters().setLogMean(true)); + Value v = new Value("thingie", Statistics.nullImplementation, new Value.Parameters().setLogMean(true)); v.put(1.0); v.put(2.0); v.put(4.0); @@ -59,7 +59,7 @@ public class ValueTestCase { @Test public void testMin() { - Value v = new Value("thingie", Statistics.nullImplementation, new Parameters().setLogMin(true)); + Value v = new Value("thingie", Statistics.nullImplementation, new Value.Parameters().setLogMin(true)); v.put(2.0); assertTrue("Min should be 2.0", 2.0 == v.getMin()); v.put(1.0); @@ -71,7 +71,7 @@ public class ValueTestCase { @Test public void testMax() { - Value v = new Value("thingie", Statistics.nullImplementation, new Parameters().setLogMax(true)); + Value v = new Value("thingie", Statistics.nullImplementation, new Value.Parameters().setLogMax(true)); v.put(-1.0); assertTrue("Max should be -1.0", -1.0 == v.getMax()); v.put(1.0); @@ -84,7 +84,7 @@ public class ValueTestCase { @Test public void testHistogram() { - Value v = new Value("thingie", Statistics.nullImplementation, new Parameters() + Value v = new Value("thingie", Statistics.nullImplementation, new Value.Parameters() .setLogHistogram(true).setHistogramId(HistogramType.REGULAR) .setLimits(new Limits(new double[] { 0.0, 1.0, 2.0 }))); v.put(-1.0); @@ -101,7 +101,7 @@ public class ValueTestCase { Logger logger = Logger.getLogger(Value.class.getName()); boolean initUseParentHandlers = logger.getUseParentHandlers(); logger.setUseParentHandlers(false); - Value v = new Value("thingie", Statistics.nullImplementation, new Parameters() + Value v = new Value("thingie", Statistics.nullImplementation, new Value.Parameters() .setLogRaw(true).setCallback(new TrivialCallback())); v.run(); assertEquals(FIRST, v.get(), delta); diff --git a/storage/src/vespa/storage/storageserver/statemanager.cpp b/storage/src/vespa/storage/storageserver/statemanager.cpp index 7596bdf1b84..7d08f738abe 100644 --- a/storage/src/vespa/storage/storageserver/statemanager.cpp +++ b/storage/src/vespa/storage/storageserver/statemanager.cpp @@ -577,21 +577,6 @@ StateManager::sendGetNodeStateReplies(framework::MilliSecTime olderThanTime, uin return true; } -namespace { - std::string getHostInfoFilename(bool advanceCount) { - static uint32_t fileCounter = 0; - static pid_t pid = getpid(); - if (advanceCount) { - ++fileCounter; - } - uint32_t fileIndex = fileCounter % 8; - std::ostringstream fileName; - fileName << vespa::Defaults::underVespaHome("tmp/hostinfo") - << "." << pid << "." << fileIndex << ".report"; - return fileName.str(); - } -} - std::string StateManager::getNodeInfo() const { @@ -631,15 +616,6 @@ StateManager::getNodeInfo() const stream << End(); stream.finalize(); - // Dump report to new report file. - std::string oldFile(getHostInfoFilename(false)); - std::string newFile(getHostInfoFilename(true)); - std::ofstream of(newFile.c_str()); - of << json.str(); - of.close(); - // If dumping went ok, delete old report file - vespalib::unlink(oldFile); - // Return report return json.str(); } diff --git a/testutil/src/main/java/com/yahoo/test/ManualClock.java b/testutil/src/main/java/com/yahoo/test/ManualClock.java index ce3ae6d77df..f5fe1ea466a 100644 --- a/testutil/src/main/java/com/yahoo/test/ManualClock.java +++ b/testutil/src/main/java/com/yahoo/test/ManualClock.java @@ -60,4 +60,10 @@ public class ManualClock extends Clock { return LocalDateTime.parse(utcIsoTime, DateTimeFormatter.ISO_DATE_TIME).atZone(ZoneOffset.UTC).toInstant(); } + @Override + public String toString() { + return "ManualClock{" + + "currentTime=" + currentTime + + '}'; + } } diff --git a/vespa-hadoop/pom.xml b/vespa-hadoop/pom.xml index 8d56aad1f1e..b93e1d8dace 100644 --- a/vespa-hadoop/pom.xml +++ b/vespa-hadoop/pom.xml @@ -207,7 +207,6 @@ <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-compiler-plugin</artifactId> - <version>3.8.1</version> <configuration> <jdkToolchain> <version>${java.version}</version> diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/operationProcessor/OperationProcessor.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/operationProcessor/OperationProcessor.java index 6633fca6ca0..5d0e063efdf 100644 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/operationProcessor/OperationProcessor.java +++ b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/operationProcessor/OperationProcessor.java @@ -145,7 +145,7 @@ public class OperationProcessor { if (retryThisOperation) { int waitTime = (int) (minTimeBetweenRetriesMs * (1 + random.nextDouble() / 3)); - log.finest("Retrying due to " + detail.toString() + " attempt " + retries + " in " + waitTime + " ms."); + log.finest("Retrying due to " + detail + " attempt " + retries + " in " + waitTime + " ms."); timeoutExecutor.schedule(() -> postToCluster(clusters.get(clusterId), documentSendInfo.getDocument()), waitTime, TimeUnit.MILLISECONDS); diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/DocumentV1ApiHandler.java b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/DocumentV1ApiHandler.java index 29fd419daa2..a102e8fffd4 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/DocumentV1ApiHandler.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/DocumentV1ApiHandler.java @@ -92,7 +92,9 @@ import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.Executors; import java.util.concurrent.Phaser; import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; import java.util.concurrent.locks.Lock; @@ -112,6 +114,7 @@ import static com.yahoo.jdisc.http.HttpRequest.Method.OPTIONS; import static com.yahoo.jdisc.http.HttpRequest.Method.POST; import static com.yahoo.jdisc.http.HttpRequest.Method.PUT; import static java.util.Objects.requireNonNull; +import static java.util.concurrent.TimeUnit.MILLISECONDS; import static java.util.logging.Level.FINE; import static java.util.logging.Level.WARNING; import static java.util.stream.Collectors.joining; @@ -211,11 +214,11 @@ public class DocumentV1ApiHandler extends AbstractRequestHandler { this.dispatcher.scheduleWithFixedDelay(this::dispatchEnqueued, executorConfig.resendDelayMillis(), executorConfig.resendDelayMillis(), - TimeUnit.MILLISECONDS); + MILLISECONDS); this.visitDispatcher.scheduleWithFixedDelay(this::dispatchVisitEnqueued, executorConfig.resendDelayMillis(), executorConfig.resendDelayMillis(), - TimeUnit.MILLISECONDS); + MILLISECONDS); } // ------------------------------------------------ Requests ------------------------------------------------- @@ -233,7 +236,7 @@ public class DocumentV1ApiHandler extends AbstractRequestHandler { // Set a higher HTTP layer timeout than the document API timeout, to prefer triggering the latter. request.setTimeout( getProperty(request, TIMEOUT, timeoutMillisParser).orElse(defaultTimeout.toMillis()) + handlerTimeout.toMillis(), - TimeUnit.MILLISECONDS); + MILLISECONDS); Path requestPath = new Path(request.getUri()); for (String path : handlers.keySet()) @@ -260,7 +263,7 @@ public class DocumentV1ApiHandler extends AbstractRequestHandler { @Override public void handleTimeout(Request request, ResponseHandler responseHandler) { - timeout((HttpRequest) request, "Timeout after " + (request.getTimeout(TimeUnit.MILLISECONDS) - handlerTimeout.toMillis()) + "ms", responseHandler); + timeout((HttpRequest) request, "Timeout after " + (request.getTimeout(MILLISECONDS) - handlerTimeout.toMillis()) + "ms", responseHandler); } @Override @@ -290,10 +293,10 @@ public class DocumentV1ApiHandler extends AbstractRequestHandler { while (outstanding.get() > 0 && clock.instant().isBefore(doom)) Thread.sleep(Math.max(1, Duration.between(clock.instant(), doom).toMillis())); - if ( ! dispatcher.awaitTermination(Duration.between(clock.instant(), doom).toMillis(), TimeUnit.MILLISECONDS)) + if ( ! dispatcher.awaitTermination(Duration.between(clock.instant(), doom).toMillis(), MILLISECONDS)) dispatcher.shutdownNow(); - if ( ! visitDispatcher.awaitTermination(Duration.between(clock.instant(), doom).toMillis(), TimeUnit.MILLISECONDS)) + if ( ! visitDispatcher.awaitTermination(Duration.between(clock.instant(), doom).toMillis(), MILLISECONDS)) visitDispatcher.shutdownNow(); } catch (InterruptedException e) { @@ -1055,18 +1058,18 @@ public class DocumentV1ApiHandler extends AbstractRequestHandler { VisitorParameters parameters = parseCommonParameters(request, path, cluster); parameters.setFieldSet(getProperty(request, FIELD_SET).orElse(path.documentType().map(type -> type + ":[document]").orElse(AllFields.NAME))); parameters.setMaxTotalHits(wantedDocumentCount); - StaticThrottlePolicy throttlePolicy; + parameters.visitInconsistentBuckets(true); + long timeoutMs = Math.max(1, request.getTimeout(MILLISECONDS) - handlerTimeout.toMillis()); if (streamed) { - throttlePolicy = new DynamicThrottlePolicy().setMinWindowSize(1).setWindowSizeIncrement(1); + StaticThrottlePolicy throttlePolicy = new DynamicThrottlePolicy().setMinWindowSize(1).setWindowSizeIncrement(1); concurrency.ifPresent(throttlePolicy::setMaxPendingCount); + parameters.setThrottlePolicy(throttlePolicy); + parameters.setTimeoutMs(timeoutMs); // Ensure visitor eventually completes. } else { - throttlePolicy = new StaticThrottlePolicy().setMaxPendingCount(Math.min(100, concurrency.orElse(1))); + parameters.setThrottlePolicy(new StaticThrottlePolicy().setMaxPendingCount(Math.min(100, concurrency.orElse(1)))); + parameters.setSessionTimeoutMs(timeoutMs); } - parameters.setThrottlePolicy(throttlePolicy); - parameters.visitInconsistentBuckets(true); - parameters.setSessionTimeoutMs(Math.max(1, request.getTimeout(TimeUnit.MILLISECONDS) - handlerTimeout.toMillis())); - return parameters; } @@ -1076,7 +1079,7 @@ public class DocumentV1ApiHandler extends AbstractRequestHandler { VisitorParameters parameters = parseCommonParameters(request, path, Optional.of(requireProperty(request, CLUSTER))); parameters.setThrottlePolicy(new DynamicThrottlePolicy().setMinWindowSize(1).setWindowSizeIncrement(1)); long timeChunk = getProperty(request, TIME_CHUNK, timeoutMillisParser).orElse(60_000L); - parameters.setSessionTimeoutMs(Math.max(1, Math.min(timeChunk, request.getTimeout(TimeUnit.MILLISECONDS) - handlerTimeout.toMillis()))); + parameters.setSessionTimeoutMs(Math.max(1, Math.min(timeChunk, request.getTimeout(MILLISECONDS) - handlerTimeout.toMillis()))); return parameters; } @@ -1223,6 +1226,7 @@ public class DocumentV1ApiHandler extends AbstractRequestHandler { AtomicReference<String> error = new AtomicReference<>(); // Set if error occurs during processing of visited documents. callback.onStart(response); VisitorControlHandler controller = new VisitorControlHandler() { + final ScheduledFuture<?> abort = streaming ? visitDispatcher.schedule(this::abort, request.getTimeout(MILLISECONDS), MILLISECONDS) : null; @Override public void onDone(CompletionCode code, String message) { super.onDone(code, message); loggingException(() -> { @@ -1258,10 +1262,12 @@ public class DocumentV1ApiHandler extends AbstractRequestHandler { response.commit(status); } }); + if (abort != null) abort.cancel(false); // Avoid keeping scheduled future alive if this completes in any other fashion. visitDispatcher.execute(() -> { phaser.arriveAndAwaitAdvance(); // We may get here while dispatching thread is still putting us in the map. visits.remove(this).destroy(); }); + } }; if (parameters.getRemoteDataHandler() == null) { diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/resource/DocumentV1ApiTest.java b/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/resource/DocumentV1ApiTest.java index 2452e19bfff..de907f70c19 100644 --- a/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/resource/DocumentV1ApiTest.java +++ b/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/resource/DocumentV1ApiTest.java @@ -257,7 +257,7 @@ public class DocumentV1ApiTest { assertEquals(1, ((StaticThrottlePolicy) parameters.getThrottlePolicy()).getMaxPendingCount()); assertEquals("[id]", parameters.getFieldSet()); assertEquals("(all the things)", parameters.getDocumentSelection()); - assertEquals(6000, parameters.getSessionTimeoutMs()); + assertEquals(6000, parameters.getTimeoutMs()); assertEquals(4, parameters.getSlices()); assertEquals(1, parameters.getSliceId()); // Put some documents in the response diff --git a/vespamalloc/src/vespamalloc/malloc/overload.h b/vespamalloc/src/vespamalloc/malloc/overload.h index 09d36a477e7..69d95ef5cdc 100644 --- a/vespamalloc/src/vespamalloc/malloc/overload.h +++ b/vespamalloc/src/vespamalloc/malloc/overload.h @@ -237,7 +237,11 @@ void __libc_free(void* ptr) __THROW __attribute__((leaf void __libc_cfree(void* ptr) __THROW __attribute__((leaf)) ALIAS("cfree"); #endif size_t __libc_malloc_usable_size(void *ptr) __THROW ALIAS("malloc_usable_size"); +#if __GLIBC_PREREQ(2, 34) +void* __libc_memalign(size_t align, size_t s) __THROW __attribute__((leaf, malloc, alloc_align(1), alloc_size(2))) ALIAS("memalign"); +#else void* __libc_memalign(size_t align, size_t s) __THROW __attribute__((leaf, malloc, alloc_size(2))) ALIAS("memalign"); +#endif int __posix_memalign(void** r, size_t a, size_t s) __THROW __nonnull((1)) ALIAS("posix_memalign"); #if __GLIBC_PREREQ(2, 33) struct mallinfo2 __libc_mallinfo2() __THROW ALIAS("mallinfo2"); |