diff options
69 files changed, 528 insertions, 299 deletions
diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java index e3446002888..91514b89b3e 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java @@ -24,6 +24,8 @@ import java.util.Optional; import java.util.Set; import java.util.concurrent.ExecutorService; +import static com.yahoo.config.provision.NodeResources.Architecture; + /** * Model context containing state provided to model factories. * @@ -118,7 +120,8 @@ public interface ModelContext { @ModelFeatureFlag(owners = {"arnej"}) default boolean avoidRenamingSummaryFeatures() { return false; } @ModelFeatureFlag(owners = {"bjorncs", "baldersheim"}) default boolean mergeGroupingResultInSearchInvoker() { return false; } @ModelFeatureFlag(owners = {"arnej"}) default boolean experimentalSdParsing() { return false; } - @ModelFeatureFlag(owners = {"hmusum"}) default String adminClusterNodeArchitecture() { return "x86_64"; } // TODO: Cluster controllers only for now + @ModelFeatureFlag(owners = {"hmusum"}) default String adminClusterNodeArchitecture() { return adminClusterArchitecture().name(); } // TODO: Remove when 7.564 is oldest version in use + @ModelFeatureFlag(owners = {"hmusum"}) default Architecture adminClusterArchitecture() { return Architecture.getDefault(); } } /** Warning: As elsewhere in this package, do not make backwards incompatible changes that will break old config models! */ diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java index 575080d3025..3e251a621ab 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java @@ -20,6 +20,8 @@ import java.util.List; import java.util.Optional; import java.util.Set; +import static com.yahoo.config.provision.NodeResources.Architecture; + /** * A test-only Properties class * @@ -81,7 +83,7 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea private List<String> environmentVariables = List.of(); private boolean avoidRenamingSummaryFeatures = false; private boolean experimentalSdParsing = false; - private String adminClusterNodeResourcesArchitecture = "x86_64"; + private Architecture adminClusterNodeResourcesArchitecture = Architecture.getDefault(); @Override public ModelContext.FeatureFlags featureFlags() { return this; } @Override public boolean multitenant() { return multitenant; } @@ -143,7 +145,7 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea @Override public List<String> environmentVariables() { return environmentVariables; } @Override public boolean avoidRenamingSummaryFeatures() { return this.avoidRenamingSummaryFeatures; } @Override public boolean experimentalSdParsing() { return this.experimentalSdParsing; } - @Override public String adminClusterNodeArchitecture() { return adminClusterNodeResourcesArchitecture; } + @Override public Architecture adminClusterArchitecture() { return adminClusterNodeResourcesArchitecture; } public TestProperties maxUnCommittedMemory(int maxUnCommittedMemory) { this.maxUnCommittedMemory = maxUnCommittedMemory; @@ -391,8 +393,8 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea return this; } - public TestProperties setAdminClusterNodeResourcesArchitecture(String architectureFunction) { - this.adminClusterNodeResourcesArchitecture = architectureFunction; + public TestProperties setAdminClusterNodeResourcesArchitecture(Architecture architecture) { + this.adminClusterNodeResourcesArchitecture = architecture; return this; } diff --git a/config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java b/config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java index 8bf6dc7f1d9..a6bcf6b0fd2 100644 --- a/config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java +++ b/config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java @@ -36,7 +36,7 @@ import java.util.stream.IntStream; */ public class InMemoryProvisioner implements HostProvisioner { - public static final NodeResources defaultResources = new NodeResources(1, 3, 10, 1); + public static final NodeResources defaultResources = new NodeResources(1, 3, 50, 1); /** * If this is true an exception is thrown when all nodes are used. @@ -153,7 +153,7 @@ public class InMemoryProvisioner implements HostProvisioner { if (alwaysReturnOneNode) nodes = 1; - int groups = requested.groups() > nodes ? nodes : requested.groups(); + int groups = Math.min(requested.groups(), nodes); List<HostSpec> allocation = new ArrayList<>(); if (groups == 1) { diff --git a/config-model/src/main/java/com/yahoo/documentmodel/NewDocumentReferenceDataType.java b/config-model/src/main/java/com/yahoo/documentmodel/NewDocumentReferenceDataType.java index 65c282e01e2..702ab835dd4 100644 --- a/config-model/src/main/java/com/yahoo/documentmodel/NewDocumentReferenceDataType.java +++ b/config-model/src/main/java/com/yahoo/documentmodel/NewDocumentReferenceDataType.java @@ -5,7 +5,6 @@ import com.yahoo.document.DataType; import com.yahoo.document.DocumentType; import com.yahoo.document.ReferenceDataType; import com.yahoo.document.StructuredDataType; -import com.yahoo.document.TemporaryStructuredDataType; import com.yahoo.document.datatypes.FieldValue; import com.yahoo.document.datatypes.ReferenceFieldValue; diff --git a/config-model/src/main/java/com/yahoo/documentmodel/NewDocumentType.java b/config-model/src/main/java/com/yahoo/documentmodel/NewDocumentType.java index dd8edbdde6c..ff5930f8f6a 100644 --- a/config-model/src/main/java/com/yahoo/documentmodel/NewDocumentType.java +++ b/config-model/src/main/java/com/yahoo/documentmodel/NewDocumentType.java @@ -7,7 +7,6 @@ import com.yahoo.document.Field; import com.yahoo.documentmodel.NewDocumentReferenceDataType; import com.yahoo.document.StructDataType; import com.yahoo.document.StructuredDataType; -import com.yahoo.document.TemporaryStructuredDataType; import com.yahoo.document.annotation.AnnotationType; import com.yahoo.document.annotation.AnnotationTypeRegistry; import com.yahoo.document.datatypes.FieldValue; diff --git a/config-model/src/main/java/com/yahoo/documentmodel/OwnedStructDataType.java b/config-model/src/main/java/com/yahoo/documentmodel/OwnedStructDataType.java new file mode 100644 index 00000000000..761a1f0963c --- /dev/null +++ b/config-model/src/main/java/com/yahoo/documentmodel/OwnedStructDataType.java @@ -0,0 +1,56 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.documentmodel; + +import com.yahoo.document.DocumentType; +import com.yahoo.document.StructDataType; + +/** + * Model for StructDataType declared in a specific document + * + * @author arnej + **/ +public final class OwnedStructDataType extends StructDataType implements OwnedType { + + private final String ownerName; + private final String uniqueName; + private boolean overrideId = false; + + public OwnedStructDataType(String name, DocumentType document) { + this(name, document.getName()); + } + + public OwnedStructDataType(String name, String owner) { + super(name); + this.ownerName = owner; + this.uniqueName = name + "@" + owner; + } + + public void enableOverride() { + this.overrideId = true; + } + + @Override + public String getOwnerName() { + return ownerName; + } + + @Override + public String getUniqueName() { + return uniqueName; + } + + @Override + public String getName() { + return overrideId ? uniqueName : super.getName(); + } + + @Override + public int getId() { + return overrideId ? getUniqueId() : super.getId(); + } + + @Override + public String toString() { + return "{OwnedStructDataType "+uniqueName+" id="+getId()+" uid="+getUniqueId()+" enable override="+overrideId+"}"; + } +} diff --git a/config-model/src/main/java/com/yahoo/documentmodel/OwnedTemporaryType.java b/config-model/src/main/java/com/yahoo/documentmodel/OwnedTemporaryType.java new file mode 100644 index 00000000000..536c10ee242 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/documentmodel/OwnedTemporaryType.java @@ -0,0 +1,41 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.documentmodel; + +import com.yahoo.document.DocumentType; +import com.yahoo.document.StructDataType; + +/** + * Proxy for a struct type declared in a specific document + * + * @author arnej + **/ +public final class OwnedTemporaryType extends StructDataType implements OwnedType { + + private final String ownerName; + private final String uniqueName; + + public OwnedTemporaryType(String name, DocumentType document) { + this(name, document.getName()); + } + + public OwnedTemporaryType(String name, String owner) { + super(name); + this.ownerName = owner; + this.uniqueName = name + "@" + owner; + } + + @Override + public String getOwnerName() { + return ownerName; + } + + @Override + public String getUniqueName() { + return uniqueName; + } + + @Override + public String toString() { + return "{OwnedTemporaryType "+uniqueName+" id="+getId()+" uid="+getUniqueId()+"}"; + } +} diff --git a/config-model/src/main/java/com/yahoo/documentmodel/OwnedType.java b/config-model/src/main/java/com/yahoo/documentmodel/OwnedType.java new file mode 100644 index 00000000000..e3a91692ca8 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/documentmodel/OwnedType.java @@ -0,0 +1,15 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.documentmodel; + +/** + * API for a type declared in a specific document + * + * @author arnej + **/ +public interface OwnedType { + String getOwnerName(); + String getUniqueName(); + default int getUniqueId() { + return getUniqueName().hashCode(); + } +} diff --git a/config-model/src/main/java/com/yahoo/documentmodel/TemporaryUnknownType.java b/config-model/src/main/java/com/yahoo/documentmodel/TemporaryUnknownType.java new file mode 100644 index 00000000000..66f6354b3f5 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/documentmodel/TemporaryUnknownType.java @@ -0,0 +1,21 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.documentmodel; + +import com.yahoo.document.StructDataType; + +/** + * Proxy for an unknown type (must resolve to struct or document) + * + * @author arnej + **/ +public final class TemporaryUnknownType extends StructDataType { + + public TemporaryUnknownType(String name) { + super(name); + } + + @Override + public String toString() { + return "{TemporaryUnknownType "+getName()+" id="+getId()+"}"; + } +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/DocumentModelBuilder.java b/config-model/src/main/java/com/yahoo/searchdefinition/DocumentModelBuilder.java index 4660b81ff72..5b7b23d69d4 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/DocumentModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/DocumentModelBuilder.java @@ -7,7 +7,6 @@ import com.yahoo.document.DataType; import com.yahoo.document.DocumentType; import com.yahoo.document.Field; import com.yahoo.document.MapDataType; -import com.yahoo.documentmodel.NewDocumentReferenceDataType; import com.yahoo.document.StructDataType; import com.yahoo.document.StructuredDataType; import com.yahoo.document.TemporaryStructuredDataType; @@ -15,7 +14,11 @@ import com.yahoo.document.WeightedSetDataType; import com.yahoo.document.annotation.AnnotationReferenceDataType; import com.yahoo.document.annotation.AnnotationType; import com.yahoo.documentmodel.DataTypeCollection; +import com.yahoo.documentmodel.NewDocumentReferenceDataType; import com.yahoo.documentmodel.NewDocumentType; +import com.yahoo.documentmodel.OwnedStructDataType; +import com.yahoo.documentmodel.OwnedTemporaryType; +import com.yahoo.documentmodel.TemporaryUnknownType; import com.yahoo.documentmodel.VespaDocumentType; import com.yahoo.searchdefinition.document.Attribute; import com.yahoo.searchdefinition.document.SDDocumentType; @@ -232,13 +235,37 @@ public class DocumentModelBuilder { } DataType original = type; if (type instanceof TemporaryStructuredDataType) { + throw new IllegalArgumentException("Cannot handle temporary: " + type); + } + if (type instanceof TemporaryUnknownType) { + // must be a known struct or document type DataType other = repo.getDataType(type.getId()); if (other == null || other == type) { + // maybe it is the name of a document type: other = getDocumentType(docs, type.getName()); } - if (other != null) { - type = other; + if (other == null) { + throw new IllegalArgumentException("No replacement found for temporary type: " + type); + } + type = other; + } else if (type instanceof OwnedTemporaryType) { + // must be replaced with the real struct type + DataType other = repo.getDataType(type.getId()); + if (other == null || other == type) { + throw new IllegalArgumentException("No replacement found for temporary type: " + type); + } + if (other instanceof OwnedStructDataType) { + var owned = (OwnedTemporaryType) type; + String ownedBy = owned.getOwnerName(); + var otherOwned = (OwnedStructDataType) other; + String otherOwnedBy = otherOwned.getOwnerName(); + if (! ownedBy.equals(otherOwnedBy)) { + throw new IllegalArgumentException("Wrong document for type: " + otherOwnedBy + " but expected " + ownedBy); + } + } else { + throw new IllegalArgumentException("Found wrong sort of type: " + other + " [" + other.getClass() + "]"); } + type = other; } else if (type instanceof DocumentType) { DataType other = getDocumentType(docs, type.getName()); if (other != null) { @@ -364,6 +391,8 @@ public class DocumentModelBuilder { for (SDDocumentType proxy : type.getInheritedTypes()) { var inherited = (StructDataType) targetDt.getDataTypeRecursive(proxy.getName()); var converted = (StructDataType) targetDt.getDataType(type.getName()); + assert(converted instanceof OwnedStructDataType); + assert(inherited instanceof OwnedStructDataType); if (! converted.inherits(inherited)) { converted.inherit(inherited); } @@ -382,15 +411,15 @@ public class DocumentModelBuilder { StructDataType s = handleStruct(sa.getSdDocType()); annotation.setDataType(s); if ((sa.getInherits() != null)) { - structInheritance.put(s, "annotation."+sa.getInherits()); + structInheritance.put(s, "annotation." + sa.getInherits()); } } else if (sa.getInherits() != null) { - StructDataType s = new StructDataType("annotation."+annotation.getName()); + StructDataType s = new OwnedStructDataType("annotation." + annotation.getName(), sdoc.getName()); if (anyParentsHavePayLoad(sa, sdoc)) { annotation.setDataType(s); addType(s); } - structInheritance.put(s, "annotation."+sa.getInherits()); + structInheritance.put(s, "annotation." + sa.getInherits()); } } else { var dt = annotation.getDataType(); @@ -532,16 +561,18 @@ public class DocumentModelBuilder { @SuppressWarnings("deprecation") private StructDataType handleStruct(SDDocumentType type) { + // System.err.println("handle struct " + type + " for doc " + targetDt.getName()); if (type.isStruct()) { var st = type.getStruct(); if (st.getName().equals(type.getName()) && (st instanceof StructDataType) && - ! (st instanceof TemporaryStructuredDataType)) + (! (st instanceof TemporaryUnknownType)) && + (! (st instanceof OwnedTemporaryType))) { return handleStruct((StructDataType) st); } } - StructDataType s = new StructDataType(type.getName()); + StructDataType s = new OwnedStructDataType(type.getName(), targetDt.getName()); for (Field f : type.getDocumentType().contentStruct().getFieldsThisTypeOnly()) { specialHandleAnnotationReference(f); s.addField(f); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/Attribute.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/Attribute.java index c592e4842a9..b466a78c69b 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/document/Attribute.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/Attribute.java @@ -8,7 +8,6 @@ import com.yahoo.document.DocumentType; import com.yahoo.document.PrimitiveDataType; import com.yahoo.documentmodel.NewDocumentReferenceDataType; import com.yahoo.document.StructuredDataType; -import com.yahoo.document.TemporaryStructuredDataType; import com.yahoo.document.TensorDataType; import com.yahoo.document.WeightedSetDataType; import com.yahoo.document.datatypes.BoolFieldValue; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/SDField.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/SDField.java index 621e7ce8571..49ae00d0663 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/document/SDField.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/SDField.java @@ -7,9 +7,10 @@ import com.yahoo.document.DocumentType; import com.yahoo.document.Field; import com.yahoo.document.MapDataType; import com.yahoo.document.StructDataType; -import com.yahoo.document.TemporaryStructuredDataType; import com.yahoo.document.TensorDataType; import com.yahoo.document.WeightedSetDataType; +import com.yahoo.documentmodel.OwnedTemporaryType; +import com.yahoo.documentmodel.TemporaryUnknownType; import com.yahoo.language.Linguistics; import com.yahoo.language.process.Embedder; import com.yahoo.language.simple.SimpleLinguistics; @@ -307,7 +308,11 @@ public class SDField extends Field implements TypedKey, FieldOperationContainer, return; } SDDocumentType subType = sdoc != null ? sdoc.getType(dataType.getName()) : null; - if (dataType instanceof TemporaryStructuredDataType && subType != null) { + if (dataType instanceof TemporaryUnknownType && subType != null) { + for (Field field : subType.fieldSet()) { + supplyStructField.accept(field.getName(), field.getDataType()); + } + } else if (dataType instanceof OwnedTemporaryType && subType != null) { for (Field field : subType.fieldSet()) { supplyStructField.accept(field.getName(), field.getDataType()); } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ConvertParsedTypes.java b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ConvertParsedTypes.java index f628db85429..8e27b581769 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ConvertParsedTypes.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ConvertParsedTypes.java @@ -4,11 +4,12 @@ package com.yahoo.searchdefinition.parser; import com.yahoo.document.DataType; import com.yahoo.document.DocumentType; import com.yahoo.document.DocumentTypeManager; -import com.yahoo.documentmodel.NewDocumentReferenceDataType; -import com.yahoo.document.StructDataType; import com.yahoo.document.PositionDataType; +import com.yahoo.document.StructDataType; import com.yahoo.document.WeightedSetDataType; import com.yahoo.document.annotation.AnnotationReferenceDataType; +import com.yahoo.documentmodel.NewDocumentReferenceDataType; +import com.yahoo.documentmodel.OwnedStructDataType; import com.yahoo.searchdefinition.document.annotation.SDAnnotationType; import java.util.ArrayList; @@ -58,7 +59,7 @@ public class ConvertParsedTypes { var doc = schema.getDocument(); for (var struct : doc.getStructs()) { String structId = doc.name() + "->" + struct.name(); - var dt = new StructDataType(struct.name()); + var dt = new OwnedStructDataType(struct.name(), doc.name()); structsFromSchemas.put(structId, dt); } for (var annotation : doc.getAnnotations()) { @@ -72,7 +73,7 @@ public class ConvertParsedTypes { if (withStruct.isPresent()) { ParsedStruct struct = withStruct.get(); String structId = doc.name() + "->" + struct.name(); - var old = structsFromSchemas.put(structId, new StructDataType(struct.name())); + var old = structsFromSchemas.put(structId, new OwnedStructDataType(struct.name(), doc.name())); assert(old == null); } } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedBlock.java b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedBlock.java index 213733f7722..151da352f2f 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedBlock.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedBlock.java @@ -15,6 +15,7 @@ public class ParsedBlock { } public final String name() { return name; } + public final String blockType() { return blockType; } protected void verifyThat(boolean check, String msg, Object ... msgDetails) { if (check) return; @@ -28,5 +29,8 @@ public class ParsedBlock { throw new IllegalArgumentException(buf.toString()); } + public String toString() { + return blockType + " '" + name + "'"; + } } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedDocument.java b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedDocument.java index ca128a93590..ea138808289 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedDocument.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedDocument.java @@ -62,8 +62,9 @@ public class ParsedDocument extends ParsedBlock { void inherit(String other) { inherited.add(other); } void addField(ParsedField field) { - String fieldName = field.name(); - verifyThat(! docFields.containsKey(fieldName), "already has field", fieldName); + String fieldName = field.name().toLowerCase(); + verifyThat(! docFields.containsKey(fieldName), + "Duplicate (case insensitively) " + field + " in document type '" + this.name() + "'"); docFields.put(fieldName, field); } @@ -81,8 +82,6 @@ public class ParsedDocument extends ParsedBlock { annotation.tagOwner(this); } - public String toString() { return "document " + name(); } - void resolveInherit(String name, ParsedDocument parsed) { verifyThat(inherited.contains(name), "resolveInherit for non-inherited name", name); verifyThat(name.equals(parsed.name()), "resolveInherit name mismatch for", name); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedType.java b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedType.java index d04277706a1..a5f00b1ce45 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedType.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedType.java @@ -32,6 +32,61 @@ class ParsedType { private boolean createIfNonExistent = false; private boolean removeIfZero = false; + public String toString() { + var buf = new StringBuilder(); + buf.append("[type ").append(variant).append("] {"); + switch (variant) { + case NONE: + break; + case BUILTIN: + buf.append(name); + break; + case POSITION: + buf.append(name); + break; + case TENSOR: + buf.append(tensorType.toString()); + break; + case ARRAY: buf + .append(" array<") + .append(valType.toString()) + .append("> "); + break; + case WSET: buf + .append(" weightedset<") + .append(valType.toString()) + .append(">"); + if (createIfNonExistent) buf.append(",createIfNonExistent"); + if (removeIfZero) buf.append(",removeIfZero"); + buf.append(" "); + break; + case MAP: buf + .append(" map<") + .append(keyType.toString()) + .append(",") + .append(valType.toString()) + .append("> "); + break; + case DOC_REFERENCE: buf + .append(" reference<") + .append(valType.toString()) + .append("> "); + break; + case ANN_REFERENCE: buf + .append(" ") + .append(toString()) + .append(" "); + break; + case STRUCT: + case DOCUMENT: + case UNKNOWN: + buf.append(" ").append(name).append(" "); + break; + } + buf.append("}"); + return buf.toString(); + } + private static Variant guessVariant(String name) { switch (name) { case "bool": return Variant.BUILTIN; diff --git a/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentManager.java b/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentManager.java index ff311795089..8867ab63e2b 100644 --- a/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentManager.java +++ b/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentManager.java @@ -9,6 +9,8 @@ import com.yahoo.document.annotation.AnnotationType; import com.yahoo.documentmodel.DataTypeCollection; import com.yahoo.documentmodel.NewDocumentReferenceDataType; import com.yahoo.documentmodel.NewDocumentType; +import com.yahoo.documentmodel.OwnedTemporaryType; +import com.yahoo.documentmodel.TemporaryUnknownType; import com.yahoo.documentmodel.VespaDocumentType; import com.yahoo.searchdefinition.document.FieldSet; import com.yahoo.vespa.documentmodel.DocumentModel; @@ -83,6 +85,12 @@ public class DocumentManager { if (dataType instanceof TemporaryStructuredDataType) { throw new IllegalArgumentException("Can not create config for temporary data type: " + dataType.getName()); } + if (dataType instanceof TemporaryUnknownType) { + throw new IllegalArgumentException("Can not create config for temporary data type: " + dataType.getName()); + } + if (dataType instanceof OwnedTemporaryType) { + throw new IllegalArgumentException("Can not create config for temporary data type: " + dataType.getName()); + } if ((dataType.getId() < 0) || (dataType.getId()> DataType.lastPredefinedDataTypeId())) { Datatype.Builder dataTypeBuilder = new Datatype.Builder(); documentConfigBuilder.datatype(dataTypeBuilder); @@ -144,8 +152,6 @@ public class DocumentManager { } buildConfig(dt.getFieldSets(), doc); buildImportedFieldsConfig(dt.getImportedFieldNames(), doc); - } else if (type instanceof TemporaryStructuredDataType) { - throw new IllegalArgumentException("Can not create config for temporary data type: " + type.getName()); } else if (type instanceof StructDataType) { StructDataType structType = (StructDataType) type; Datatype.Structtype.Builder structBuilder = new Datatype.Structtype.Builder(); @@ -348,7 +354,11 @@ public class DocumentManager { indexMap.setDone(type); if (type instanceof TemporaryStructuredDataType) { throw new IllegalArgumentException("Can not create config for temporary data type: " + type.getName()); - } if (type instanceof StructDataType) { + } else if (type instanceof TemporaryUnknownType) { + throw new IllegalArgumentException("Can not create config for temporary data type: " + type.getName()); + } else if (type instanceof OwnedTemporaryType) { + throw new IllegalArgumentException("Can not create config for temporary data type: " + type.getName()); + } else if (type instanceof StructDataType) { docTypeBuildOneType((StructDataType) type, documentBuilder, indexMap); } else if (type instanceof ArrayDataType) { docTypeBuildOneType((ArrayDataType) type, documentBuilder, indexMap); diff --git a/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentTypes.java b/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentTypes.java index 1240dac8bf1..e1a28c8114f 100644 --- a/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentTypes.java +++ b/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentTypes.java @@ -8,6 +8,8 @@ import com.yahoo.document.annotation.AnnotationType; import com.yahoo.documentmodel.DataTypeCollection; import com.yahoo.documentmodel.NewDocumentReferenceDataType; import com.yahoo.documentmodel.NewDocumentType; +import com.yahoo.documentmodel.OwnedTemporaryType; +import com.yahoo.documentmodel.TemporaryUnknownType; import com.yahoo.documentmodel.VespaDocumentType; import com.yahoo.searchdefinition.document.FieldSet; import com.yahoo.vespa.documentmodel.DocumentModel; @@ -111,6 +113,12 @@ public class DocumentTypes { built.add(type.getId()); DocumenttypesConfig.Documenttype.Datatype.Builder dataTypeBuilder = new DocumenttypesConfig.Documenttype.Datatype.Builder(); dataTypeBuilder.id(type.getId()); + if (type instanceof TemporaryUnknownType) { + throw new IllegalArgumentException("Can not create config for temporary data type: " + type.getName()); + } + if (type instanceof OwnedTemporaryType) { + throw new IllegalArgumentException("Can not create config for temporary data type: " + type.getName()); + } if (type instanceof StructDataType) { buildConfig((StructDataType) type, dataTypeBuilder, documentBuilder, built); } else if (type instanceof ArrayDataType) { diff --git a/config-model/src/main/javacc/SDParser.jj b/config-model/src/main/javacc/SDParser.jj index 018531616fb..ab0cdefc355 100644 --- a/config-model/src/main/javacc/SDParser.jj +++ b/config-model/src/main/javacc/SDParser.jj @@ -826,9 +826,10 @@ SDDocumentType structDefinition(Schema schema, SDDocumentType repo) : // empty } SDDocumentType sdtype = repo.getOwnedType(struct.getDocumentName()); - DataType stype = sdtype != null - ? sdtype.getStruct() - : TemporaryStructuredDataType.create(struct.getName()); + if (sdtype != null) { + throw new ParseException("Struct '" + name + "' is already defined."); + } + DataType stype = new OwnedTemporaryType(name, repo.getName()); struct.setStruct(stype); return struct; } @@ -847,7 +848,6 @@ DataType dataType() : DataType arrayType = null; DataType wsetType = null; TensorType tensorType; - TemporaryStructuredDataType referenceType; String referencedDoc; } { @@ -863,9 +863,9 @@ DataType dataType() : DataType type = VespaDocumentType.INSTANCE.getDataType(typeName); if (type == null) { - // we are basically creating TemporaryStructDataType instances for ANYTHING here!! + // we are basically creating TemporaryUnknownType instances for ANYTHING here!! // we must do this and clean them up later. - type = TemporaryStructuredDataType.create(typeName); + type = new TemporaryUnknownType(typeName); } if (isArrayOldStyle) { diff --git a/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java b/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java index 91aff3935ab..183ab56d45f 100644 --- a/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java +++ b/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java @@ -48,6 +48,9 @@ import java.util.function.Function; import java.util.stream.Collectors; import static com.yahoo.config.model.test.TestUtil.joinLines; +import static com.yahoo.config.provision.NodeResources.Architecture; +import static com.yahoo.config.provision.NodeResources.DiskSpeed; +import static com.yahoo.config.provision.NodeResources.StorageType; import static com.yahoo.vespa.defaults.Defaults.getDefaults; import static com.yahoo.vespa.model.search.NodeResourcesTuning.GB; import static com.yahoo.vespa.model.search.NodeResourcesTuning.reservedMemoryGb; @@ -550,7 +553,7 @@ public class ModelProvisioningTest { assertEquals(3, subGroups.get(0).getNodes().size()); assertEquals(0, subGroups.get(0).getNodes().get(0).getDistributionKey()); assertEquals("bar/storage/0", subGroups.get(0).getNodes().get(0).getConfigId()); - assertEquals("node-1-3-10-57", subGroups.get(0).getNodes().get(0).getHostName()); + assertEquals("node-1-3-50-57", subGroups.get(0).getNodes().get(0).getHostName()); assertEquals(1, subGroups.get(0).getNodes().get(1).getDistributionKey()); assertEquals("bar/storage/1", subGroups.get(0).getNodes().get(1).getConfigId()); assertEquals(2, subGroups.get(0).getNodes().get(2).getDistributionKey()); @@ -559,13 +562,13 @@ public class ModelProvisioningTest { assertEquals(3, subGroups.get(1).getNodes().size()); assertEquals(3, subGroups.get(1).getNodes().get(0).getDistributionKey()); assertEquals("bar/storage/3", subGroups.get(1).getNodes().get(0).getConfigId()); - assertEquals("node-1-3-10-54", subGroups.get(1).getNodes().get(0).getHostName()); + assertEquals("node-1-3-50-54", subGroups.get(1).getNodes().get(0).getHostName()); assertEquals(4, subGroups.get(1).getNodes().get(1).getDistributionKey()); assertEquals("bar/storage/4", subGroups.get(1).getNodes().get(1).getConfigId()); assertEquals(5, subGroups.get(1).getNodes().get(2).getDistributionKey()); assertEquals("bar/storage/5", subGroups.get(1).getNodes().get(2).getConfigId()); // ... - assertEquals("node-1-3-10-51", subGroups.get(2).getNodes().get(0).getHostName()); + assertEquals("node-1-3-50-51", subGroups.get(2).getNodes().get(0).getHostName()); // ... assertEquals("8", subGroups.get(8).getIndex()); assertEquals(3, subGroups.get(8).getNodes().size()); @@ -584,14 +587,14 @@ public class ModelProvisioningTest { assertEquals(1, subGroups.get(0).getNodes().size()); assertEquals(0, subGroups.get(0).getNodes().get(0).getDistributionKey()); assertEquals("baz/storage/0", subGroups.get(0).getNodes().get(0).getConfigId()); - assertEquals("node-1-3-10-27", subGroups.get(0).getNodes().get(0).getHostName()); + assertEquals("node-1-3-50-27", subGroups.get(0).getNodes().get(0).getHostName()); assertEquals("1", subGroups.get(1).getIndex()); assertEquals(1, subGroups.get(1).getNodes().size()); assertEquals(1, subGroups.get(1).getNodes().get(0).getDistributionKey()); assertEquals("baz/storage/1", subGroups.get(1).getNodes().get(0).getConfigId()); - assertEquals("node-1-3-10-26", subGroups.get(1).getNodes().get(0).getHostName()); + assertEquals("node-1-3-50-26", subGroups.get(1).getNodes().get(0).getHostName()); // ... - assertEquals("node-1-3-10-25", subGroups.get(2).getNodes().get(0).getHostName()); + assertEquals("node-1-3-50-25", subGroups.get(2).getNodes().get(0).getHostName()); // ... assertEquals("26", subGroups.get(26).getIndex()); assertEquals(1, subGroups.get(26).getNodes().size()); @@ -691,7 +694,7 @@ public class ModelProvisioningTest { assertEquals(3, subGroups.get(0).getNodes().size()); assertEquals(0, subGroups.get(0).getNodes().get(0).getDistributionKey()); assertEquals("bar/storage/0", subGroups.get(0).getNodes().get(0).getConfigId()); - assertEquals("node-1-3-10-57", subGroups.get(0).getNodes().get(0).getHostName()); + assertEquals("node-1-3-50-57", subGroups.get(0).getNodes().get(0).getHostName()); assertEquals(1, subGroups.get(0).getNodes().get(1).getDistributionKey()); assertEquals("bar/storage/1", subGroups.get(0).getNodes().get(1).getConfigId()); assertEquals(2, subGroups.get(0).getNodes().get(2).getDistributionKey()); @@ -700,13 +703,13 @@ public class ModelProvisioningTest { assertEquals(3, subGroups.get(1).getNodes().size()); assertEquals(3, subGroups.get(1).getNodes().get(0).getDistributionKey()); assertEquals("bar/storage/3", subGroups.get(1).getNodes().get(0).getConfigId()); - assertEquals("node-1-3-10-54", subGroups.get(1).getNodes().get(0).getHostName()); + assertEquals("node-1-3-50-54", subGroups.get(1).getNodes().get(0).getHostName()); assertEquals(4, subGroups.get(1).getNodes().get(1).getDistributionKey()); assertEquals("bar/storage/4", subGroups.get(1).getNodes().get(1).getConfigId()); assertEquals(5, subGroups.get(1).getNodes().get(2).getDistributionKey()); assertEquals("bar/storage/5", subGroups.get(1).getNodes().get(2).getConfigId()); // ... - assertEquals("node-1-3-10-51", subGroups.get(2).getNodes().get(0).getHostName()); + assertEquals("node-1-3-50-51", subGroups.get(2).getNodes().get(0).getHostName()); // ... assertEquals("8", subGroups.get(8).getIndex()); assertEquals(3, subGroups.get(8).getNodes().size()); @@ -725,14 +728,14 @@ public class ModelProvisioningTest { assertEquals(1, subGroups.get(0).getNodes().size()); assertEquals(0, subGroups.get(0).getNodes().get(0).getDistributionKey()); assertEquals("baz/storage/0", subGroups.get(0).getNodes().get(0).getConfigId()); - assertEquals("node-1-3-10-27", subGroups.get(0).getNodes().get(0).getHostName()); + assertEquals("node-1-3-50-27", subGroups.get(0).getNodes().get(0).getHostName()); assertEquals("1", subGroups.get(1).getIndex()); assertEquals(1, subGroups.get(1).getNodes().size()); assertEquals(1, subGroups.get(1).getNodes().get(0).getDistributionKey()); assertEquals("baz/storage/1", subGroups.get(1).getNodes().get(0).getConfigId()); - assertEquals("node-1-3-10-26", subGroups.get(1).getNodes().get(0).getHostName()); + assertEquals("node-1-3-50-26", subGroups.get(1).getNodes().get(0).getHostName()); // ... - assertEquals("node-1-3-10-25", subGroups.get(2).getNodes().get(0).getHostName()); + assertEquals("node-1-3-50-25", subGroups.get(2).getNodes().get(0).getHostName()); // ... assertEquals("26", subGroups.get(26).getIndex()); assertEquals(1, subGroups.get(26).getNodes().size()); @@ -767,9 +770,9 @@ public class ModelProvisioningTest { ClusterControllerContainerCluster clusterControllers = model.getAdmin().getClusterControllers(); assertEquals(3, clusterControllers.getContainers().size()); assertEquals("cluster-controllers", clusterControllers.getName()); - assertEquals("node-1-3-10-03", clusterControllers.getContainers().get(0).getHostName()); - assertEquals("node-1-3-10-02", clusterControllers.getContainers().get(1).getHostName()); - assertEquals("node-1-3-10-01", clusterControllers.getContainers().get(2).getHostName()); + assertEquals("node-1-3-50-03", clusterControllers.getContainers().get(0).getHostName()); + assertEquals("node-1-3-50-02", clusterControllers.getContainers().get(1).getHostName()); + assertEquals("node-1-3-50-01", clusterControllers.getContainers().get(2).getHostName()); // Check content cluster ContentCluster cluster = model.getContentClusters().get("bar"); @@ -782,19 +785,19 @@ public class ModelProvisioningTest { assertEquals(1, subGroups.get(0).getNodes().size()); assertEquals(0, subGroups.get(0).getNodes().get(0).getDistributionKey()); assertEquals("bar/storage/0", subGroups.get(0).getNodes().get(0).getConfigId()); - assertEquals("node-1-3-10-11", subGroups.get(0).getNodes().get(0).getHostName()); + assertEquals("node-1-3-50-11", subGroups.get(0).getNodes().get(0).getHostName()); // second group assertEquals("1", subGroups.get(1).getIndex()); assertEquals(1, subGroups.get(1).getNodes().size()); assertEquals(1, subGroups.get(1).getNodes().get(0).getDistributionKey()); assertEquals("bar/storage/1", subGroups.get(1).getNodes().get(0).getConfigId()); - assertEquals("node-1-3-10-10", subGroups.get(1).getNodes().get(0).getHostName()); + assertEquals("node-1-3-50-10", subGroups.get(1).getNodes().get(0).getHostName()); // ... last group assertEquals("7", subGroups.get(7).getIndex()); assertEquals(1, subGroups.get(7).getNodes().size()); assertEquals(7, subGroups.get(7).getNodes().get(0).getDistributionKey()); assertEquals("bar/storage/7", subGroups.get(7).getNodes().get(0).getConfigId()); - assertEquals("node-1-3-10-04", subGroups.get(7).getNodes().get(0).getHostName()); + assertEquals("node-1-3-50-04", subGroups.get(7).getNodes().get(0).getHostName()); } @Test @@ -811,15 +814,15 @@ public class ModelProvisioningTest { int numberOfHosts = 11; VespaModelTester tester = new VespaModelTester(); tester.addHosts(numberOfHosts); - VespaModel model = tester.createModel(services, true, "node-1-3-10-09"); + VespaModel model = tester.createModel(services, true, "node-1-3-50-09"); assertEquals(numberOfHosts, model.getRoot().hostSystem().getHosts().size()); // Check slobroks clusters assertEquals("Includes retired node", 1+3, model.getAdmin().getSlobroks().size()); - assertEquals("node-1-3-10-11", model.getAdmin().getSlobroks().get(0).getHostName()); - assertEquals("node-1-3-10-10", model.getAdmin().getSlobroks().get(1).getHostName()); - assertEquals("node-1-3-10-08", model.getAdmin().getSlobroks().get(2).getHostName()); - assertEquals("Included in addition because it is retired", "node-1-3-10-09", model.getAdmin().getSlobroks().get(3).getHostName()); + assertEquals("node-1-3-50-11", model.getAdmin().getSlobroks().get(0).getHostName()); + assertEquals("node-1-3-50-10", model.getAdmin().getSlobroks().get(1).getHostName()); + assertEquals("node-1-3-50-08", model.getAdmin().getSlobroks().get(2).getHostName()); + assertEquals("Included in addition because it is retired", "node-1-3-50-09", model.getAdmin().getSlobroks().get(3).getHostName()); } @Test @@ -836,16 +839,16 @@ public class ModelProvisioningTest { int numberOfHosts = 12; VespaModelTester tester = new VespaModelTester(); tester.addHosts(numberOfHosts); - VespaModel model = tester.createModel(services, true, "node-1-3-10-03", "node-1-3-10-04"); + VespaModel model = tester.createModel(services, true, "node-1-3-50-03", "node-1-3-50-04"); assertEquals(10+2, model.getRoot().hostSystem().getHosts().size()); // Check slobroks clusters assertEquals("Includes retired node", 3+2, model.getAdmin().getSlobroks().size()); - assertEquals("node-1-3-10-12", model.getAdmin().getSlobroks().get(0).getHostName()); - assertEquals("node-1-3-10-11", model.getAdmin().getSlobroks().get(1).getHostName()); - assertEquals("node-1-3-10-10", model.getAdmin().getSlobroks().get(2).getHostName()); - assertEquals("Included in addition because it is retired", "node-1-3-10-04", model.getAdmin().getSlobroks().get(3).getHostName()); - assertEquals("Included in addition because it is retired", "node-1-3-10-03", model.getAdmin().getSlobroks().get(4).getHostName()); + assertEquals("node-1-3-50-12", model.getAdmin().getSlobroks().get(0).getHostName()); + assertEquals("node-1-3-50-11", model.getAdmin().getSlobroks().get(1).getHostName()); + assertEquals("node-1-3-50-10", model.getAdmin().getSlobroks().get(2).getHostName()); + assertEquals("Included in addition because it is retired", "node-1-3-50-04", model.getAdmin().getSlobroks().get(3).getHostName()); + assertEquals("Included in addition because it is retired", "node-1-3-50-03", model.getAdmin().getSlobroks().get(4).getHostName()); } @Test @@ -865,19 +868,19 @@ public class ModelProvisioningTest { int numberOfHosts = 16; VespaModelTester tester = new VespaModelTester(); tester.addHosts(numberOfHosts); - VespaModel model = tester.createModel(services, true, "node-1-3-10-15", "node-1-3-10-05", "node-1-3-10-04"); + VespaModel model = tester.createModel(services, true, "node-1-3-50-15", "node-1-3-50-05", "node-1-3-50-04"); assertEquals(numberOfHosts, model.getRoot().hostSystem().getHosts().size()); // Check slobroks clusters // ... from cluster default assertEquals("Includes retired node", 7, model.getAdmin().getSlobroks().size()); - assertEquals("node-1-3-10-16", model.getAdmin().getSlobroks().get(0).getHostName()); - assertEquals("node-1-3-10-14", model.getAdmin().getSlobroks().get(1).getHostName()); - assertEquals("Included in addition because it is retired", "node-1-3-10-15", model.getAdmin().getSlobroks().get(2).getHostName()); + assertEquals("node-1-3-50-16", model.getAdmin().getSlobroks().get(0).getHostName()); + assertEquals("node-1-3-50-14", model.getAdmin().getSlobroks().get(1).getHostName()); + assertEquals("Included in addition because it is retired", "node-1-3-50-15", model.getAdmin().getSlobroks().get(2).getHostName()); // ... from cluster bar - assertEquals("node-1-3-10-03", model.getAdmin().getSlobroks().get(3).getHostName()); - assertEquals("Included in addition because it is retired", "node-1-3-10-05", model.getAdmin().getSlobroks().get(5).getHostName()); - assertEquals("Included in addition because it is retired", "node-1-3-10-04", model.getAdmin().getSlobroks().get(6).getHostName()); + assertEquals("node-1-3-50-03", model.getAdmin().getSlobroks().get(3).getHostName()); + assertEquals("Included in addition because it is retired", "node-1-3-50-05", model.getAdmin().getSlobroks().get(5).getHostName()); + assertEquals("Included in addition because it is retired", "node-1-3-50-04", model.getAdmin().getSlobroks().get(6).getHostName()); } @Test @@ -1104,7 +1107,7 @@ public class ModelProvisioningTest { int numberOfHosts = 3; VespaModelTester tester = new VespaModelTester(); tester.addHosts(numberOfHosts); - VespaModel model = tester.createModel(services, false, "node-1-3-10-03"); + VespaModel model = tester.createModel(services, false, "node-1-3-50-03"); assertEquals(numberOfHosts, model.getRoot().hostSystem().getHosts().size()); ContentCluster cluster = model.getContentClusters().get("bar"); @@ -1133,7 +1136,7 @@ public class ModelProvisioningTest { int numberOfHosts = 5; VespaModelTester tester = new VespaModelTester(); tester.addHosts(numberOfHosts); - VespaModel model = tester.createModel(services, false, "node-1-3-10-05", "node-1-3-10-04", "node-1-3-10-03"); + VespaModel model = tester.createModel(services, false, "node-1-3-50-05", "node-1-3-50-04", "node-1-3-50-03"); assertEquals(numberOfHosts, model.getRoot().hostSystem().getHosts().size()); ContentCluster cluster = model.getContentClusters().get("bar"); @@ -1162,7 +1165,7 @@ public class ModelProvisioningTest { int numberOfHosts = 3; VespaModelTester tester = new VespaModelTester(); tester.addHosts(numberOfHosts); - VespaModel model = tester.createModel(services, false, false, true, "node-1-3-10-03"); + VespaModel model = tester.createModel(services, false, false, true, "node-1-3-50-03"); assertEquals(numberOfHosts, model.getRoot().hostSystem().getHosts().size()); ContentCluster cluster = model.getContentClusters().get("bar"); @@ -1474,10 +1477,12 @@ public class ModelProvisioningTest { } @Test - public void testUseArm64NodesForClusterControllers() { + public void testUseArm64NodesForAdminCluster() { String services = "<?xml version='1.0' encoding='utf-8' ?>" + "<services>" + + " <admin version='4.0'>" + + " </admin>" + " <container version='1.0' id='container'>" + " <nodes count='2'>" + " <resources vcpu='2' memory='8Gb' disk='30Gb'/>" + @@ -1495,25 +1500,34 @@ public class ModelProvisioningTest { VespaModelTester tester = new VespaModelTester(); tester.setHosted(true); - tester.setAdminClusterArchitecture("arm64"); - tester.addHosts(new NodeResources(13.5, 100, 1000, 0.3), 6); - tester.addHosts(new NodeResources(85, 200, 1000_000_000, 0.3), 20); - tester.addHosts(new NodeResources(0.5, 2, 10, 0.3, NodeResources.DiskSpeed.any, NodeResources.StorageType.any, NodeResources.Architecture.arm64), 3); + tester.setAdminClusterArchitecture(Architecture.arm64); + tester.useDedicatedNodeForLogserver(true); + tester.addHosts(new NodeResources(13.5, 100, 1000, 0.3), 4); + tester.addHosts(new NodeResources(0.5, 2, 50, 0.3, DiskSpeed.fast, StorageType.any, Architecture.arm64), 4); // 3 ccs, 1 logserver VespaModel model = tester.createModel(services, true, true); List<HostResource> hosts = model.getRoot().hostSystem().getHosts(); - assertEquals(7, hosts.size()); - Set<HostResource> clusterControllerResources = - hosts.stream() - .filter(host -> host.getHostInfo().getServices().stream() - .anyMatch(service -> service.getServiceType().equals("container-clustercontroller"))) - .collect(Collectors.toSet()); + assertEquals(8, hosts.size()); + + Set<HostResource> clusterControllerResources = getHostResourcesForService(hosts, "container-clustercontroller"); assertEquals(3, clusterControllerResources.size()); - assertTrue(clusterControllerResources.stream().allMatch(host -> host.realResources().architecture().name().equals("arm64"))); + assertTrue(clusterControllerResources.stream().allMatch(host -> host.realResources().architecture() == Architecture.arm64)); - // Other hosts should be x86_64 + Set<HostResource> logserverResources = getHostResourcesForService(hosts, "logserver-container"); + assertEquals(1, logserverResources.size()); + assertTrue(logserverResources.stream().allMatch(host -> host.realResources().architecture() == Architecture.arm64)); + + // Other hosts should be default assertTrue(hosts.stream() .filter(host -> !clusterControllerResources.contains(host)) - .allMatch(host -> host.realResources().architecture().name().equals("x86_64"))); + .filter(host -> !logserverResources.contains(host)) + .allMatch(host -> host.realResources().architecture() == Architecture.getDefault())); + } + + private Set<HostResource> getHostResourcesForService(List<HostResource> hosts, String service) { + return hosts.stream() + .filter(host -> host.getHostInfo().getServices().stream() + .anyMatch(s -> s.getServiceType().equals(service))) + .collect(Collectors.toSet()); } @Test @@ -2132,7 +2146,7 @@ public class ModelProvisioningTest { "</services>"; VespaModelTester tester = new VespaModelTester(); tester.addHosts(4); - VespaModel model = tester.createModel(servicesXml, true, "node-1-3-10-04"); + VespaModel model = tester.createModel(servicesXml, true, "node-1-3-50-04"); ApplicationContainerCluster cluster = model.getContainerClusters().get("zk"); assertEquals(1, cluster.getContainers().stream().filter(Container::isRetired).count()); assertEquals(3, cluster.getContainers().stream().filter(c -> !c.isRetired()).count()); @@ -2161,7 +2175,7 @@ public class ModelProvisioningTest { assertTrue("Initial servers are not joining", config.build().server().stream().noneMatch(ZookeeperServerConfig.Server::joining)); } { - VespaModel nextModel = tester.createModel(Zone.defaultZone(), servicesXml.apply(3), true, false, false, 0, Optional.of(model), new DeployState.Builder(), "node-1-3-10-04", "node-1-3-10-03"); + VespaModel nextModel = tester.createModel(Zone.defaultZone(), servicesXml.apply(3), true, false, false, 0, Optional.of(model), new DeployState.Builder(), "node-1-3-50-04", "node-1-3-50-03"); ApplicationContainerCluster cluster = nextModel.getContainerClusters().get("zk"); ZookeeperServerConfig.Builder config = new ZookeeperServerConfig.Builder(); cluster.getContainers().forEach(c -> c.getConfig(config)); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/DocumentGraphValidatorTest.java b/config-model/src/test/java/com/yahoo/searchdefinition/DocumentGraphValidatorTest.java index 8c8cb59e9d9..5bbb751585b 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/DocumentGraphValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/DocumentGraphValidatorTest.java @@ -3,7 +3,6 @@ package com.yahoo.searchdefinition; import com.yahoo.config.model.test.MockApplicationPackage; import com.yahoo.documentmodel.NewDocumentReferenceDataType; -import com.yahoo.document.TemporaryStructuredDataType; import com.yahoo.searchdefinition.document.SDDocumentType; import com.yahoo.searchdefinition.document.SDField; import com.yahoo.searchdefinition.document.TemporarySDField; diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/DocumentReferenceResolverTest.java b/config-model/src/test/java/com/yahoo/searchdefinition/DocumentReferenceResolverTest.java index b23ad4eb5b6..fe1b19be64b 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/DocumentReferenceResolverTest.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/DocumentReferenceResolverTest.java @@ -4,7 +4,6 @@ package com.yahoo.searchdefinition; import com.yahoo.config.model.test.MockApplicationPackage; import com.yahoo.document.DataType; import com.yahoo.documentmodel.NewDocumentReferenceDataType; -import com.yahoo.document.TemporaryStructuredDataType; import com.yahoo.searchdefinition.document.SDDocumentType; import com.yahoo.searchdefinition.document.SDField; import org.junit.Rule; diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/SchemaOrdererTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/SchemaOrdererTestCase.java index ed3f5cb0ba6..11e21c7915d 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/SchemaOrdererTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/SchemaOrdererTestCase.java @@ -3,7 +3,6 @@ package com.yahoo.searchdefinition.derived; import com.yahoo.config.model.test.MockApplicationPackage; import com.yahoo.documentmodel.NewDocumentReferenceDataType; -import com.yahoo.document.TemporaryStructuredDataType; import com.yahoo.searchdefinition.DocumentReference; import com.yahoo.searchdefinition.DocumentReferences; import com.yahoo.searchdefinition.Schema; diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/VsmFieldsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/VsmFieldsTestCase.java index 2d6b3acc4dd..9855ca30ebc 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/VsmFieldsTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/VsmFieldsTestCase.java @@ -5,7 +5,6 @@ import com.yahoo.config.model.application.provider.MockFileRegistry; import com.yahoo.config.model.deploy.TestProperties; import com.yahoo.config.model.test.MockApplicationPackage; import com.yahoo.documentmodel.NewDocumentReferenceDataType; -import com.yahoo.document.TemporaryStructuredDataType; import com.yahoo.searchdefinition.Application; import com.yahoo.searchdefinition.Schema; import com.yahoo.searchdefinition.document.SDDocumentType; diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/parser/IntermediateParserTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/parser/IntermediateParserTestCase.java index 8bd04af8c54..36a72381156 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/parser/IntermediateParserTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/parser/IntermediateParserTestCase.java @@ -69,7 +69,7 @@ public class IntermediateParserTestCase { " }", "}"); var e = assertThrows(IllegalArgumentException.class, () -> parseString(input)); - assertEquals("schema 'foo' error: already has document foo so cannot add document foo2", e.getMessage()); + assertEquals("schema 'foo' error: already has document 'foo' so cannot add document 'foo2'", e.getMessage()); } void checkFileParses(String fileName) throws Exception { diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/parser/ParsedDocumentTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/parser/ParsedDocumentTestCase.java index 79d26fab404..9ae7b3aa4b8 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/parser/ParsedDocumentTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/parser/ParsedDocumentTestCase.java @@ -21,7 +21,10 @@ public class ParsedDocumentTestCase { var e = assertThrows(IllegalArgumentException.class, () -> doc.addField(new ParsedField("zap", stringType))); System.err.println("As expected: "+e); - assertEquals("document 'foo' error: already has field zap", e.getMessage()); + assertEquals("document 'foo' error: Duplicate (case insensitively) field 'zap' in document type 'foo'", e.getMessage()); + e = assertThrows(IllegalArgumentException.class, () -> + doc.addField(new ParsedField("ZAP", stringType))); + assertEquals("document 'foo' error: Duplicate (case insensitively) field 'ZAP' in document type 'foo'", e.getMessage()); } } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ParentChildSearchModel.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ParentChildSearchModel.java index 0b615595794..6ddacd066b1 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ParentChildSearchModel.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ParentChildSearchModel.java @@ -7,7 +7,6 @@ import com.yahoo.config.model.deploy.TestProperties; import com.yahoo.config.model.test.MockApplicationPackage; import com.yahoo.document.DataType; import com.yahoo.documentmodel.NewDocumentReferenceDataType; -import com.yahoo.document.TemporaryStructuredDataType; import com.yahoo.searchdefinition.Application; import com.yahoo.searchdefinition.DocumentReference; import com.yahoo.searchdefinition.DocumentReferences; diff --git a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerClusterTest.java index c00a5f65d08..7bf08461df7 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerClusterTest.java @@ -104,7 +104,7 @@ public class MetricsProxyContainerClusterTest { private void assertNodeConfig(MetricsNodesConfig.Node node) { assertTrue(node.role().startsWith("container/foo/0/")); - assertTrue(node.hostname().startsWith("node-1-3-10-")); + assertTrue(node.hostname().startsWith("node-1-3-50-")); assertEquals(MetricsProxyContainer.BASEPORT, node.metricsPort()); assertEquals(MetricsV1Handler.VALUES_PATH, node.metricsPath()); } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerTest.java b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerTest.java index ad3a163f052..9b1ff54482b 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerTest.java @@ -136,7 +136,7 @@ public class MetricsProxyContainerTest { NodeInfoConfig config = hostedModel.getConfig(NodeInfoConfig.class, metricsV2Handler.getConfigId()); assertTrue(config.role().startsWith("content/my-content/0/")); - assertTrue(config.hostname().startsWith("node-1-3-10-")); + assertTrue(config.hostname().startsWith("node-1-3-50-")); } @Test diff --git a/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTester.java b/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTester.java index b9a5fde9301..41f9b4c7b55 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTester.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTester.java @@ -30,6 +30,7 @@ import java.util.List; import java.util.Map; import java.util.Optional; +import static com.yahoo.config.provision.NodeResources.Architecture; import static com.yahoo.vespa.model.test.utils.ApplicationPackageUtils.generateSchemas; /** @@ -53,7 +54,7 @@ public class VespaModelTester { private final Map<NodeResources, Collection<Host>> hostsByResources = new HashMap<>(); private ApplicationId applicationId = ApplicationId.defaultId(); private boolean useDedicatedNodeForLogserver = false; - private String adminClusterArchitecture = "x86_64"; + private Architecture adminClusterArchitecture = Architecture.getDefault();; private HostProvisioner provisioner; public VespaModelTester() { @@ -102,7 +103,9 @@ public class VespaModelTester { public void setHosted(boolean hosted) { this.hosted = hosted; } /** Sets architecture to use for admin clusters. Default: x86_64 */ - public void setAdminClusterArchitecture(String architecture) { this.adminClusterArchitecture = architecture; } + public void setAdminClusterArchitecture(Architecture architecture) { + this.adminClusterArchitecture = architecture; + } /** Sets the tenant, application name, and instance name of the model being built. */ public void setApplicationId(String tenant, String applicationName, String instanceName) { diff --git a/config/src/vespa/config/frt/frtconnectionpool.cpp b/config/src/vespa/config/frt/frtconnectionpool.cpp index 2ae812cd76e..0c11401e052 100644 --- a/config/src/vespa/config/frt/frtconnectionpool.cpp +++ b/config/src/vespa/config/frt/frtconnectionpool.cpp @@ -73,11 +73,13 @@ FRTConnectionPool::getNextRoundRobin() FRTConnection* nextFRTConnection = nullptr; if ( ! ready.empty()) { - int sel = _selectIdx % (int)ready.size(); + unsigned int sel = _selectIdx % (int)ready.size(); + LOG_ASSERT(sel < ready.size()); _selectIdx = sel + 1; nextFRTConnection = ready[sel]; } else if ( ! suspended.empty()) { - int sel = _selectIdx % (int)suspended.size(); + unsigned int sel = _selectIdx % (int)suspended.size(); + LOG_ASSERT(sel < suspended.size()); _selectIdx = sel + 1; nextFRTConnection = suspended[sel]; } @@ -113,10 +115,12 @@ FRTConnectionPool::getNextHashBased() FRTConnection* nextFRTConnection = nullptr; if ( ! ready.empty()) { - int sel = std::abs(hashCode(_hostname) % (int)ready.size()); + unsigned int sel = std::abs(hashCode(_hostname) % (int)ready.size()); + LOG_ASSERT(sel < ready.size()); nextFRTConnection = ready[sel]; } else if ( ! suspended.empty() ){ - int sel = std::abs(hashCode(_hostname) % (int)suspended.size()); + unsigned int sel = std::abs(hashCode(_hostname) % (int)suspended.size()); + LOG_ASSERT(sel < suspended.size()); nextFRTConnection = suspended[sel]; } return nextFRTConnection; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java index 1cfc7758def..718a25cc225 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java @@ -42,6 +42,7 @@ import java.util.Set; import java.util.concurrent.ExecutorService; import java.util.function.ToIntFunction; +import static com.yahoo.config.provision.NodeResources.Architecture; import static com.yahoo.vespa.config.server.ConfigServerSpec.fromConfig; import static com.yahoo.vespa.flags.FetchVector.Dimension.CLUSTER_TYPE; @@ -211,7 +212,7 @@ public class ModelContextImpl implements ModelContext { private final boolean avoidRenamingSummaryFeatures; private final boolean mergeGroupingResultInSearchInvoker; private final boolean experimentalSdParsing; - private final String adminClusterNodeResourcesArchitecture; + private final Architecture adminClusterNodeResourcesArchitecture; public FeatureFlags(FlagSource source, ApplicationId appId, Version version) { this.defaultTermwiseLimit = flagValue(source, appId, version, Flags.DEFAULT_TERM_WISE_LIMIT); @@ -261,7 +262,7 @@ public class ModelContextImpl implements ModelContext { this.avoidRenamingSummaryFeatures = flagValue(source, appId, version, Flags.AVOID_RENAMING_SUMMARY_FEATURES); this.mergeGroupingResultInSearchInvoker = flagValue(source, appId, version, Flags.MERGE_GROUPING_RESULT_IN_SEARCH_INVOKER); this.experimentalSdParsing = flagValue(source, appId, version, Flags.EXPERIMENTAL_SD_PARSING); - this.adminClusterNodeResourcesArchitecture = flagValue(source, appId, version, PermanentFlags.ADMIN_CLUSTER_NODE_ARCHITECTURE); + this.adminClusterNodeResourcesArchitecture = Architecture.valueOf(flagValue(source, appId, version, PermanentFlags.ADMIN_CLUSTER_NODE_ARCHITECTURE)); } @Override public double defaultTermwiseLimit() { return defaultTermwiseLimit; } @@ -313,7 +314,8 @@ public class ModelContextImpl implements ModelContext { @Override public boolean avoidRenamingSummaryFeatures() { return avoidRenamingSummaryFeatures; } @Override public boolean mergeGroupingResultInSearchInvoker() { return mergeGroupingResultInSearchInvoker; } @Override public boolean experimentalSdParsing() { return experimentalSdParsing; } - @Override public String adminClusterNodeArchitecture() { return adminClusterNodeResourcesArchitecture; } + @Override public String adminClusterNodeArchitecture() { return adminClusterArchitecture().name(); } // TODO: Remove when 7.564 is oldest version in use + @Override public Architecture adminClusterArchitecture() { return adminClusterNodeResourcesArchitecture; } private static <V> V flagValue(FlagSource source, ApplicationId appId, Version vespaVersion, UnboundFlag<? extends V, ?, ?> flag) { return flag.bindTo(source) diff --git a/configutil/src/apps/configstatus/main.cpp b/configutil/src/apps/configstatus/main.cpp index cd0424633d9..3656013cf2e 100644 --- a/configutil/src/apps/configstatus/main.cpp +++ b/configutil/src/apps/configstatus/main.cpp @@ -87,7 +87,7 @@ void Application::usage() { int Application::Main() { parseOpts(); - + fprintf(stderr, "Getting config from: %s\n", _specString.c_str()); config::ServerSpec spec(_specString); config::ConfigUri uri = config::ConfigUri::createFromSpec(_cfgId, spec); ConfigStatus status(_flags, uri); diff --git a/configutil/src/lib/configstatus.cpp b/configutil/src/lib/configstatus.cpp index 841d1604866..98a6bca7ba3 100644 --- a/configutil/src/lib/configstatus.cpp +++ b/configutil/src/lib/configstatus.cpp @@ -109,7 +109,7 @@ public: MyHttpHandler::~MyHttpHandler() = default; -ConfigStatus::ConfigStatus(Flags flags, const config::ConfigUri uri) +ConfigStatus::ConfigStatus(Flags flags, const config::ConfigUri &uri) : _cfg(), _flags(flags), _generation(0) { if (_flags.verbose) { diff --git a/configutil/src/lib/configstatus.h b/configutil/src/lib/configstatus.h index 4d792748419..7d658c9f2c2 100644 --- a/configutil/src/lib/configstatus.h +++ b/configutil/src/lib/configstatus.h @@ -22,7 +22,7 @@ public: {} }; - ConfigStatus(Flags flags, const config::ConfigUri uri); + ConfigStatus(Flags flags, const config::ConfigUri &uri); ~ConfigStatus(); int action(); diff --git a/configutil/src/lib/modelinspect.cpp b/configutil/src/lib/modelinspect.cpp index d840a7d45ca..68544a55df4 100644 --- a/configutil/src/lib/modelinspect.cpp +++ b/configutil/src/lib/modelinspect.cpp @@ -21,7 +21,7 @@ ModelInspect::Flags::Flags(const Flags &) = default; ModelInspect::Flags & ModelInspect::Flags::operator = (const Flags &) = default; ModelInspect::Flags::~Flags() { } -ModelInspect::ModelInspect(Flags flags, const config::ConfigUri uri, std::ostream &out) +ModelInspect::ModelInspect(Flags flags, const config::ConfigUri &uri, std::ostream &out) : _cfg(), _flags(flags), _out(out) { if (_flags.verbose) { diff --git a/configutil/src/lib/modelinspect.h b/configutil/src/lib/modelinspect.h index 33ef94b259b..0c677b9d174 100644 --- a/configutil/src/lib/modelinspect.h +++ b/configutil/src/lib/modelinspect.h @@ -20,7 +20,7 @@ public: ~Flags(); }; - ModelInspect(Flags flags, const config::ConfigUri uri, std::ostream &out); + ModelInspect(Flags flags, const config::ConfigUri &uri, std::ostream &out); virtual ~ModelInspect(); int action(int cnt, char *argv[]); diff --git a/container-core/src/main/java/com/yahoo/processing/request/CompoundName.java b/container-core/src/main/java/com/yahoo/processing/request/CompoundName.java index f0a11626236..d571bf583d6 100644 --- a/container-core/src/main/java/com/yahoo/processing/request/CompoundName.java +++ b/container-core/src/main/java/com/yahoo/processing/request/CompoundName.java @@ -67,7 +67,7 @@ public final class CompoundName { this.name = name; this.lowerCasedName = toLowerCase(name); - if (compounds.size()==1 && compounds.get(0).isEmpty()) + if (compounds.size() == 1 && compounds.get(0).isEmpty()) this.compounds = ImmutableList.of(); else this.compounds = ImmutableList.copyOf(compounds); diff --git a/container-search/abi-spec.json b/container-search/abi-spec.json index 8ac5aaa127d..73d4b99b382 100644 --- a/container-search/abi-spec.json +++ b/container-search/abi-spec.json @@ -720,6 +720,7 @@ "public static final enum com.yahoo.prelude.query.Item$ItemType WORD", "public static final enum com.yahoo.prelude.query.Item$ItemType INT", "public static final enum com.yahoo.prelude.query.Item$ItemType PHRASE", + "public static final enum com.yahoo.prelude.query.Item$ItemType MULTI_TERM", "public static final enum com.yahoo.prelude.query.Item$ItemType PREFIX", "public static final enum com.yahoo.prelude.query.Item$ItemType SUBSTRING", "public static final enum com.yahoo.prelude.query.Item$ItemType NEAR", @@ -741,7 +742,6 @@ "public static final enum com.yahoo.prelude.query.Item$ItemType GEO_LOCATION_TERM", "public static final enum com.yahoo.prelude.query.Item$ItemType TRUE", "public static final enum com.yahoo.prelude.query.Item$ItemType FALSE", - "public static final enum com.yahoo.prelude.query.Item$ItemType MULTI_TERM", "public final int code" ] }, diff --git a/container-search/src/main/java/com/yahoo/prelude/query/Item.java b/container-search/src/main/java/com/yahoo/prelude/query/Item.java index 06fe07d3895..2e0c3cf8593 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/Item.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/Item.java @@ -26,10 +26,7 @@ import java.util.Objects; */ public abstract class Item implements Cloneable { - /** - * The definitions in Item.ItemType must match the ones in - * searchlib/src/vespa/searchlib/parsequery/parse.h - */ + // These must match the types in searchlib/src/vespa/searchlib/parsequery/parse.h public enum ItemType { OR(0), AND(1), @@ -38,7 +35,7 @@ public abstract class Item implements Cloneable { WORD(4), INT(5), PHRASE(6), - // 7 was PAREN, unused in Vespa 7 + MULTI_TERM(7), PREFIX(8), SUBSTRING(9), NEAR(11), @@ -59,8 +56,7 @@ public abstract class Item implements Cloneable { NEAREST_NEIGHBOR(26), GEO_LOCATION_TERM(27), TRUE(28), - FALSE(29), - MULTI_TERM(30); + FALSE(29); public final int code; @@ -70,10 +66,7 @@ public abstract class Item implements Cloneable { } - /** - * The definitions in Item.ItemCreator must match the ones in - * searchlib/src/searchlib/parsequery/parse.h - */ + // These must match the definitions in searchlib/src/searchlib/parsequery/parse.h public enum ItemCreator { ORIG(0), @@ -266,11 +259,11 @@ public abstract class Item implements Cloneable { } /** - * Returns an integer that contains all feature flags for this item. This must be kept in sync with the flags - * defined in searchlib/parsequery/parse.h. + * Returns an integer that contains all feature flags for this item. * * @return the feature flags */ + // This must be kept in sync with the flags in searchlib/parsequery/parse.h. private byte getFlagsFeature() { byte FLAGS_NORANK = 0x01; byte FLAGS_SPECIALTOKEN = 0x02; @@ -419,7 +412,7 @@ public abstract class Item implements Cloneable { * the back-end to identify specific items for ranking purposes. * * @param label label for this item - **/ + */ public void setLabel(String label) { setHasUniqueID(true); this.label = label; diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/DocIdNsSpecificValue.java b/container-search/src/main/java/com/yahoo/search/grouping/request/DocIdNsSpecificValue.java index 353e92c723f..b7d8ce9fc77 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/request/DocIdNsSpecificValue.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/request/DocIdNsSpecificValue.java @@ -10,9 +10,7 @@ package com.yahoo.search.grouping.request; */ public class DocIdNsSpecificValue extends DocumentValue { - /** - * Constructs a new instance of this class. - */ + /** Constructs a new instance of this class. */ public DocIdNsSpecificValue() { this(null, null); } diff --git a/container-search/src/main/java/com/yahoo/search/query/QueryTree.java b/container-search/src/main/java/com/yahoo/search/query/QueryTree.java index 115a2f6dbdc..3dac5648660 100644 --- a/container-search/src/main/java/com/yahoo/search/query/QueryTree.java +++ b/container-search/src/main/java/com/yahoo/search/query/QueryTree.java @@ -48,7 +48,7 @@ public class QueryTree extends CompositeItem { return getRoot().encode(buffer); } - //Lets not pollute toString() by adding "ROOT" + // Let's not pollute toString() by adding "ROOT" protected void appendHeadingString(StringBuilder sb) { } @@ -65,7 +65,7 @@ public class QueryTree extends CompositeItem { if (this.getItemCount() == 0) // initializing super.addItem(root); else - setItem(0,root); // replacing + setItem(0, root); // replacing } @Override @@ -88,7 +88,7 @@ public class QueryTree extends CompositeItem { @Override public void addItem(Item item) { - if (getItemCount()==0) + if (getItemCount() == 0) super.addItem(item); else throw new RuntimeException("Programming error: Cannot add multiple roots"); @@ -96,8 +96,8 @@ public class QueryTree extends CompositeItem { @Override public void addItem(int index, Item item) { - if (getItemCount()==0 && index==0) - super.addItem(index,item); + if (getItemCount() == 0 && index == 0) + super.addItem(index, item); else throw new RuntimeException("Programming error: Cannot add multiple roots, have '" + getRoot() + "'"); } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/IssueHandler.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/IssueHandler.java index 6ee5b5a314f..dc8b22ac32d 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/IssueHandler.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/IssueHandler.java @@ -100,4 +100,12 @@ public interface IssueHandler { */ boolean issueExists(Issue issue); + /** + * Returns information about project identified by the project key + * + * @param projectKey The project key to find information for + * @return Project info for project + * @throws RuntimeException exception if project not found + */ + ProjectInfo projectInfo(String projectKey); } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/MockIssueHandler.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/MockIssueHandler.java index 021f05f4d1f..257d2ff5e67 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/MockIssueHandler.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/MockIssueHandler.java @@ -24,6 +24,7 @@ public class MockIssueHandler implements IssueHandler { private final Clock clock; private final AtomicLong counter = new AtomicLong(); private final Map<IssueId, MockIssue> issues = new HashMap<>(); + private final Map<String, ProjectInfo> projects = new HashMap<>(); @Inject @SuppressWarnings("unused") @@ -112,6 +113,11 @@ public class MockIssueHandler implements IssueHandler { return issues.values().stream().anyMatch(i -> i.issue.summary().equals(issue.summary())); } + @Override + public ProjectInfo projectInfo(String projectKey) { + return projects.get(projectKey); + } + public MockIssueHandler close(IssueId issueId) { issues.get(issueId).open = false; touch(issueId); @@ -137,6 +143,10 @@ public class MockIssueHandler implements IssueHandler { issues.get(issueId).updated = clock.instant(); } + public void addProject(String projectKey, ProjectInfo projectInfo) { + projects.put(projectKey, projectInfo); + } + private static class PropertyInfo { private List<List<User>> contacts = Collections.emptyList(); diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/ProjectInfo.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/ProjectInfo.java new file mode 100644 index 00000000000..8bf2ebfd092 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/ProjectInfo.java @@ -0,0 +1,31 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.integration.organization; + +import java.util.Map; + +/** + * @author jvenstad + * @author mortent + */ +public class ProjectInfo { + + private final String id; + private final Map<String, String> componentIds; + + public ProjectInfo(String id, Map<String, String> componentIds) { + this.id = id; + this.componentIds = componentIds; + } + + public boolean hasComponent(String component) { + return componentIds.containsKey(component); + } + + public String id() { + return id; + } + + public Map<String, String> componentIds() { + return componentIds; + } +}
\ No newline at end of file diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index 246f3fff17e..d9a38a5b578 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -1883,6 +1883,13 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { private HttpResponse content(String tenantName, String applicationName, String instanceName, String environment, String region, String restPath, HttpRequest request) { DeploymentId deploymentId = new DeploymentId(ApplicationId.from(tenantName, applicationName, instanceName), requireZone(environment, region)); + + String normalizedRestPath = URI.create("content/" + restPath).normalize().toString(); + // Only content/ is allowed + if ( ! normalizedRestPath.startsWith("content/")) { + return ErrorResponse.forbidden("Access denied"); + } + return controller.serviceRegistry().configServer().getApplicationPackageContent(deploymentId, "/" + restPath, request.getUri()); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java index 8ab43f15c89..f94f87b0f46 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java @@ -500,6 +500,9 @@ public class ApplicationApiTest extends ControllerContainerTest { .userIdentity(USER_ID), "INFO - All good"); + // Get content/../foo + tester.assertResponse(request("/application/v4/tenant/tenant2/application/application1/instance/default/environment/dev/region/us-east-1/content/%2E%2E%2Ffoo", GET).userIdentity(USER_ID), + "{\"error-code\":\"FORBIDDEN\",\"message\":\"Access denied\"}", 403); // Get content - root tester.assertResponse(request("/application/v4/tenant/tenant2/application/application1/instance/default/environment/dev/region/us-east-1/content/", GET).userIdentity(USER_ID), "{\"path\":\"/\"}"); diff --git a/document/src/vespa/document/serialization/annotationdeserializer.cpp b/document/src/vespa/document/serialization/annotationdeserializer.cpp index e3192459574..9e7a35cfdcb 100644 --- a/document/src/vespa/document/serialization/annotationdeserializer.cpp +++ b/document/src/vespa/document/serialization/annotationdeserializer.cpp @@ -119,9 +119,10 @@ void AnnotationDeserializer::readAnnotation(Annotation & annotation) { if (features & 2) { // has value uint32_t data_type_id = readValue<uint32_t>(_stream); - const DataType *data_type = _repo.getDataType(data_type_id); + const DataType *data_type = type->getDataType(); if (!data_type) { - LOG(warning, "Unknown data type %d", data_type_id); + LOG(warning, "Skipping payload (data type %d) for annotation type %s", + data_type_id, type->getName().c_str()); _stream.adjustReadPos(size - sizeof(uint32_t)); } else { FieldValue::UP value(data_type->createFieldValue()); diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index fb75fcd8c81..88d21f69f7a 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -370,7 +370,7 @@ public class Flags { ZONE_ID, APPLICATION_ID); public static final UnboundBooleanFlag REUSE_NODE_INDEXES = defineFeatureFlag( - "reuse-node-indexes", false, + "reuse-node-indexes", true, List.of("bratseth"), "2022-02-25", "2022-04-25", "Whether we should reuse earlier indexes when allocating new nodes", "Takes effect immediately", diff --git a/flags/src/main/java/com/yahoo/vespa/flags/PermanentFlags.java b/flags/src/main/java/com/yahoo/vespa/flags/PermanentFlags.java index e3960d995e9..bc199f7160e 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/PermanentFlags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/PermanentFlags.java @@ -260,7 +260,7 @@ public class PermanentFlags { "Architecture to use for node resources. Used when implicitly creating admin clusters " + "(logserver, clustercontroller). Valid values: x86_64, arm64", "Takes effect on next redeployment", - ZONE_ID, APPLICATION_ID, CLUSTER_ID, CLUSTER_TYPE); + ZONE_ID, APPLICATION_ID); private PermanentFlags() {} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocationOptimizer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocationOptimizer.java index 30432c1c078..43a78e3a315 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocationOptimizer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocationOptimizer.java @@ -47,7 +47,7 @@ public class AllocationOptimizer { limits = Limits.of(new ClusterResources(minimumNodes, 1, NodeResources.unspecified()), new ClusterResources(maximumNodes, maximumNodes, NodeResources.unspecified())); else - limits = atLeast(minimumNodes, limits).fullySpecified(current.clusterSpec().type(), nodeRepository); + limits = atLeast(minimumNodes, limits).fullySpecified(current.clusterSpec().type(), nodeRepository, clusterModel.application().id()); Optional<AllocatableClusterResources> bestAllocation = Optional.empty(); NodeList hosts = nodeRepository.nodes().list().hosts(); for (int groups = limits.min().groups(); groups <= limits.max().groups(); groups++) { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Limits.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Limits.java index 3c9cbfe2620..ef45592ad18 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Limits.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Limits.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.autoscale; +import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Capacity; import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.ClusterSpec; @@ -59,10 +60,10 @@ public class Limits { return resources; } - public Limits fullySpecified(ClusterSpec.Type type, NodeRepository nodeRepository) { + public Limits fullySpecified(ClusterSpec.Type type, NodeRepository nodeRepository, ApplicationId applicationId) { if (this.isEmpty()) throw new IllegalStateException("Unspecified limits can not be made fully specified"); - var defaultResources = new CapacityPolicies(nodeRepository).defaultNodeResources(type); + var defaultResources = new CapacityPolicies(nodeRepository).defaultNodeResources(type, applicationId); var specifiedMin = min.nodeResources().isUnspecified() ? min.with(defaultResources) : min; var specifiedMax = max.nodeResources().isUnspecified() ? max.with(defaultResources) : max; return new Limits(specifiedMin, specifiedMax); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/PassthroughLoadBalancerService.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/PassthroughLoadBalancerService.java deleted file mode 100644 index 23ddce3ba9e..00000000000 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/PassthroughLoadBalancerService.java +++ /dev/null @@ -1,44 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.provision.lb; - -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.ClusterSpec; -import com.yahoo.config.provision.NodeType; - -import java.util.Comparator; -import java.util.Optional; -import java.util.Set; - -/** - * Implementation of a load balancer service that returns a real as the load balancer instance. This is intended for - * development purposes. - * - * @author mpolden - */ -public class PassthroughLoadBalancerService implements LoadBalancerService { - - @Override - public LoadBalancerInstance create(LoadBalancerSpec spec, boolean force) { - var real = spec.reals().stream() - .min(Comparator.naturalOrder()) - .orElseThrow(() -> new IllegalArgumentException("No reals given")); - return new LoadBalancerInstance(real.hostname(), Optional.empty(), Set.of(real.port()), - Set.of(real.ipAddress() + "/32"), Set.of()); - } - - @Override - public void remove(ApplicationId application, ClusterSpec.Id cluster) { - // Nothing to remove - } - - @Override - public Protocol protocol() { - return Protocol.ipv4; - } - - @Override - public boolean supports(NodeType nodeType, ClusterSpec.Type clusterType) { - return true; - } - -} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/SharedLoadBalancerService.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/SharedLoadBalancerService.java index 7dbebd1fc47..1345dfb8e65 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/SharedLoadBalancerService.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/SharedLoadBalancerService.java @@ -5,16 +5,10 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.NodeType; -import com.yahoo.vespa.hosted.provision.Node; -import com.yahoo.vespa.hosted.provision.NodeList; -import com.yahoo.vespa.hosted.provision.NodeRepository; -import com.yahoo.vespa.hosted.provision.node.IP; -import java.util.Comparator; import java.util.Objects; import java.util.Optional; import java.util.Set; -import java.util.stream.Collectors; /** * This implementation of {@link LoadBalancerService} returns the load balancer(s) that exists by default in the shared @@ -27,28 +21,18 @@ import java.util.stream.Collectors; */ public class SharedLoadBalancerService implements LoadBalancerService { - private static final Comparator<Node> hostnameComparator = Comparator.comparing(Node::hostname); - - private final NodeRepository nodeRepository; private final String vipHostname; - public SharedLoadBalancerService(NodeRepository nodeRepository, String vipHostname) { - this.nodeRepository = Objects.requireNonNull(nodeRepository); + public SharedLoadBalancerService(String vipHostname) { this.vipHostname = Objects.requireNonNull(vipHostname); } @Override public LoadBalancerInstance create(LoadBalancerSpec spec, boolean force) { - NodeList proxyNodes = nodeRepository.nodes().list().nodeType(NodeType.proxy).sortedBy(hostnameComparator); - if (proxyNodes.isEmpty()) throw new IllegalStateException("No proxy nodes found in node-repository"); - Set<String> networks = proxyNodes.stream() - .flatMap(node -> node.ipConfig().primary().stream()) - .map(SharedLoadBalancerService::withPrefixLength) - .collect(Collectors.toSet()); return new LoadBalancerInstance(HostName.from(vipHostname), Optional.empty(), - Set.of(4080, 4443), - networks, + Set.of(4443), + Set.of(), spec.reals()); } @@ -68,11 +52,4 @@ public class SharedLoadBalancerService implements LoadBalancerService { return nodeType == NodeType.tenant && clusterType.isContainer(); } - private static String withPrefixLength(String address) { - if (IP.isV6(address)) { - return address + "/128"; - } - return address + "/32"; - } - } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java index 15decde0d7c..2d36108a1a6 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java @@ -66,7 +66,7 @@ public class NodeRepositoryMaintenance extends AbstractComponent { maintainers.add(new ScalingSuggestionsMaintainer(nodeRepository, defaults.scalingSuggestionsInterval, metric)); maintainers.add(new SwitchRebalancer(nodeRepository, defaults.switchRebalancerInterval, metric, deployer)); - provisionServiceProvider.getLoadBalancerService(nodeRepository) + provisionServiceProvider.getLoadBalancerService() .map(lbService -> new LoadBalancerExpirer(nodeRepository, defaults.loadBalancerExpirerInterval, lbService, metric)) .ifPresent(maintainers::add); provisionServiceProvider.getHostProvisioner() diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java index 45e9efc1a1d..f7d5b966c12 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java @@ -8,12 +8,15 @@ import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.Zone; +import com.yahoo.vespa.flags.FlagSource; import com.yahoo.vespa.flags.PermanentFlags; import com.yahoo.vespa.hosted.provision.NodeRepository; import java.util.function.Function; import static com.yahoo.config.provision.NodeResources.Architecture; +import static com.yahoo.vespa.flags.FetchVector.Dimension.APPLICATION_ID; +import static com.yahoo.vespa.flags.PermanentFlags.ADMIN_CLUSTER_NODE_ARCHITECTURE; /** * Defines the policies for assigning cluster capacity in various environments @@ -25,12 +28,12 @@ public class CapacityPolicies { private final Zone zone; private final Function<ClusterSpec.Type, Boolean> sharedHosts; - private final Architecture architectureForAdminCluster; + private final FlagSource flagSource; public CapacityPolicies(NodeRepository nodeRepository) { this.zone = nodeRepository.zone(); this.sharedHosts = type -> PermanentFlags.SHARED_HOST.bindTo(nodeRepository.flagSource()).value().isEnabled(type.name()); - this.architectureForAdminCluster = Architecture.valueOf(PermanentFlags.ADMIN_CLUSTER_NODE_ARCHITECTURE.bindTo(nodeRepository.flagSource()).value()); + this.flagSource = nodeRepository.flagSource(); } public Capacity applyOn(Capacity capacity, ApplicationId application) { @@ -73,11 +76,16 @@ public class CapacityPolicies { return target; } - public NodeResources defaultNodeResources(ClusterSpec.Type clusterType) { + public NodeResources defaultNodeResources(ClusterSpec.Type clusterType, ApplicationId applicationId) { if (clusterType == ClusterSpec.Type.admin) { + Architecture architecture = Architecture.valueOf( + ADMIN_CLUSTER_NODE_ARCHITECTURE.bindTo(flagSource) + .with(APPLICATION_ID, applicationId.serializedForm()) + .value()); + return zone.getCloud().dynamicProvisioning() && ! sharedHosts.apply(clusterType) ? - new NodeResources(0.5, 4, 50, 0.3).with(architectureForAdminCluster) : - new NodeResources(0.5, 2, 50, 0.3).with(architectureForAdminCluster); + new NodeResources(0.5, 4, 50, 0.3).with(architecture) : + new NodeResources(0.5, 2, 50, 0.3).with(architecture); } return zone.getCloud().dynamicProvisioning() ? diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/EmptyProvisionServiceProvider.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/EmptyProvisionServiceProvider.java index 9a7fa672cfc..e7332f6474d 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/EmptyProvisionServiceProvider.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/EmptyProvisionServiceProvider.java @@ -18,7 +18,7 @@ public class EmptyProvisionServiceProvider implements ProvisionServiceProvider { private final HostResourcesCalculator hostResourcesCalculator = new IdentityHostResourcesCalculator(); @Override - public Optional<LoadBalancerService> getLoadBalancerService(NodeRepository nodeRepository) { + public Optional<LoadBalancerService> getLoadBalancerService() { return Optional.empty(); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java index 97fe14caef6..e24d71f22f2 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java @@ -70,7 +70,7 @@ public class NodeRepositoryProvisioner implements Provisioner { this.allocationOptimizer = new AllocationOptimizer(nodeRepository); this.capacityPolicies = new CapacityPolicies(nodeRepository); this.zone = zone; - this.loadBalancerProvisioner = provisionServiceProvider.getLoadBalancerService(nodeRepository) + this.loadBalancerProvisioner = provisionServiceProvider.getLoadBalancerService() .map(lbService -> new LoadBalancerProvisioner(nodeRepository, lbService)); this.nodeResourceLimits = new NodeResourceLimits(nodeRepository); this.preparer = new Preparer(nodeRepository, @@ -106,14 +106,12 @@ public class NodeRepositoryProvisioner implements Provisioner { logIfDownscaled(requested.minResources().nodes(), actual.minResources().nodes(), cluster, logger); groups = target.groups(); - resources = target.nodeResources().isUnspecified() ? capacityPolicies.defaultNodeResources(cluster.type()) - : target.nodeResources(); + resources = getNodeResources(cluster, target.nodeResources(), application); nodeSpec = NodeSpec.from(target.nodes(), resources, exclusive, actual.canFail()); } else { groups = 1; // type request with multiple groups is not supported - resources = requested.minResources().nodeResources().isUnspecified() ? capacityPolicies.defaultNodeResources(cluster.type()) - : requested.minResources().nodeResources(); + resources = getNodeResources(cluster, requested.minResources().nodeResources(), application); nodeSpec = NodeSpec.from(requested.type()); } var reuseIndexes = Flags.REUSE_NODE_INDEXES.bindTo(nodeRepository.flagSource()) @@ -122,6 +120,12 @@ public class NodeRepositoryProvisioner implements Provisioner { return asSortedHosts(preparer.prepare(application, cluster, nodeSpec, groups, reuseIndexes), resources); } + private NodeResources getNodeResources(ClusterSpec cluster, NodeResources nodeResources, ApplicationId applicationId) { + return nodeResources.isUnspecified() + ? capacityPolicies.defaultNodeResources(cluster.type(), applicationId) + : nodeResources; + } + @Override public void activate(Collection<HostSpec> hosts, ActivationContext context, ApplicationTransaction transaction) { validate(hosts); @@ -170,16 +174,16 @@ public class NodeRepositoryProvisioner implements Provisioner { boolean firstDeployment = nodes.isEmpty(); AllocatableClusterResources currentResources = firstDeployment // start at min, preserve current resources otherwise - ? new AllocatableClusterResources(initialResourcesFrom(requested, clusterSpec), clusterSpec, nodeRepository) + ? new AllocatableClusterResources(initialResourcesFrom(requested, clusterSpec, application.id()), clusterSpec, nodeRepository) : new AllocatableClusterResources(nodes.asList(), nodeRepository); var clusterModel = new ClusterModel(application, cluster, clusterSpec, nodes, nodeRepository.metricsDb(), nodeRepository.clock()); return within(Limits.of(requested), currentResources, firstDeployment, clusterModel); } - private ClusterResources initialResourcesFrom(Capacity requested, ClusterSpec clusterSpec) { + private ClusterResources initialResourcesFrom(Capacity requested, ClusterSpec clusterSpec, ApplicationId applicationId) { var initial = requested.minResources(); if (initial.nodeResources().isUnspecified()) - initial = initial.with(capacityPolicies.defaultNodeResources(clusterSpec.type())); + initial = initial.with(capacityPolicies.defaultNodeResources(clusterSpec.type(), applicationId)); return initial; } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisionServiceProvider.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisionServiceProvider.java index c5835ec525a..6e301b7724c 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisionServiceProvider.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisionServiceProvider.java @@ -1,7 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.provisioning; -import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.lb.LoadBalancerService; import java.util.Optional; @@ -13,7 +12,7 @@ import java.util.Optional; */ public interface ProvisionServiceProvider { - Optional<LoadBalancerService> getLoadBalancerService(NodeRepository nodeRepository); + Optional<LoadBalancerService> getLoadBalancerService(); Optional<HostProvisioner> getHostProvisioner(); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockProvisionServiceProvider.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockProvisionServiceProvider.java index 9f6143e5daa..d72c5959082 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockProvisionServiceProvider.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockProvisionServiceProvider.java @@ -2,7 +2,6 @@ package com.yahoo.vespa.hosted.provision.testutils; import com.google.inject.Inject; -import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.lb.LoadBalancerService; import com.yahoo.vespa.hosted.provision.lb.LoadBalancerServiceMock; import com.yahoo.vespa.hosted.provision.provisioning.EmptyProvisionServiceProvider; @@ -38,7 +37,7 @@ public class MockProvisionServiceProvider implements ProvisionServiceProvider { } @Override - public Optional<LoadBalancerService> getLoadBalancerService(NodeRepository nodeRepository) { + public Optional<LoadBalancerService> getLoadBalancerService() { return loadBalancerService; } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java index a04a3828f13..40e38d752ff 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java @@ -15,7 +15,6 @@ import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.Zone; -import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.Nodelike; import com.yahoo.vespa.hosted.provision.provisioning.CapacityPolicies; @@ -240,7 +239,7 @@ public class AutoscalingTest { ClusterSpec cluster1 = tester.clusterSpec(ClusterSpec.Type.container, "cluster1"); NodeResources defaultResources = - new CapacityPolicies(tester.nodeRepository()).defaultNodeResources(cluster1.type()); + new CapacityPolicies(tester.nodeRepository()).defaultNodeResources(cluster1.type(), application1); // deploy tester.deploy(application1, cluster1, Capacity.from(min, max)); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/lb/PassthroughLoadBalancerServiceTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/lb/PassthroughLoadBalancerServiceTest.java deleted file mode 100644 index 5fcc2afb85d..00000000000 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/lb/PassthroughLoadBalancerServiceTest.java +++ /dev/null @@ -1,30 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.provision.lb; - -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.ClusterSpec; -import com.yahoo.config.provision.HostName; -import org.junit.Test; - -import java.util.Set; - -import static org.junit.Assert.assertEquals; - -/** - * @author mpolden - */ -public class PassthroughLoadBalancerServiceTest { - - @Test - public void create() { - var lbService = new PassthroughLoadBalancerService(); - var real = new Real(HostName.from("host1.example.com"), "192.0.2.10"); - var reals = Set.of(real, new Real(HostName.from("host2.example.com"), "192.0.2.11")); - var instance = lbService.create(new LoadBalancerSpec(ApplicationId.from("tenant1", "app1", "default"), - ClusterSpec.Id.from("c1"), reals), false); - assertEquals(real.hostname(), instance.hostname()); - assertEquals(Set.of(real.port()), instance.ports()); - assertEquals(Set.of(real.ipAddress() + "/32"), instance.networks()); - } - -} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/lb/SharedLoadBalancerServiceTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/lb/SharedLoadBalancerServiceTest.java index 28b40bb7642..4975e670e86 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/lb/SharedLoadBalancerServiceTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/lb/SharedLoadBalancerServiceTest.java @@ -4,8 +4,6 @@ package com.yahoo.vespa.hosted.provision.lb; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.HostName; -import com.yahoo.config.provision.NodeType; -import com.yahoo.vespa.hosted.provision.provisioning.ProvisioningTester; import org.junit.Test; import java.util.Optional; @@ -18,8 +16,7 @@ import static org.junit.Assert.assertEquals; */ public class SharedLoadBalancerServiceTest { - private final ProvisioningTester tester = new ProvisioningTester.Builder().build(); - private final SharedLoadBalancerService loadBalancerService = new SharedLoadBalancerService(tester.nodeRepository(), "vip.example.com"); + private final SharedLoadBalancerService loadBalancerService = new SharedLoadBalancerService("vip.example.com"); private final ApplicationId applicationId = ApplicationId.from("tenant1", "application1", "default"); private final ClusterSpec.Id clusterId = ClusterSpec.Id.from("qrs1"); private final Set<Real> reals = Set.of( @@ -29,18 +26,12 @@ public class SharedLoadBalancerServiceTest { @Test public void test_create_lb() { - tester.makeReadyNodes(2, "default", NodeType.proxy); var lb = loadBalancerService.create(new LoadBalancerSpec(applicationId, clusterId, reals), false); assertEquals(HostName.from("vip.example.com"), lb.hostname()); assertEquals(Optional.empty(), lb.dnsZone()); - assertEquals(Set.of("127.0.0.1/32", "127.0.0.2/32", "::1/128", "::2/128"), lb.networks()); - assertEquals(Set.of(4080, 4443), lb.ports()); - } - - @Test(expected = IllegalStateException.class) - public void test_exception_on_missing_proxies() { - loadBalancerService.create(new LoadBalancerSpec(applicationId, clusterId, reals), false); + assertEquals(Set.of(), lb.networks()); + assertEquals(Set.of(4443), lb.ports()); } @Test diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/DoubleValue.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/DoubleValue.java index 6154b5ea76c..d8443cfd1ef 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/DoubleValue.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/DoubleValue.java @@ -129,8 +129,8 @@ public final class DoubleValue extends DoubleCompatibleValue { /** Returns this or a mutable copy assigned the given value */ private DoubleValue mutable(double value) { - DoubleValue mutable=this.asMutable(); - mutable.value=value; + DoubleValue mutable = this.asMutable(); + mutable.value = value; return mutable; } @@ -147,7 +147,7 @@ public final class DoubleValue extends DoubleCompatibleValue { @Override public boolean equals(Object other) { - if (this==other) return true; + if (this == other) return true; if ( ! (other instanceof Value)) return false; if ( ! ((Value) other).hasDouble()) return false; return this.asDouble() == ((Value) other).asDouble(); diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/TruthOperator.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/TruthOperator.java index 6f264903122..fc259867923 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/TruthOperator.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/TruthOperator.java @@ -21,7 +21,7 @@ public enum TruthOperator implements Serializable { private final String operatorString; TruthOperator(String operatorString) { - this.operatorString=operatorString; + this.operatorString = operatorString; } /** Perform the truth operation on the input */ diff --git a/searchlib/src/vespa/searchlib/parsequery/parse.h b/searchlib/src/vespa/searchlib/parsequery/parse.h index c9b7940b887..34ea692c370 100644 --- a/searchlib/src/vespa/searchlib/parsequery/parse.h +++ b/searchlib/src/vespa/searchlib/parsequery/parse.h @@ -33,7 +33,7 @@ public: ITEM_TERM = 4, ITEM_NUMTERM = 5, ITEM_PHRASE = 6, - /* removed: ITEM_PAREN = 7, */ + ITEM_MULTI_TERM = 7, ITEM_PREFIXTERM = 8, ITEM_SUBSTRINGTERM = 9, ITEM_ANY = 10, @@ -56,7 +56,7 @@ public: ITEM_GEO_LOCATION_TERM = 27, ITEM_TRUE = 28, ITEM_FALSE = 29, - ITEM_MULTI_TERM = 30, + ITEM_MAX = 30, // Indicates how long tables must be. ITEM_UNDEF = 31, }; diff --git a/searchlib/src/vespa/searchlib/transactionlog/domain.cpp b/searchlib/src/vespa/searchlib/transactionlog/domain.cpp index 928497bf09d..0af1ffa7b33 100644 --- a/searchlib/src/vespa/searchlib/transactionlog/domain.cpp +++ b/searchlib/src/vespa/searchlib/transactionlog/domain.cpp @@ -214,9 +214,9 @@ Domain::triggerSyncNow(std::unique_ptr<vespalib::IDestructorCallback> after_sync std::unique_lock guard(_currentChunkMutex); commitAndTransferResponses(guard); } - _singleCommitter->execute(makeLambdaTask([after_sync=std::move(after_sync), domainPart=getActivePart()]() { + _singleCommitter->execute(makeLambdaTask([this, after_sync=std::move(after_sync)]() { (void) after_sync; - domainPart->sync(); + getActivePart()->sync(); })); } diff --git a/searchsummary/src/tests/extractkeywords/simplequerystackitem.cpp b/searchsummary/src/tests/extractkeywords/simplequerystackitem.cpp index b31173c74c2..f3706bfc7c1 100644 --- a/searchsummary/src/tests/extractkeywords/simplequerystackitem.cpp +++ b/searchsummary/src/tests/extractkeywords/simplequerystackitem.cpp @@ -186,8 +186,9 @@ SimpleQueryStackItem::AppendBuffer(RawBuf *buf) const buf->appendCompressedPositiveNumber(_arg2); // allow_approximate buf->appendCompressedPositiveNumber(_arg3); // explore_additional_hits break; - case ITEM_PREDICATE_QUERY: // not handled at all here case ITEM_MULTI_TERM: // TODO: handle + case ITEM_PREDICATE_QUERY: // not handled at all here + case ITEM_MAX: case ITEM_UNDEF: abort(); break; diff --git a/vespamalloc/src/tests/test1/testatomic.cpp b/vespamalloc/src/tests/test1/testatomic.cpp index 5ce52330f77..948616065e6 100644 --- a/vespamalloc/src/tests/test1/testatomic.cpp +++ b/vespamalloc/src/tests/test1/testatomic.cpp @@ -2,6 +2,7 @@ #include <vespa/vespalib/testkit/testapp.h> #include <vespamalloc/malloc/allocchunk.h> #include <vespamalloc/malloc/mmappool.h> +#include <unistd.h> TEST("verify lock freeness of atomics"){ { @@ -23,29 +24,32 @@ TEST("verify lock freeness of atomics"){ TEST("test explicit mmap/munmap") { vespamalloc::MMapPool mmapPool; + size_t page_size = getpagesize(); + size_t mmap1_size = 3 * page_size; + size_t mmap2_size = 7 * page_size; EXPECT_EQUAL(0u, mmapPool.getNumMappings()); EXPECT_EQUAL(0u, mmapPool.getMmappedBytes()); - void * mmap1 = mmapPool.mmap(0xe000); + void * mmap1 = mmapPool.mmap(mmap1_size); EXPECT_EQUAL(1u, mmapPool.getNumMappings()); - EXPECT_EQUAL(0xe000u, mmapPool.getMmappedBytes()); - EXPECT_EQUAL(0xe000u, mmapPool.get_size(mmap1)); + EXPECT_EQUAL(mmap1_size, mmapPool.getMmappedBytes()); + EXPECT_EQUAL(mmap1_size, mmapPool.get_size(mmap1)); mmapPool.unmap(mmap1); EXPECT_EQUAL(0u, mmapPool.getNumMappings()); EXPECT_EQUAL(0u, mmapPool.getMmappedBytes()); - mmap1 = mmapPool.mmap(0xe000); + mmap1 = mmapPool.mmap(mmap1_size); EXPECT_EQUAL(1u, mmapPool.getNumMappings()); - EXPECT_EQUAL(0xe000u, mmapPool.getMmappedBytes()); - EXPECT_EQUAL(0xe000u, mmapPool.get_size(mmap1)); + EXPECT_EQUAL(mmap1_size, mmapPool.getMmappedBytes()); + EXPECT_EQUAL(mmap1_size, mmapPool.get_size(mmap1)); - void * mmap2 = mmapPool.mmap(0x1e000); + void * mmap2 = mmapPool.mmap(mmap2_size); EXPECT_EQUAL(2u, mmapPool.getNumMappings()); - EXPECT_EQUAL(0x2c000u, mmapPool.getMmappedBytes()); - EXPECT_EQUAL(0xe000u, mmapPool.get_size(mmap1)); - EXPECT_EQUAL(0x1e000u, mmapPool.get_size(mmap2)); + EXPECT_EQUAL(mmap1_size + mmap2_size, mmapPool.getMmappedBytes()); + EXPECT_EQUAL(mmap1_size, mmapPool.get_size(mmap1)); + EXPECT_EQUAL(mmap2_size, mmapPool.get_size(mmap2)); mmapPool.unmap(mmap1); EXPECT_EQUAL(1u, mmapPool.getNumMappings()); - EXPECT_EQUAL(0x1e000u, mmapPool.getMmappedBytes()); + EXPECT_EQUAL(mmap2_size, mmapPool.getMmappedBytes()); mmapPool.unmap(mmap2); EXPECT_EQUAL(0u, mmapPool.getNumMappings()); EXPECT_EQUAL(0u, mmapPool.getMmappedBytes()); |