diff options
Diffstat (limited to 'config-model/src/main/java')
21 files changed, 206 insertions, 103 deletions
diff --git a/config-model/src/main/java/com/yahoo/config/model/CommonConfigsProducer.java b/config-model/src/main/java/com/yahoo/config/model/CommonConfigsProducer.java index fad097d1edb..094b11dcbc7 100644 --- a/config-model/src/main/java/com/yahoo/config/model/CommonConfigsProducer.java +++ b/config-model/src/main/java/com/yahoo/config/model/CommonConfigsProducer.java @@ -21,7 +21,6 @@ import com.yahoo.vespa.config.content.AllClustersBucketSpacesConfig; * This interface describes the configs that are produced by the model producer root. * * @author Ulf Lilleengen - * @since 5.1 */ public interface CommonConfigsProducer extends DocumentmanagerConfig.Producer, DocumenttypesConfig.Producer, diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java index 5183a3ca587..49c968a1d91 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java @@ -72,6 +72,7 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea private double diskBloatFactor = 0.2; private boolean distributorEnhancedMaintenanceScheduling = false; private boolean asyncApplyBucketDiff = false; + private List<String> zoneDnsSuffixes = List.of(); @Override public ModelContext.FeatureFlags featureFlags() { return this; } @Override public boolean multitenant() { return multitenant; } @@ -124,6 +125,7 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea @Override public boolean distributorEnhancedMaintenanceScheduling() { return distributorEnhancedMaintenanceScheduling; } @Override public int maxUnCommittedMemory() { return maxUnCommittedMemory; } @Override public boolean asyncApplyBucketDiff() { return asyncApplyBucketDiff; } + @Override public List<String> zoneDnsSuffixes() { return zoneDnsSuffixes; } public TestProperties maxUnCommittedMemory(int maxUnCommittedMemory) { this.maxUnCommittedMemory = maxUnCommittedMemory; @@ -320,6 +322,11 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea return this; } + public TestProperties setZoneDnsSuffixes(List<String> zoneDnsSuffixes) { + this.zoneDnsSuffixes = List.copyOf(zoneDnsSuffixes); + return this; + } + public static class Spec implements ConfigServerSpec { private final String hostName; diff --git a/config-model/src/main/java/com/yahoo/documentmodel/NewDocumentType.java b/config-model/src/main/java/com/yahoo/documentmodel/NewDocumentType.java index 08a0f8b9882..d98869e9dd3 100644 --- a/config-model/src/main/java/com/yahoo/documentmodel/NewDocumentType.java +++ b/config-model/src/main/java/com/yahoo/documentmodel/NewDocumentType.java @@ -368,9 +368,9 @@ public final class NewDocumentType extends StructuredDataType implements DataTyp @Override public String toString() { return name; } - public final String getName() { return name; } + public String getName() { return name; } - public final int getId() { return id; } + public int getId() { return id; } @Override public int hashCode() { return name.hashCode(); } diff --git a/config-model/src/main/java/com/yahoo/documentmodel/VespaDocumentType.java b/config-model/src/main/java/com/yahoo/documentmodel/VespaDocumentType.java index 4899029c4b0..b29e4704f62 100644 --- a/config-model/src/main/java/com/yahoo/documentmodel/VespaDocumentType.java +++ b/config-model/src/main/java/com/yahoo/documentmodel/VespaDocumentType.java @@ -16,7 +16,7 @@ public class VespaDocumentType { public static NewDocumentType INSTANCE = newInstance(); - public static DataTypeName NAME = new DataTypeName("document"); + public static final DataTypeName NAME = new DataTypeName("document"); private static NewDocumentType newInstance() { NewDocumentType vespa = new NewDocumentType(new NewDocumentType.Name(8, "document")); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java index b5d86f5bb38..52fd5286bcf 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java @@ -98,6 +98,8 @@ public class RankProfile implements Cloneable { private String inheritedSummaryFeatures; private Set<ReferenceNode> matchFeatures; + private String inheritedMatchFeatures; + private Set<ReferenceNode> rankFeatures; /** The properties of this - a multimap */ @@ -519,8 +521,30 @@ public class RankProfile implements Cloneable { this.inheritedSummaryFeatures = parentProfile; } + /** + * Sets the name of a profile this should inherit the match features of. + * Without setting this, this will either have the match features of the parent, + * or if match features are set in this, only have the match features in this. + * With this set the resulting match features of this will be the superset of those defined in this and + * the final (with inheritance included) match features of the given parent. + * The profile must be the profile which is directly inherited by this. + * + */ + public void setInheritedMatchFeatures(String parentProfile) { + if ( ! parentProfile.equals(inheritedName)) + throw new IllegalArgumentException("This rank profile ("+name+") can only inherit the match features of its parent, '" + + inheritedName + ", but attemtping to inherit '" + parentProfile); + this.inheritedMatchFeatures = parentProfile; + } + /** Returns a read-only view of the match features to use in this profile. This is never null */ public Set<ReferenceNode> getMatchFeatures() { + if (inheritedMatchFeatures != null && matchFeatures != null) { + Set<ReferenceNode> combined = new HashSet<>(); + combined.addAll(getInherited().getMatchFeatures()); + combined.addAll(matchFeatures); + return Collections.unmodifiableSet(combined); + } if (matchFeatures != null) return Collections.unmodifiableSet(matchFeatures); if (getInherited() != null) return getInherited().getMatchFeatures(); return Set.of(); @@ -532,7 +556,7 @@ public class RankProfile implements Cloneable { matchFeatures.add(feature); } - /** Adds the content of the given feature list to the internal list of summary features. */ + /** Adds the content of the given feature list to the internal list of match features. */ public void addMatchFeatures(FeatureList features) { for (ReferenceNode feature : features) { addMatchFeature(feature); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/SDDocumentTypeOrderer.java b/config-model/src/main/java/com/yahoo/searchdefinition/SDDocumentTypeOrderer.java index 8a6f16586d2..aa43c00f461 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/SDDocumentTypeOrderer.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/SDDocumentTypeOrderer.java @@ -4,6 +4,7 @@ package com.yahoo.searchdefinition; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.document.*; import com.yahoo.document.annotation.AnnotationReferenceDataType; +import com.yahoo.documentmodel.NewDocumentType; import com.yahoo.searchdefinition.document.SDDocumentType; import com.yahoo.searchdefinition.document.TemporarySDDocumentType; @@ -15,10 +16,10 @@ import java.util.logging.Level; */ public class SDDocumentTypeOrderer { - private Map<DataTypeName, SDDocumentType> createdSDTypes = new LinkedHashMap<>(); - private Set<Integer> seenTypes = new LinkedHashSet<>(); + private final Map<DataTypeName, SDDocumentType> createdSDTypes = new LinkedHashMap<>(); + private final Set<Integer> seenTypes = new LinkedHashSet<>(); List<SDDocumentType> processingOrder = new LinkedList<>(); - private DeployLogger deployLogger; + private final DeployLogger deployLogger; public SDDocumentTypeOrderer(List<SDDocumentType> sdTypes, DeployLogger deployLogger) { this.deployLogger = deployLogger; @@ -36,28 +37,12 @@ public class SDDocumentTypeOrderer { public void process() { for (SDDocumentType type : createdSDTypes.values()) { - process(type); + process(type, type); } } - private void process(SDDocumentType type) { - List<DataTypeName> toReplace = new ArrayList<>(); - for (SDDocumentType sdoc : type.getInheritedTypes()) { - if (sdoc instanceof TemporarySDDocumentType) { - toReplace.add(sdoc.getDocumentName()); - } - } - for (DataTypeName name : toReplace) { - SDDocumentType inherited = createdSDTypes.get(name); - if (inherited == null) { - throw new IllegalStateException("Document type '" + name + "' not found."); - } - process(inherited); - type.inherit(inherited); - } - visit(type); - } - private void visit(SDDocumentType docOrStruct) { + private void process(SDDocumentType docOrStruct, SDDocumentType owningDocument) { + resolveAndProcessInheritedTemporaryTypes(docOrStruct, owningDocument); int id; if (docOrStruct.isStruct()) { id = new StructDataType(docOrStruct.getName()).getId(); @@ -71,16 +56,38 @@ public class SDDocumentTypeOrderer { seenTypes.add((new StructDataType(docOrStruct.getName()).getId())); } - for (Field field : docOrStruct.fieldSet()) { if (!seenTypes.contains(field.getDataType().getId())) { //we haven't seen this before, do it - visit(field.getDataType()); + visit(field.getDataType(), owningDocument); } } processingOrder.add(docOrStruct); } + private void resolveAndProcessInheritedTemporaryTypes(SDDocumentType type, SDDocumentType owningDocument) { + List<DataTypeName> toReplace = new ArrayList<>(); + for (SDDocumentType sdoc : type.getInheritedTypes()) { + if (sdoc instanceof TemporarySDDocumentType) { + toReplace.add(sdoc.getDocumentName()); + } + } + for (DataTypeName name : toReplace) { + SDDocumentType inherited; + if (type.isStruct()) { + inherited = owningDocument.allTypes().get(new NewDocumentType.Name(name.getName())); + if (inherited == null) throw new IllegalStateException("Struct '" + name + "' not found in " + owningDocument); + process(inherited, owningDocument); + } + else { + inherited = createdSDTypes.get(name); + if (inherited == null) throw new IllegalStateException("Document type '" + name + "' not found"); + process(inherited, inherited); + } + type.inherit(inherited); + } + } + private SDDocumentType find(String name) { SDDocumentType sdDocType = createdSDTypes.get(new DataTypeName(name)); if (sdDocType != null) { @@ -95,27 +102,28 @@ public class SDDocumentTypeOrderer { } return null; } - private void visit(DataType type) { + + private void visit(DataType type, SDDocumentType owningDocument) { if (type instanceof StructuredDataType) { StructuredDataType structType = (StructuredDataType) type; SDDocumentType sdDocType = find(structType.getName()); if (sdDocType == null) { - throw new IllegalArgumentException("Could not find struct '" + type.getName() + "'."); + throw new IllegalArgumentException("Could not find struct '" + type.getName() + "'"); } - visit(sdDocType); + process(sdDocType, owningDocument); return; } if (type instanceof MapDataType) { MapDataType mType = (MapDataType) type; - visit(mType.getValueType()); - visit(mType.getKeyType()); + visit(mType.getValueType(), owningDocument); + visit(mType.getKeyType(), owningDocument); } else if (type instanceof WeightedSetDataType) { WeightedSetDataType wType = (WeightedSetDataType) type; - visit(wType.getNestedType()); + visit(wType.getNestedType(), owningDocument); } else if (type instanceof CollectionDataType) { CollectionDataType cType = (CollectionDataType) type; - visit(cType.getNestedType()); + visit(cType.getNestedType(), owningDocument); } else if (type instanceof AnnotationReferenceDataType) { //do nothing } else if (type instanceof PrimitiveDataType) { @@ -128,4 +136,5 @@ public class SDDocumentTypeOrderer { deployLogger.logApplicationPackage(Level.WARNING, "Unknown type : " + type); } } + } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/Schema.java b/config-model/src/main/java/com/yahoo/searchdefinition/Schema.java index 512d908ce5b..c7a7ecd1d08 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/Schema.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/Schema.java @@ -514,21 +514,14 @@ public class Schema implements ImmutableSchema { return Collections.unmodifiableList(allIndices); } - /** - * Adds an explicitly defined summary to this search definition - * - * @param summary The summary to add. - */ + /** Adds an explicitly defined summary to this search definition */ public void addSummary(DocumentSummary summary) { summaries.put(summary.getName(), summary); } /** - * <p>Returns a summary class defined by this search definition, or null if no summary with this name is defined. - * The default summary, named "default" is always present.</p> - * - * @param name the name of the summary to get. - * @return Summary found. + * Returns a summary class defined by this search definition, or null if no summary with this name is defined. + * The default summary, named "default" is always present. */ public DocumentSummary getSummary(String name) { var summary = summaries.get(name); @@ -540,9 +533,6 @@ public class Schema implements ImmutableSchema { /** * Returns the first explicit instance found of a summary field with this name, or null if not present (implicitly * or explicitly) in any summary class. - * - * @param name The name of the summaryfield to get. - * @return SummaryField to return. */ public SummaryField getSummaryField(String name) { for (DocumentSummary summary : summaries.values()) { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/SummaryClass.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/SummaryClass.java index 68966d39d7d..6c233aacf30 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/SummaryClass.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/SummaryClass.java @@ -31,9 +31,9 @@ public class SummaryClass extends Derived { private final boolean omitSummaryFeatures; /** The summary fields of this indexed by name */ - private Map<String,SummaryClassField> fields = new java.util.LinkedHashMap<>(); + private final Map<String,SummaryClassField> fields = new java.util.LinkedHashMap<>(); - private DeployLogger deployLogger; + private final DeployLogger deployLogger; private final Random random = new Random(7); @@ -78,9 +78,9 @@ public class SummaryClass extends Derived { private void addField(String name, DataType type, SummaryTransform transform) { if (fields.containsKey(name)) { SummaryClassField sf = fields.get(name); - if (!SummaryClassField.convertDataType(type, transform, rawAsBase64).equals(sf.getType())) { - deployLogger.logApplicationPackage(Level.WARNING, "Conflicting definition of field " + name + ". " + - "Declared as type " + sf.getType() + " and " + type); + if ( SummaryClassField.convertDataType(type, transform, rawAsBase64) != sf.getType()) { + deployLogger.logApplicationPackage(Level.WARNING, "Conflicting definition of field " + name + + ". " + "Declared as type " + sf.getType() + " and " + type); } } else { fields.put(name, new SummaryClassField(name, type, transform, rawAsBase64)); @@ -106,6 +106,7 @@ public class SummaryClass extends Derived { public int getFieldCount() { return fields.size(); } + @Override public int hashCode() { int number = 1; int hash = getName().hashCode(); @@ -143,6 +144,7 @@ public class SummaryClass extends Derived { @Override protected String getDerivedName() { return "summary"; } + @Override public String toString() { return "summary class " + getName(); } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/SDDocumentType.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/SDDocumentType.java index 6424db1c2dd..60b06d3655c 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/document/SDDocumentType.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/SDDocumentType.java @@ -1,7 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.searchdefinition.document; -import com.yahoo.document.CompressionConfig; import com.yahoo.document.DataType; import com.yahoo.document.DataTypeName; import com.yahoo.document.DocumentType; @@ -127,6 +126,14 @@ public class SDDocumentType implements Cloneable, Serializable { return list; } + public Map<NewDocumentType.Name, SDDocumentType> allTypes() { + Map<NewDocumentType.Name, SDDocumentType> map = new LinkedHashMap<>(); + for (SDDocumentType inherited : inheritedTypes.values()) + map.putAll(inherited.allTypes()); + map.putAll(ownedTypes); + return map; + } + /** * Creates a new document type. * The document type id will be generated as a hash from the document type name. @@ -145,13 +152,11 @@ public class SDDocumentType implements Cloneable, Serializable { * Creates a new document type. * The document type id will be generated as a hash from the document type name. * - * @param name The name of the new document type + * @param name the name of the new document type * @param schema check for type ID collisions in this search definition */ - @SuppressWarnings("deprecation") public SDDocumentType(String name, Schema schema) { docType = new DocumentType(name); - docType.contentStruct().setCompressionConfig(new CompressionConfig()); validateId(schema); inherit(VESPA_DOCUMENT); } @@ -161,7 +166,7 @@ public class SDDocumentType implements Cloneable, Serializable { public SDDocumentType setStruct(DataType structType) { if (structType != null) { this.structType = structType; - inheritedTypes.clear(); + inheritedTypes.remove(VESPA_DOCUMENT.getDocumentName()); } else { if (docType.contentStruct() != null) { this.structType = docType.contentStruct(); @@ -200,7 +205,7 @@ public class SDDocumentType implements Cloneable, Serializable { if (schema.getDocument(getName()) == null) return; SDDocumentType doc = schema.getDocument(); throw new IllegalArgumentException("Failed creating document type '" + getName() + "', " + - "document type '" + doc.getName() + "' already uses ID '" + doc.getName() + "'"); + "document type '" + doc.getName() + "' already uses ID '" + doc.getName() + "'"); } public void setFieldId(SDField field, int id) { @@ -293,6 +298,8 @@ public class SDDocumentType implements Cloneable, Serializable { return fieldSet().iterator(); } + /** Returns the number of fields in this only, not including inherited fields */ + // TODO: Remove public int getFieldCount() { return docType.getFieldCount(); } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImplicitSummaries.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImplicitSummaries.java index 6b09234c469..9eb8b921e81 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImplicitSummaries.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImplicitSummaries.java @@ -29,17 +29,16 @@ public class ImplicitSummaries extends Processor { @Override public void process(boolean validate, boolean documentsOnly) { - DocumentSummary defaultSummary = schema.getSummary("default"); + DocumentSummary defaultSummary = schema.getSummariesInThis().get("default"); if (defaultSummary == null) { defaultSummary = new DocumentSummary("default", schema); - defaultSummary.setFromDisk(true); + defaultSummary.setFromDisk(true); // As we add documentid to this schema.addSummary(defaultSummary); } for (SDField field : schema.allConcreteFields()) { collectSummaries(field, schema, validate); } - for (DocumentSummary documentSummary : schema.getSummaries().values()) { documentSummary.purgeImplicits(); } @@ -50,7 +49,7 @@ public class ImplicitSummaries extends Processor { } private void collectSummaries(SDField field , Schema schema, boolean validate) { - SummaryField addedSummaryField=null; + SummaryField addedSummaryField = null; // Implicit String fieldName = field.getName(); @@ -65,7 +64,7 @@ public class ImplicitSummaries extends Processor { } if (fieldSummaryField != null) { for (String dest : fieldSummaryField.getDestinations()) { - DocumentSummary summary = schema.getSummary(dest); + DocumentSummary summary = schema.getSummariesInThis().get(dest); if (summary != null) { summary.add(fieldSummaryField); } @@ -114,7 +113,7 @@ public class ImplicitSummaries extends Processor { } private DocumentSummary getOrCreateAttributePrefetchSummary(Schema schema) { - DocumentSummary summary = schema.getSummary("attributeprefetch"); + DocumentSummary summary = schema.getSummariesInThis().get("attributeprefetch"); if (summary == null) { summary = new DocumentSummary("attributeprefetch", schema); schema.addSummary(summary); @@ -165,13 +164,13 @@ public class ImplicitSummaries extends Processor { throw newProcessException(schema, summaryField, "Source field '" + fieldName + "' does not exist."); } if (! sourceField.doesSummarying() && - ! summaryField.getTransform().equals(SummaryTransform.ATTRIBUTE) && - ! summaryField.getTransform().equals(SummaryTransform.GEOPOS)) + summaryField.getTransform() != SummaryTransform.ATTRIBUTE && + summaryField.getTransform() != SummaryTransform.GEOPOS) { // Summary transform attribute may indicate that the ilscript was rewritten to remove summary // by another search that uses this same field in inheritance. deployLogger.logApplicationPackage(Level.WARNING, "Ignoring " + summaryField + ": " + sourceField + - " is not creating a summary value in its indexing statement"); + " is not creating a summary value in its indexing statement"); return false; } @@ -210,7 +209,7 @@ public class ImplicitSummaries extends Processor { } private void addToDestination(String destinationName, SummaryField summaryField, Schema schema) { - DocumentSummary destination = schema.getSummary(destinationName); + DocumentSummary destination = schema.getSummariesInThis().get(destinationName); if (destination == null) { destination = new DocumentSummary(destinationName, schema); schema.addSummary(destination); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/MakeDefaultSummaryTheSuperSet.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/MakeDefaultSummaryTheSuperSet.java index a82c8d0c6be..49a56bafe2a 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/MakeDefaultSummaryTheSuperSet.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/MakeDefaultSummaryTheSuperSet.java @@ -10,20 +10,20 @@ import com.yahoo.vespa.documentmodel.SummaryTransform; import com.yahoo.vespa.model.container.search.QueryProfiles; /** - * <p>All summary fields which are not attributes + * All summary fields which are not attributes * must currently be present in the default summary class, * since the default summary class also defines the docsum.dat format. * This processor adds any missing summaries to the default summary. * When that is decoupled from the actual summaries returned, this * processor can be removed. Note: the StreamingSummary also takes advantage of - * the fact that default is the superset.</p> + * the fact that default is the superset. * - * <p>All other summary logic should work unchanged without this processing step + * All other summary logic should work unchanged without this processing step * except that IndexStructureValidator.validateSummaryFields must be changed to * consider all summaries, not just the default, i.e change to - * if (search.getSummaryField(expr.getFieldName()) == null)</p> + * if (search.getSummaryField(expr.getFieldName()) == null) * - * <p>This must be done after other summary processors.</p> + * This must be done after other summary processors. * * @author bratseth */ @@ -35,7 +35,7 @@ public class MakeDefaultSummaryTheSuperSet extends Processor { @Override public void process(boolean validate, boolean documentsOnly) { - DocumentSummary defaultSummary= schema.getSummary("default"); + DocumentSummary defaultSummary= schema.getSummariesInThis().get("default"); for (SummaryField summaryField : schema.getUniqueNamedSummaryFields().values() ) { if (defaultSummary.getSummaryField(summaryField.getName()) != null) continue; if (summaryField.getTransform() == SummaryTransform.ATTRIBUTE) continue; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/PredicateProcessor.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/PredicateProcessor.java index 73b75d6e23a..493bf9b5251 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/PredicateProcessor.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/PredicateProcessor.java @@ -68,7 +68,7 @@ public class PredicateProcessor extends Processor { attribute.setDensePostingListThreshold(threshold); addPredicateOptimizationIlScript(field, booleanDefinition); } - DocumentSummary summary = schema.getSummary("attributeprefetch"); + DocumentSummary summary = schema.getSummariesInThis().get("attributeprefetch"); if (summary != null) { summary.remove(attribute.getName()); } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ReferenceFieldsProcessor.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ReferenceFieldsProcessor.java index 19bfb41289d..57833fecc7a 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ReferenceFieldsProcessor.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ReferenceFieldsProcessor.java @@ -51,7 +51,7 @@ public class ReferenceFieldsProcessor extends Processor { } private void removeFromAttributePrefetchSummaryClass(SDField field) { - DocumentSummary summary = schema.getSummary("attributeprefetch"); + DocumentSummary summary = schema.getSummariesInThis().get("attributeprefetch"); if (summary != null) { summary.remove(field.getName()); } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryConsistency.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryConsistency.java index 8b86674e4d0..e2e0cf94bb8 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryConsistency.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryConsistency.java @@ -43,7 +43,7 @@ public class SummaryConsistency extends Processor { private void assertConsistency(SummaryField summaryField, Schema schema, boolean validate) { // Compare to default: - SummaryField existingDefault = schema.getSummary("default").getSummaryField(summaryField.getName()); + SummaryField existingDefault = schema.getSummariesInThis().get("default").getSummaryField(summaryField.getName()); if (existingDefault != null) { if (validate) assertConsistentTypes(existingDefault, summaryField); diff --git a/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentManager.java b/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentManager.java index 64b528ad858..8605389e1c2 100644 --- a/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentManager.java +++ b/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentManager.java @@ -117,13 +117,6 @@ public class DocumentManager { Datatype.Structtype.Builder structBuilder = new Datatype.Structtype.Builder(); builder.structtype(structBuilder); structBuilder.name(structType.getName()); - if (structType.getCompressionConfig().type.getCode() != 0) { - structBuilder. - compresstype(Datatype.Structtype.Compresstype.Enum.valueOf(structType.getCompressionConfig().type.toString())). - compresslevel(structType.getCompressionConfig().compressionLevel). - compressthreshold((int)structType.getCompressionConfig().threshold). - compressminsize((int)structType.getCompressionConfig().minsize); - } for (com.yahoo.document.Field field : structType.getFieldsThisTypeOnly()) { Datatype.Structtype.Field.Builder fieldBuilder = new Datatype.Structtype.Field.Builder(); structBuilder.field(fieldBuilder); diff --git a/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentTypes.java b/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentTypes.java index a89ef74a749..3b0b63f277e 100644 --- a/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentTypes.java +++ b/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentTypes.java @@ -136,13 +136,6 @@ public class DocumentTypes { DocumenttypesConfig.Documenttype.Datatype.Sstruct.Builder structBuilder = new DocumenttypesConfig.Documenttype.Datatype.Sstruct.Builder(); dataTypeBuilder.sstruct(structBuilder); structBuilder.name(type.getName()); - if (type.getCompressionConfig().type.getCode() != 0) { - structBuilder.compression(new DocumenttypesConfig.Documenttype.Datatype.Sstruct.Compression.Builder(). - type(DocumenttypesConfig.Documenttype.Datatype.Sstruct.Compression.Type.Enum.valueOf(type.getCompressionConfig().type.toString())). - level(type.getCompressionConfig().compressionLevel). - threshold((int) type.getCompressionConfig().threshold). - minsize((int) type.getCompressionConfig().minsize)); - } for (com.yahoo.document.Field field : type.getFields()) { DocumenttypesConfig.Documenttype.Datatype.Sstruct.Field.Builder builder = new DocumenttypesConfig.Documenttype.Datatype.Sstruct.Field.Builder(); diff --git a/config-model/src/main/java/com/yahoo/vespa/documentmodel/DocumentSummary.java b/config-model/src/main/java/com/yahoo/vespa/documentmodel/DocumentSummary.java index 3dda498b0be..59b4e1a2f9b 100644 --- a/config-model/src/main/java/com/yahoo/vespa/documentmodel/DocumentSummary.java +++ b/config-model/src/main/java/com/yahoo/vespa/documentmodel/DocumentSummary.java @@ -88,7 +88,7 @@ public class DocumentSummary extends FieldView { String sourceName = j.next().getName(); if (sourceName.equals(summaryField.getName())) continue; SummaryField sourceField=getSummaryField(sourceName); - if (sourceField==null) continue; + if (sourceField == null) continue; if (!sourceField.isImplicit()) continue; falseImplicits.add(sourceField); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java b/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java index d6c66a635d4..f584b4cd207 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java @@ -18,6 +18,7 @@ import com.yahoo.config.model.ApplicationConfigProducerRoot; import com.yahoo.config.model.ConfigModelRegistry; import com.yahoo.config.model.ConfigModelRepo; import com.yahoo.config.model.NullConfigModelRegistry; +import com.yahoo.config.model.api.ApplicationClusterInfo; import com.yahoo.config.model.api.HostInfo; import com.yahoo.config.model.api.Model; import com.yahoo.config.model.api.Provisioned; @@ -696,4 +697,8 @@ public final class VespaModel extends AbstractConfigProducerRoot implements Seri .collect(Collectors.toSet()); } + @Override + public Set<ApplicationClusterInfo> applicationClusterInfo() { + return Set.copyOf(getContainerClusters().values()); + } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java index c3897f49c44..a5dc26e19e3 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java @@ -7,10 +7,14 @@ import com.yahoo.component.ComponentId; import com.yahoo.component.ComponentSpecification; import com.yahoo.config.FileReference; import com.yahoo.config.application.api.ComponentInfo; +import com.yahoo.config.model.api.ApplicationClusterEndpoint; +import com.yahoo.config.model.api.ApplicationClusterInfo; +import com.yahoo.config.model.api.ContainerEndpoint; import com.yahoo.config.model.api.Model; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.producer.AbstractConfigProducer; import com.yahoo.config.provision.AllocatedHosts; +import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.HostSpec; import com.yahoo.container.bundle.BundleInstantiationSpecification; import com.yahoo.container.di.config.ApplicationBundlesConfig; @@ -26,6 +30,7 @@ import com.yahoo.vespa.config.search.RankProfilesConfig; import com.yahoo.vespa.config.search.core.OnnxModelsConfig; import com.yahoo.vespa.config.search.core.RankingConstantsConfig; import com.yahoo.vespa.config.search.core.RankingExpressionsConfig; +import com.yahoo.vespa.model.AbstractService; import com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyContainer; import com.yahoo.vespa.model.container.component.BindingPattern; import com.yahoo.vespa.model.container.component.Component; @@ -39,6 +44,7 @@ import com.yahoo.vespa.model.utils.FileSender; import java.util.ArrayList; import java.util.Collection; import java.util.LinkedHashSet; +import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; @@ -60,7 +66,8 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat ServletPathsConfig.Producer, ContainerMbusConfig.Producer, MetricsProxyApiConfig.Producer, - ZookeeperServerConfig.Producer { + ZookeeperServerConfig.Producer, + ApplicationClusterInfo { public static final String METRICS_V2_HANDLER_CLASS = MetricsV2Handler.class.getName(); public static final BindingPattern METRICS_V2_HANDLER_BINDING_1 = SystemBindingPattern.fromHttpPath(MetricsV2Handler.V2_PATH); @@ -88,6 +95,8 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat private Integer memoryPercentage = null; + private List<ApplicationClusterEndpoint> endpointList = List.of(); + public ApplicationContainerCluster(AbstractConfigProducer<?> parent, String configSubId, String clusterId, DeployState deployState) { super(parent, configSubId, clusterId, deployState, true); this.tlsClientAuthority = deployState.tlsClientAuthority(); @@ -115,6 +124,7 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat protected void doPrepare(DeployState deployState) { addAndSendApplicationBundles(deployState); sendUserConfiguredFiles(deployState); + createEndpointList(deployState); } private void addAndSendApplicationBundles(DeployState deployState) { @@ -184,6 +194,60 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat */ public Optional<Integer> getMemoryPercentage() { return Optional.ofNullable(memoryPercentage); } + /* + Create list of endpoints, these will be consumed later by the LBservicesProducer + */ + private void createEndpointList(DeployState deployState) { + if(!deployState.isHosted()) return; + if(deployState.getProperties().applicationId().instance().isTester()) return; + List<ApplicationClusterEndpoint> endpoints = new ArrayList<>(); + // Add zone local endpoints using zone dns suffixes, tenant, application and cluster id. + // For now support both L7 and L4 routing + + List<String> hosts = getContainers().stream() + .map(AbstractService::getHostName) + .collect(Collectors.toList()); + for(String suffix : deployState.getProperties().zoneDnsSuffixes()) { + // L4 + ApplicationClusterEndpoint.DnsName l4Name = ApplicationClusterEndpoint.DnsName.sharedL4NameFrom( + ClusterSpec.Id.from(getName()), + deployState.getProperties().applicationId(), + suffix); + endpoints.add(ApplicationClusterEndpoint.builder() + .zoneScope() + .sharedL4Routing() + .dnsName(l4Name) + .hosts(hosts) + .build()); + + // L7 + ApplicationClusterEndpoint.DnsName l7Name = ApplicationClusterEndpoint.DnsName.sharedNameFrom( + ClusterSpec.Id.from(getName()), + deployState.getProperties().applicationId(), + suffix); + endpoints.add(ApplicationClusterEndpoint.builder() + .zoneScope() + .sharedRouting() + .dnsName(l7Name) + .hosts(hosts) + .build()); + } + + // Then get all endpoints provided by controller. Can be created with L4 routing only + Set<ContainerEndpoint> endpointsFromController = deployState.getEndpoints(); + endpointsFromController.stream() + .filter(ce -> ce.clusterId().equals(getName())) + .forEach(ce -> ce.names().forEach( + name -> endpoints.add(ApplicationClusterEndpoint.builder() + .scope(ce.scope()) + .sharedL4Routing() + .dnsName(ApplicationClusterEndpoint.DnsName.from(name)) + .hosts(hosts) + .build()) + )); + endpointList = List.copyOf(endpoints); + } + @Override public void getConfig(ApplicationBundlesConfig.Builder builder) { applicationBundles.stream().map(FileReference::value) @@ -293,6 +357,11 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat null)))); } + @Override + public List<ApplicationClusterEndpoint> endpoints() { + return endpointList; + } + public static class MbusParams { // the amount of the maxpendingbytes to process concurrently, typically 0.2 (20%) final Double maxConcurrentFactor; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java index 39d4d7ec6c8..527897a3266 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java @@ -8,8 +8,10 @@ import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.application.api.DeploymentInstanceSpec; import com.yahoo.config.application.api.DeploymentSpec; +import com.yahoo.config.application.api.Endpoint; import com.yahoo.config.model.ConfigModelContext; import com.yahoo.config.model.ConfigModelContext.ApplicationType; +import com.yahoo.config.model.api.ApplicationClusterEndpoint; import com.yahoo.config.model.api.ConfigServerSpec; import com.yahoo.config.model.api.ContainerEndpoint; import com.yahoo.config.model.api.EndpointCertificateSecrets; @@ -281,6 +283,8 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { private void addCloudSecretStore(ApplicationContainerCluster cluster, Element secretStoreElement, DeployState deployState) { if ( ! deployState.isHosted()) return; + if ( ! cluster.getZone().system().isPublic()) + throw new RuntimeException("cloud secret store is not supported in non-public system, please see documentation"); CloudSecretStore cloudSecretStore = new CloudSecretStore(); Map<String, TenantSecretStore> secretStoresByName = deployState.getProperties().tenantSecretStores() .stream() @@ -338,9 +342,11 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { private void setRotations(Container container, Set<ContainerEndpoint> endpoints, String containerClusterName) { var rotationsProperty = endpoints.stream() - .filter(endpoint -> endpoint.clusterId().equals(containerClusterName)) - .flatMap(endpoint -> endpoint.names().stream()) - .collect(Collectors.toUnmodifiableSet()); + .filter(endpoint -> endpoint.clusterId().equals(containerClusterName)) + // Only consider global endpoints. + .filter(endpoint -> endpoint.scope() == ApplicationClusterEndpoint.Scope.global) + .flatMap(endpoint -> endpoint.names().stream()) + .collect(Collectors.toUnmodifiableSet()); // Build the comma delimited list of endpoints this container should be known as. // Confusingly called 'rotations' for legacy reasons. diff --git a/config-model/src/main/java/com/yahoo/vespa/model/package-info.java b/config-model/src/main/java/com/yahoo/vespa/model/package-info.java index 48a19037569..be1506c9418 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/package-info.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/package-info.java @@ -36,7 +36,7 @@ com.yahoo.config.model.producer.AbstractConfigProducer allocation</a>. </p> - <h3>Config Generation</h3> + <h2>Config Generation</h2> <p>The method {@link com.yahoo.vespa.model.VespaModel#getConfig(com.yahoo.config.ConfigInstance.Builder, String) @@ -61,7 +61,7 @@ com.yahoo.config.model.producer.AbstractConfigProducer method and by user defined config. </p> - <h4>Example:</h4> + <h3>Example:</h3> <p> Say we have a config named 'sample' with an integer parameter named 'v'. If the VespaModel root node's {@link @@ -99,7 +99,7 @@ com.yahoo.config.model.producer.AbstractConfigProducer </p> - <h3 id="plugin_loading">Plugin Loading</h3> + <h2 id="plugin_loading">Plugin Loading</h2> <p>Each highest-level node in the setup file from the user's application specification corresponds to a {@link @@ -116,7 +116,7 @@ com.yahoo.config.model.builder.xml.ConfigModelBuilder ConfigModelBuilder}. The <p>The built models are given to other models that depends on it. </p> - <h4>Important notes for plugin developers:</h4> + <h3>Important notes for plugin developers:</h3> <ul> <li>The constructors of all child classes of {@link @@ -138,7 +138,7 @@ com.yahoo.config.model.producer.AbstractConfigProducer </ul> - <h3 id="port_allocation">Port Allocation</h3> + <h2 id="port_allocation">Port Allocation</h2> <p>Each {@link com.yahoo.vespa.model.Host Host} has an available dynamic port range running from {@link |