diff options
Diffstat (limited to 'config-model/src/main/java/com')
68 files changed, 541 insertions, 189 deletions
diff --git a/config-model/src/main/java/com/yahoo/config/model/ApplicationConfigProducerRoot.java b/config-model/src/main/java/com/yahoo/config/model/ApplicationConfigProducerRoot.java index d9d7dd9eaf0..f324ceef5ab 100644 --- a/config-model/src/main/java/com/yahoo/config/model/ApplicationConfigProducerRoot.java +++ b/config-model/src/main/java/com/yahoo/config/model/ApplicationConfigProducerRoot.java @@ -233,6 +233,37 @@ public class ApplicationConfigProducerRoot extends TreeConfigProducer<AnyConfigP } } + // add cluster type? + // add cluster name? + public record StatePortInfo(String hostName, int portNumber, + String serviceName, String serviceType) + {} + + public List<StatePortInfo> getStatePorts() { + List<StatePortInfo> result = new ArrayList<>(); + for (HostResource modelHost : hostSystem().getHosts()) { + String hostName = modelHost.getHostname(); + for (Service modelService : modelHost.getServices()) { + String serviceName = modelService.getServiceName(); + String serviceType = modelService.getServiceType(); + PortsMeta portsMeta = modelService.getPortsMeta(); + for (int i = 0; i < portsMeta.getNumPorts(); i++) { + int portNumber = modelService.getRelativePort(i); + boolean hasState = false; + boolean isHttp = false; + for (String tag : portsMeta.getTagsAt(i)) { + if (tag.equals("state")) hasState = true; + if (tag.equals("http")) isHttp = true; + } + if (hasState && isHttp) { + result.add(new StatePortInfo(hostName, portNumber, serviceName, serviceType)); + } + } + } + } + return result; + } + private List<Services.Builder> getServices(HostResource modelHost) { List<Services.Builder> ret = new ArrayList<>(); for (Service modelService : modelHost.getServices()) { diff --git a/config-model/src/main/java/com/yahoo/config/model/ConfigModelRegistry.java b/config-model/src/main/java/com/yahoo/config/model/ConfigModelRegistry.java index 51bd01de5bc..f25f130eed0 100644 --- a/config-model/src/main/java/com/yahoo/config/model/ConfigModelRegistry.java +++ b/config-model/src/main/java/com/yahoo/config/model/ConfigModelRegistry.java @@ -5,7 +5,7 @@ import com.yahoo.config.model.builder.xml.ConfigModelBuilder; import com.yahoo.config.model.builder.xml.ConfigModelId; import java.util.Collection; -import java.util.Collections; +import java.util.List; /** * A resolver of implementations of named config models. @@ -44,7 +44,7 @@ public abstract class ConfigModelRegistry { @Override public Collection<ConfigModelBuilder> resolve(ConfigModelId id) { - return Collections.emptyList(); + return List.of(); } } diff --git a/config-model/src/main/java/com/yahoo/config/model/MapConfigModelRegistry.java b/config-model/src/main/java/com/yahoo/config/model/MapConfigModelRegistry.java index 8fe1372ef2f..7242d3de1dd 100644 --- a/config-model/src/main/java/com/yahoo/config/model/MapConfigModelRegistry.java +++ b/config-model/src/main/java/com/yahoo/config/model/MapConfigModelRegistry.java @@ -7,7 +7,6 @@ import com.yahoo.config.model.builder.xml.ConfigModelBuilder; import com.yahoo.config.model.builder.xml.ConfigModelId; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collection; import java.util.HashSet; import java.util.List; @@ -58,7 +57,7 @@ public class MapConfigModelRegistry extends ConfigModelRegistry { @SafeVarargs @SuppressWarnings("varargs") public static ConfigModelRegistry createFromList(ConfigModelBuilder<? extends ConfigModel> ... builders) { - return new MapConfigModelRegistry(Arrays.asList(builders)); + return new MapConfigModelRegistry(List.of(builders)); } } diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java b/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java index f19341098f4..57a75bd8a38 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java @@ -50,7 +50,6 @@ import java.io.Reader; import java.io.UncheckedIOException; import java.time.Instant; import java.util.Collection; -import java.util.Collections; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -329,7 +328,7 @@ public class DeployState implements ConfigDefinitionStore { private Optional<ConfigDefinitionRepo> configDefinitionRepo = Optional.empty(); private Optional<Model> previousModel = Optional.empty(); private Set<ContainerEndpoint> endpoints = Set.of(); - private Collection<MlModelImporter> modelImporters = Collections.emptyList(); + private Collection<MlModelImporter> modelImporters = List.of(); private Zone zone = Zone.defaultZone(); private Instant now = Instant.now(); private Version wantedNodeVespaVersion = Vtag.currentVersion; diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java index 11d3a48ee51..3c45588a054 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java @@ -19,7 +19,6 @@ import com.yahoo.vespa.model.container.ApplicationContainerCluster; import java.net.URI; import java.security.cert.X509Certificate; import java.util.Collection; -import java.util.Collections; import java.util.List; import java.util.Optional; import java.util.Set; @@ -35,10 +34,10 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea private boolean multitenant = false; private ApplicationId applicationId = ApplicationId.defaultId(); - private List<ConfigServerSpec> configServerSpecs = Collections.emptyList(); + private List<ConfigServerSpec> configServerSpecs = List.of(); private boolean hostedVespa = false; private Zone zone = Zone.defaultZone(); - private final Set<ContainerEndpoint> endpoints = Collections.emptySet(); + private final Set<ContainerEndpoint> endpoints = Set.of(); private boolean useDedicatedNodeForLogserver = false; private double defaultTermwiseLimit = 1.0; private String jvmGCOptions = null; @@ -55,10 +54,10 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea private double feedConcurrency = 0.5; private double feedNiceness = 0.0; private int maxActivationInhibitedOutOfSyncGroups = 0; - private List<TenantSecretStore> tenantSecretStores = Collections.emptyList(); + private List<TenantSecretStore> tenantSecretStores = List.of(); private String jvmOmitStackTraceInFastThrowOption; private boolean allowDisableMtls = true; - private List<X509Certificate> operatorCertificates = Collections.emptyList(); + private List<X509Certificate> operatorCertificates = List.of(); private double resourceLimitDisk = 0.75; private double resourceLimitMemory = 0.8; private double minNodeRatioPerGroup = 0.0; @@ -83,6 +82,8 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea private boolean allowUserFilters = true; private List<DataplaneToken> dataplaneTokens; private int contentLayerMetadataFeatureLevel = 0; + private int persistenceThreadMaxFeedOpBatchSize = 1; + private boolean logserverOtelCol = false; @Override public ModelContext.FeatureFlags featureFlags() { return this; } @Override public boolean multitenant() { return multitenant; } @@ -139,6 +140,8 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea @Override public boolean allowUserFilters() { return allowUserFilters; } @Override public List<DataplaneToken> dataplaneTokens() { return dataplaneTokens; } @Override public int contentLayerMetadataFeatureLevel() { return contentLayerMetadataFeatureLevel; } + @Override public int persistenceThreadMaxFeedOpBatchSize() { return persistenceThreadMaxFeedOpBatchSize; } + @Override public boolean logserverOtelCol() { return logserverOtelCol; } public TestProperties sharedStringRepoNoReclaim(boolean sharedStringRepoNoReclaim) { this.sharedStringRepoNoReclaim = sharedStringRepoNoReclaim; @@ -369,6 +372,16 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea return this; } + public TestProperties setPersistenceThreadMaxFeedOpBatchSize(int maxBatchSize) { + this.persistenceThreadMaxFeedOpBatchSize = maxBatchSize; + return this; + } + + public TestProperties setLogserverOtelCol(boolean logserverOtelCol) { + this.logserverOtelCol = logserverOtelCol; + return this; + } + public static class Spec implements ConfigServerSpec { private final String hostName; diff --git a/config-model/src/main/java/com/yahoo/config/model/graph/ModelNode.java b/config-model/src/main/java/com/yahoo/config/model/graph/ModelNode.java index e411a8e3e77..dcf1c8629a4 100644 --- a/config-model/src/main/java/com/yahoo/config/model/graph/ModelNode.java +++ b/config-model/src/main/java/com/yahoo/config/model/graph/ModelNode.java @@ -135,7 +135,7 @@ public class ModelNode<MODEL extends ConfigModel> implements ConfigModelInstance } // For collections, we don't require that dependency has been added, we just give an empty collection if (isCollection(param)) - return Collections.emptyList(); + return List.of(); throw new IllegalArgumentException("Unable to find constructor argument " + param + " for " + clazz.getName()); } diff --git a/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java b/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java index 5d3db7f676a..365543a549b 100644 --- a/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java +++ b/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java @@ -31,7 +31,6 @@ import java.io.UncheckedIOException; import java.nio.file.Files; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collections; import java.util.HashMap; import java.util.Iterator; import java.util.LinkedHashMap; @@ -143,7 +142,7 @@ public class MockApplicationPackage implements ApplicationPackage { @Override public Map<ConfigDefinitionKey, UnparsedConfigDefinition> getAllExistingConfigDefs() { - return Collections.emptyMap(); + return Map.of(); } @Override @@ -203,7 +202,7 @@ public class MockApplicationPackage implements ApplicationPackage { } public List<ComponentInfo> getComponentsInfo(Version vespaVersion) { - return Collections.emptyList(); + return List.of(); } public QueryProfileRegistry getQueryProfiles() { return queryProfileRegistry; } @@ -240,7 +239,7 @@ public class MockApplicationPackage implements ApplicationPackage { private File root = new File("nonexisting"); private String hosts = null; private String services = null; - private List<String> schemas = Collections.emptyList(); + private List<String> schemas = List.of(); private Map<Path, MockApplicationFile> files = new LinkedHashMap<>(); private String schemaDir = null; private String deploymentSpec = null; @@ -277,12 +276,12 @@ public class MockApplicationPackage implements ApplicationPackage { } public Builder withSearchDefinition(String searchDefinition) { - this.schemas = Collections.singletonList(searchDefinition); + this.schemas = List.of(searchDefinition); return this; } public Builder withSchemas(List<String> searchDefinition) { - this.schemas = Collections.unmodifiableList(searchDefinition); + this.schemas = List.copyOf(searchDefinition); return this; } @@ -369,8 +368,8 @@ public class MockApplicationPackage implements ApplicationPackage { } private List<NamedReader> asNamedReaderList(String value) { - if (value == null) return Collections.emptyList(); - return Collections.singletonList(new NamedReader(extractId(value) + ".xml", new StringReader(value))); + if (value == null) return List.of(); + return List.of(new NamedReader(extractId(value) + ".xml", new StringReader(value))); } private String extractId(String xmlStringWithIdAttribute) { diff --git a/config-model/src/main/java/com/yahoo/config/model/test/TestUtil.java b/config-model/src/main/java/com/yahoo/config/model/test/TestUtil.java index 94969aa5324..509c10c7fc9 100644 --- a/config-model/src/main/java/com/yahoo/config/model/test/TestUtil.java +++ b/config-model/src/main/java/com/yahoo/config/model/test/TestUtil.java @@ -6,7 +6,6 @@ import com.yahoo.config.model.builder.xml.XmlHelper; import org.w3c.dom.Element; import java.io.StringReader; import java.util.ArrayList; -import java.util.Arrays; import java.util.List; /** @@ -20,7 +19,7 @@ public class TestUtil { public static Element parse(String... xmlLines) { List<String> lines = new ArrayList<>(); lines.add("<?xml version='1.0' encoding='utf-8' ?>"); - lines.addAll(Arrays.asList(xmlLines)); + lines.addAll(List.of(xmlLines)); try { return XmlHelper.getDocument(new StringReader(CollectionUtil.mkString(lines, "\n").replace("'", "\""))).getDocumentElement(); diff --git a/config-model/src/main/java/com/yahoo/documentmodel/NewDocumentType.java b/config-model/src/main/java/com/yahoo/documentmodel/NewDocumentType.java index 77e123fefef..c5140dd6fc8 100644 --- a/config-model/src/main/java/com/yahoo/documentmodel/NewDocumentType.java +++ b/config-model/src/main/java/com/yahoo/documentmodel/NewDocumentType.java @@ -25,7 +25,6 @@ import java.util.Map; import java.util.Optional; import java.util.Set; -import static java.util.Collections.emptySet; /** * @author baldersheim @@ -47,7 +46,7 @@ public final class NewDocumentType extends StructuredDataType implements DataTyp private final Set<String> importedFieldNames; public NewDocumentType(Name name) { - this(name, emptySet()); + this(name, Set.of()); } public NewDocumentType(Name name, Set<Name> documentReferences, Set<String> importedFieldNames) { @@ -60,7 +59,7 @@ public final class NewDocumentType extends StructuredDataType implements DataTyp } public NewDocumentType(Name name, Set<Name> documentReferences) { - this(name, documentReferences, emptySet()); + this(name, documentReferences, Set.of()); } public NewDocumentType(Name name, @@ -74,8 +73,8 @@ public final class NewDocumentType extends StructuredDataType implements DataTyp if (fs != null) { this.fieldSets.addAll(fs.userFieldSets().values()); for (FieldSet f : fs.builtInFieldSets().values()) { - if ((f.getName() != BuiltInFieldSets.INTERNAL_FIELDSET_NAME) && - (f.getName() != BuiltInFieldSets.SEARCH_FIELDSET_NAME)) { + if (!BuiltInFieldSets.INTERNAL_FIELDSET_NAME.equals(f.getName()) && + !BuiltInFieldSets.SEARCH_FIELDSET_NAME.equals(f.getName())) { fieldSets.add(f); } } diff --git a/config-model/src/main/java/com/yahoo/schema/ApplicationBuilder.java b/config-model/src/main/java/com/yahoo/schema/ApplicationBuilder.java index 2e964ac3624..749cdbfd8b7 100644 --- a/config-model/src/main/java/com/yahoo/schema/ApplicationBuilder.java +++ b/config-model/src/main/java/com/yahoo/schema/ApplicationBuilder.java @@ -27,7 +27,6 @@ import java.io.IOException; import java.nio.file.Files; import java.util.ArrayList; import java.util.Collection; -import java.util.Collections; import java.util.Comparator; import java.util.HashSet; import java.util.List; @@ -351,7 +350,7 @@ public class ApplicationBuilder { RankProfileRegistry rankProfileRegistry, QueryProfileRegistry queryprofileRegistry) throws IOException, ParseException { - return createFromFiles(Collections.singletonList(fileName), new MockFileRegistry(), deployLogger, new TestProperties(), + return createFromFiles(List.of(fileName), new MockFileRegistry(), deployLogger, new TestProperties(), rankProfileRegistry, queryprofileRegistry); } @@ -414,8 +413,8 @@ public class ApplicationBuilder { queryProfileRegistry); var fnli = Files.list(new File(dir).toPath()) - .map(p -> p.toString()) - .filter(fn -> AbstractApplicationPackage.validSchemaFilename(fn)) + .map(java.nio.file.Path::toString) + .filter(AbstractApplicationPackage::validSchemaFilename) .sorted(); for (var i = fnli.iterator(); i.hasNext(); ) { builder.addSchemaFile(i.next()); diff --git a/config-model/src/main/java/com/yahoo/schema/FieldSets.java b/config-model/src/main/java/com/yahoo/schema/FieldSets.java index 1dbf496992b..6464bd1681e 100644 --- a/config-model/src/main/java/com/yahoo/schema/FieldSets.java +++ b/config-model/src/main/java/com/yahoo/schema/FieldSets.java @@ -47,11 +47,7 @@ public class FieldSets { * @param field field to add to field set */ public void addBuiltInFieldSetItem(String setName, String field) { - if (builtInFieldSets.get(setName) == null) { - // First entry in this set - builtInFieldSets.put(setName, new FieldSet(setName)); - } - builtInFieldSets.get(setName).addFieldName(field); + builtInFieldSets.computeIfAbsent(setName, FieldSet::new).addFieldName(field); } /** Returns the built in field sets, unmodifiable */ diff --git a/config-model/src/main/java/com/yahoo/schema/RankProfile.java b/config-model/src/main/java/com/yahoo/schema/RankProfile.java index 22bf1880cd7..82ed45028b3 100644 --- a/config-model/src/main/java/com/yahoo/schema/RankProfile.java +++ b/config-model/src/main/java/com/yahoo/schema/RankProfile.java @@ -32,7 +32,6 @@ import java.io.IOException; import java.io.Reader; import java.io.StringReader; import java.util.ArrayList; -import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; @@ -487,12 +486,12 @@ public class RankProfile implements Cloneable { } void setFirstPhaseRanking(RankingExpression rankingExpression) { - this.firstPhaseRanking = new RankingExpressionFunction(new ExpressionFunction(FIRST_PHASE, Collections.emptyList(), rankingExpression), false); + this.firstPhaseRanking = new RankingExpressionFunction(new ExpressionFunction(FIRST_PHASE, List.of(), rankingExpression), false); } public void setFirstPhaseRanking(String expression) { try { - firstPhaseRanking = new RankingExpressionFunction(parseRankingExpression(FIRST_PHASE, Collections.emptyList(), expression), false); + firstPhaseRanking = new RankingExpressionFunction(parseRankingExpression(FIRST_PHASE, List.of(), expression), false); } catch (ParseException e) { throw new IllegalArgumentException("Illegal first phase ranking function", e); } @@ -515,7 +514,7 @@ public class RankProfile implements Cloneable { public void setSecondPhaseRanking(String expression) { try { - secondPhaseRanking = new RankingExpressionFunction(parseRankingExpression(SECOND_PHASE, Collections.emptyList(), expression), false); + secondPhaseRanking = new RankingExpressionFunction(parseRankingExpression(SECOND_PHASE, List.of(), expression), false); } catch (ParseException e) { throw new IllegalArgumentException("Illegal second phase ranking function", e); @@ -535,7 +534,7 @@ public class RankProfile implements Cloneable { public void setGlobalPhaseRanking(String expression) { try { - globalPhaseRanking = new RankingExpressionFunction(parseRankingExpression(GLOBAL_PHASE, Collections.emptyList(), expression), false); + globalPhaseRanking = new RankingExpressionFunction(parseRankingExpression(GLOBAL_PHASE, List.of(), expression), false); } catch (ParseException e) { throw new IllegalArgumentException("Illegal global-phase ranking function", e); @@ -1036,7 +1035,7 @@ public class RankProfile implements Cloneable { Map<Reference, TensorType> featureTypes = featureTypes(); // Function compiling first pass: compile inline functions without resolving other functions Map<String, RankingExpressionFunction> inlineFunctions = - compileFunctions(this::getInlineFunctions, queryProfiles, featureTypes, importedModels, Collections.emptyMap(), expressionTransforms); + compileFunctions(this::getInlineFunctions, queryProfiles, featureTypes, importedModels, Map.of(), expressionTransforms); firstPhaseRanking = compile(this.getFirstPhase(), queryProfiles, featureTypes, importedModels, constants(), inlineFunctions, expressionTransforms); secondPhaseRanking = compile(this.getSecondPhase(), queryProfiles, featureTypes, importedModels, constants(), inlineFunctions, expressionTransforms); diff --git a/config-model/src/main/java/com/yahoo/schema/RankProfileRegistry.java b/config-model/src/main/java/com/yahoo/schema/RankProfileRegistry.java index 0fddcbd4cc7..4deee1dce9f 100644 --- a/config-model/src/main/java/com/yahoo/schema/RankProfileRegistry.java +++ b/config-model/src/main/java/com/yahoo/schema/RankProfileRegistry.java @@ -4,9 +4,7 @@ package com.yahoo.schema; import com.yahoo.schema.document.SDDocumentType; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collection; -import java.util.HashSet; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -27,7 +25,7 @@ public class RankProfileRegistry { private static final String globalRankProfilesKey = "[global]"; /* These rank profiles can be overridden: 'default' rank profile, as that is documented to work. And 'unranked'. */ - static final Set<String> overridableRankProfileNames = new HashSet<>(Arrays.asList("default", "unranked")); + static final Set<String> overridableRankProfileNames = Set.of("default", "unranked"); public static RankProfileRegistry createRankProfileRegistryWithBuiltinRankProfiles(Schema schema) { RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); diff --git a/config-model/src/main/java/com/yahoo/schema/derived/Deriver.java b/config-model/src/main/java/com/yahoo/schema/derived/Deriver.java index 9774868db9c..0c669e29538 100644 --- a/config-model/src/main/java/com/yahoo/schema/derived/Deriver.java +++ b/config-model/src/main/java/com/yahoo/schema/derived/Deriver.java @@ -7,7 +7,6 @@ import com.yahoo.schema.parser.ParseException; import com.yahoo.vespa.configmodel.producers.DocumentManager; import com.yahoo.vespa.configmodel.producers.DocumentTypes; import java.io.IOException; -import java.util.Collections; import java.util.List; /** @@ -30,7 +29,7 @@ public class Deriver { } public static DocumentmanagerConfig.Builder getDocumentManagerConfig(String sd) { - return getDocumentManagerConfig(Collections.singletonList(sd)); + return getDocumentManagerConfig(List.of(sd)); } public static DocumentmanagerConfig.Builder getDocumentManagerConfig(List<String> schemas) { @@ -38,7 +37,7 @@ public class Deriver { } public static DocumenttypesConfig.Builder getDocumentTypesConfig(String schema) { - return getDocumentTypesConfig(Collections.singletonList(schema)); + return getDocumentTypesConfig(List.of(schema)); } public static DocumenttypesConfig.Builder getDocumentTypesConfig(List<String> schemas) { diff --git a/config-model/src/main/java/com/yahoo/schema/derived/IndexSchema.java b/config-model/src/main/java/com/yahoo/schema/derived/IndexSchema.java index 34ed9f2f60c..677e4b31bdf 100644 --- a/config-model/src/main/java/com/yahoo/schema/derived/IndexSchema.java +++ b/config-model/src/main/java/com/yahoo/schema/derived/IndexSchema.java @@ -14,7 +14,6 @@ import com.yahoo.vespa.config.search.IndexschemaConfig; import java.io.IOException; import java.util.ArrayList; -import java.util.Collections; import java.util.LinkedHashMap; import java.util.LinkedList; import java.util.List; @@ -147,7 +146,7 @@ public class IndexSchema extends Derived { static List<Field> flattenField(Field field) { DataType fieldType = field.getDataType(); if (fieldType.getPrimitiveType() != null){ - return Collections.singletonList(field); + return List.of(field); } if (fieldType instanceof ArrayDataType) { List<Field> ret = new LinkedList<>(); diff --git a/config-model/src/main/java/com/yahoo/schema/document/Attribute.java b/config-model/src/main/java/com/yahoo/schema/document/Attribute.java index 999d040b48c..ac936ee989e 100644 --- a/config-model/src/main/java/com/yahoo/schema/document/Attribute.java +++ b/config-model/src/main/java/com/yahoo/schema/document/Attribute.java @@ -5,7 +5,9 @@ import com.yahoo.document.ArrayDataType; import com.yahoo.document.CollectionDataType; import com.yahoo.document.DataType; import com.yahoo.document.DocumentType; +import com.yahoo.document.MapDataType; import com.yahoo.document.PrimitiveDataType; +import com.yahoo.document.StructDataType; import com.yahoo.documentmodel.NewDocumentReferenceDataType; import com.yahoo.document.StructuredDataType; import com.yahoo.document.TensorDataType; @@ -143,8 +145,8 @@ public final class Attribute implements Cloneable, Serializable { } /** Creates an attribute with default settings */ - public Attribute(String name, DataType fieldType) { - this(name, convertDataType(fieldType), convertCollectionType(fieldType), convertTensorType(fieldType), convertTargetType(fieldType)); + public Attribute(String schemaName, String fieldName, String name, DataType fieldType) { + this(name, convertDataType(schemaName, fieldName, fieldType), convertCollectionType(fieldType), convertTensorType(fieldType), convertTargetType(fieldType)); setRemoveIfZero(fieldType instanceof WeightedSetDataType wsdt && wsdt.removeIfZero()); setCreateIfNonExistent(fieldType instanceof WeightedSetDataType wsdt && wsdt.createIfNonExistent()); } @@ -266,12 +268,26 @@ public final class Attribute implements Cloneable, Serializable { private void setType(Type type) { this.type=type; } public void setCollectionType(CollectionType type) { this.collectionType=type; } + private static void failDataType(String schemaName, String fieldName, String dataType) throws IllegalArgumentException { + throw new IllegalArgumentException("For schema '" + schemaName + "': Field '" + fieldName + "' of type '" + dataType + "' cannot be an attribute. " + + "Instead specify the struct fields to be searchable as attribute"); + } + public static void validateDataType(String schemaName, String fieldName, DataType fieldType) throws IllegalArgumentException { + if (fieldType instanceof MapDataType mapType) { + failDataType(schemaName, fieldName, "map<" + mapType.getKeyType().getName() + "," + mapType.getValueType().getName() + ">"); + } + if (fieldType instanceof ArrayDataType arrayType && arrayType.getNestedType() instanceof StructDataType nestedType) { + failDataType(schemaName, fieldName, "array<" + nestedType.getName() + ">"); + } + } + /** Converts to the right attribute type from a field datatype */ - public static Type convertDataType(DataType fieldType) { + public static Type convertDataType(String schemaName, String fieldName, DataType fieldType) { + validateDataType(schemaName, fieldName, fieldType); if (fieldType instanceof NewDocumentReferenceDataType) { return Type.REFERENCE; } else if (fieldType instanceof CollectionDataType) { - return convertDataType(((CollectionDataType) fieldType).getNestedType()); + return convertDataType(schemaName, fieldName, ((CollectionDataType) fieldType).getNestedType()); } FieldValue fval = fieldType.createFieldValue(); if (fval instanceof StringFieldValue) { diff --git a/config-model/src/main/java/com/yahoo/schema/document/ImmutableImportedSDField.java b/config-model/src/main/java/com/yahoo/schema/document/ImmutableImportedSDField.java index cb98cb79e01..7ade7ada061 100644 --- a/config-model/src/main/java/com/yahoo/schema/document/ImmutableImportedSDField.java +++ b/config-model/src/main/java/com/yahoo/schema/document/ImmutableImportedSDField.java @@ -10,7 +10,6 @@ import com.yahoo.vespa.indexinglanguage.expressions.Expression; import com.yahoo.vespa.indexinglanguage.expressions.ScriptExpression; import java.util.Collection; -import java.util.Collections; import java.util.List; import java.util.Map; @@ -131,7 +130,7 @@ public class ImmutableImportedSDField implements ImmutableSDField { @Override public Map<String, String> getAliasToName() { - return Collections.emptyMap(); + return Map.of(); } @Override diff --git a/config-model/src/main/java/com/yahoo/schema/document/SDField.java b/config-model/src/main/java/com/yahoo/schema/document/SDField.java index 538cb56d210..f165141b16e 100644 --- a/config-model/src/main/java/com/yahoo/schema/document/SDField.java +++ b/config-model/src/main/java/com/yahoo/schema/document/SDField.java @@ -398,22 +398,23 @@ public class SDField extends Field implements TypedKey, ImmutableSDField { } /** Parse an indexing expression which will use the simple linguistics implementation suitable for testing */ - public void parseIndexingScript(String script) { - parseIndexingScript(script, new SimpleLinguistics(), Embedder.throwsOnUse.asMap()); + public void parseIndexingScript(String schemaName, String script) { + parseIndexingScript(schemaName, script, new SimpleLinguistics(), Embedder.throwsOnUse.asMap()); } - public void parseIndexingScript(String script, Linguistics linguistics, Map<String, Embedder> embedders) { + public void parseIndexingScript(String schemaName, String script, Linguistics linguistics, Map<String, Embedder> embedders) { try { ScriptParserContext config = new ScriptParserContext(linguistics, embedders); config.setInputStream(new IndexingInput(script)); - setIndexingScript(ScriptExpression.newInstance(config)); + setIndexingScript(schemaName, ScriptExpression.newInstance(config)); } catch (ParseException e) { throw new IllegalArgumentException("Failed to parse script '" + script + "'", e); } } /** Sets the indexing script of this, or null to not use a script */ - public void setIndexingScript(ScriptExpression exp) { + + public void setIndexingScript(String schemaName, ScriptExpression exp) { if (exp == null) { exp = new ScriptExpression(); } @@ -441,13 +442,13 @@ public class SDField extends Field implements TypedKey, ImmutableSDField { } Attribute attribute = attributes.get(fieldName); if (attribute == null) { - addAttribute(new Attribute(fieldName, getDataType())); + addAttribute(new Attribute(schemaName, fieldName, fieldName, getDataType())); } } }.visit(indexingScript); } for (SDField structField : getStructFields()) { - structField.setIndexingScript(exp); + structField.setIndexingScript(schemaName, exp); } } diff --git a/config-model/src/main/java/com/yahoo/schema/fieldoperation/FieldOperation.java b/config-model/src/main/java/com/yahoo/schema/fieldoperation/FieldOperation.java index 22a9eed2914..5a0c2c3c915 100644 --- a/config-model/src/main/java/com/yahoo/schema/fieldoperation/FieldOperation.java +++ b/config-model/src/main/java/com/yahoo/schema/fieldoperation/FieldOperation.java @@ -12,7 +12,7 @@ import com.yahoo.schema.document.SDField; public interface FieldOperation extends Comparable<FieldOperation> { /** Apply this operation on the given field */ - void apply(SDField field); + void apply(String schemaName, SDField field); @Override default int compareTo(FieldOperation other) { diff --git a/config-model/src/main/java/com/yahoo/schema/fieldoperation/IndexingOperation.java b/config-model/src/main/java/com/yahoo/schema/fieldoperation/IndexingOperation.java index f5366c4b07a..11065f040ea 100644 --- a/config-model/src/main/java/com/yahoo/schema/fieldoperation/IndexingOperation.java +++ b/config-model/src/main/java/com/yahoo/schema/fieldoperation/IndexingOperation.java @@ -28,8 +28,8 @@ public class IndexingOperation implements FieldOperation { public ScriptExpression getScript() { return script; } - public void apply(SDField field) { - field.setIndexingScript(script); + public void apply(String schemaName, SDField field) { + field.setIndexingScript(schemaName, script); } /** Creates an indexing operation which will use the simple linguistics implementation suitable for testing */ diff --git a/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedFields.java b/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedFields.java index e3ca0090408..053a5ac777b 100644 --- a/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedFields.java +++ b/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedFields.java @@ -2,6 +2,7 @@ package com.yahoo.schema.parser; import com.yahoo.document.DataType; +import com.yahoo.schema.document.GeoPos; import com.yahoo.schema.parser.ConvertParsedTypes.TypeResolver; import com.yahoo.schema.Index; import com.yahoo.schema.Schema; @@ -49,10 +50,10 @@ public class ConvertParsedFields { (exactMatchTerminator -> field.getMatching().setExactMatchTerminator(exactMatchTerminator)); } - void convertSorting(SDField field, ParsedSorting parsed, String name) { + void convertSorting(Schema schema, SDField field, ParsedSorting parsed, String name) { Attribute attribute = field.getAttributes().get(name); if (attribute == null) { - attribute = new Attribute(name, field.getDataType()); + attribute = new Attribute(schema.getName(), field.getName(), name, field.getDataType()); field.addAttribute(attribute); } Sorting sorting = attribute.getSorting(); @@ -66,7 +67,7 @@ public class ConvertParsedFields { parsed.getLocale().ifPresent(locale -> sorting.setLocale(locale)); } - void convertAttribute(SDField field, ParsedAttribute parsed) { + void convertAttribute(Schema schema, SDField field, ParsedAttribute parsed) { String name = parsed.name(); String fieldName = field.getName(); Attribute attribute = null; @@ -76,7 +77,7 @@ public class ConvertParsedFields { if (attribute == null) { attribute = field.getAttributes().get(name); if (attribute == null) { - attribute = new Attribute(name, field.getDataType()); + attribute = new Attribute(schema.getName(), field.getName(), name, field.getDataType()); field.addAttribute(attribute); } } @@ -102,7 +103,7 @@ public class ConvertParsedFields { } var sorting = parsed.getSorting(); if (sorting.isPresent()) { - convertSorting(field, sorting.get(), name); + convertSorting(schema, field, sorting.get(), name); } } @@ -143,13 +144,16 @@ public class ConvertParsedFields { convertMatchSettings(field, parsed.matchSettings()); var indexing = parsed.getIndexing(); if (indexing.isPresent()) { - field.setIndexingScript(indexing.get().script()); + field.setIndexingScript(schema.getName(), indexing.get().script()); + } + if (field.doesAttributing() && !GeoPos.isAnyPos(field.getDataType())) { + Attribute.validateDataType(schema.getName(), field.getName(), field.getDataType()); } parsed.getWeight().ifPresent(value -> field.setWeight(value)); parsed.getStemming().ifPresent(value -> field.setStemming(value)); parsed.getNormalizing().ifPresent(value -> convertNormalizing(field, value)); for (var attribute : parsed.getAttributes()) { - convertAttribute(field, attribute); + convertAttribute(schema, field, attribute); } for (var summaryField : parsed.getSummaryFields()) { var dataType = field.getDataType(); @@ -190,7 +194,7 @@ public class ConvertParsedFields { convertCommonFieldSettings(schema, structField, parsed); } - private void convertExtraFieldSettings(SDField field, ParsedField parsed) { + private void convertExtraFieldSettings(Schema schema, SDField field, ParsedField parsed) { String name = parsed.name(); for (var dictOp : parsed.getDictionaryOptions()) { var dictionary = field.getOrSetDictionary(); @@ -208,7 +212,7 @@ public class ConvertParsedFields { field.getAliasToName().put(alias, parsed.lookupAliasedFrom(alias)); } parsed.getRankTypes().forEach((indexName, rankType) -> convertRankType(field, indexName, rankType)); - parsed.getSorting().ifPresent(sortInfo -> convertSorting(field, sortInfo, name)); + parsed.getSorting().ifPresent(sortInfo -> convertSorting(schema, field, sortInfo, name)); if (parsed.hasBolding()) { // TODO must it be so ugly: SummaryField summaryField = field.getSummaryField(name, true); @@ -288,7 +292,7 @@ public class ConvertParsedFields { DataType dataType = context.resolveType(parsed.getType()); var field = new SDField(document, name, dataType); convertCommonFieldSettings(schema, field, parsed); - convertExtraFieldSettings(field, parsed); + convertExtraFieldSettings(schema, field, parsed); document.addField(field); return field; } @@ -298,7 +302,7 @@ public class ConvertParsedFields { DataType dataType = context.resolveType(parsed.getType()); var field = new SDField(schema.getDocument(), name, dataType); convertCommonFieldSettings(schema, field, parsed); - convertExtraFieldSettings(field, parsed); + convertExtraFieldSettings(schema, field, parsed); schema.addExtraField(field); } diff --git a/config-model/src/main/java/com/yahoo/schema/processing/CreatePositionZCurve.java b/config-model/src/main/java/com/yahoo/schema/processing/CreatePositionZCurve.java index 7bd66ad8f0b..deb57e157f3 100644 --- a/config-model/src/main/java/com/yahoo/schema/processing/CreatePositionZCurve.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/CreatePositionZCurve.java @@ -86,14 +86,14 @@ public class CreatePositionZCurve extends Processor { SummaryTransform.DISTANCE, summaryTo, validate); } // clear indexing script - field.setIndexingScript(null); + field.setIndexingScript(schema.getName(), null); SDField posX = field.getStructField(PositionDataType.FIELD_X); if (posX != null) { - posX.setIndexingScript(null); + posX.setIndexingScript(schema.getName(), null); } SDField posY = field.getStructField(PositionDataType.FIELD_Y); if (posY != null) { - posY.setIndexingScript(null); + posY.setIndexingScript(schema.getName(), null); } if (doesSummary) ensureCompatibleSummary(field, zName, field.getName(), @@ -118,7 +118,7 @@ public class CreatePositionZCurve extends Processor { ScriptExpression script = inputField.getIndexingScript(); script = (ScriptExpression)new RemoveSummary(inputField.getName()).convert(script); script = (ScriptExpression)new PerformZCurve(field, fieldName).convert(script); - field.setIndexingScript(script); + field.setIndexingScript(schema.getName(), script); return field; } diff --git a/config-model/src/main/java/com/yahoo/schema/processing/ExactMatch.java b/config-model/src/main/java/com/yahoo/schema/processing/ExactMatch.java index a12183262c4..056c37a9830 100644 --- a/config-model/src/main/java/com/yahoo/schema/processing/ExactMatch.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/ExactMatch.java @@ -75,7 +75,7 @@ public class ExactMatch extends Processor { } ScriptExpression script = field.getIndexingScript(); if (new ExpressionSearcher<>(IndexExpression.class).containedIn(script)) { - field.setIndexingScript((ScriptExpression)new MyProvider(schema).convert(field.getIndexingScript())); + field.setIndexingScript(schema.getName(), (ScriptExpression)new MyProvider(schema).convert(field.getIndexingScript())); } } diff --git a/config-model/src/main/java/com/yahoo/schema/processing/IndexingInputs.java b/config-model/src/main/java/com/yahoo/schema/processing/IndexingInputs.java index 53ebd136e08..a5ba67d6976 100644 --- a/config-model/src/main/java/com/yahoo/schema/processing/IndexingInputs.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/IndexingInputs.java @@ -36,7 +36,7 @@ public class IndexingInputs extends Processor { if (validate) new VerifyInputExpression(schema, field).visit(script); - field.setIndexingScript(script); + field.setIndexingScript(schema.getName(), script); } } diff --git a/config-model/src/main/java/com/yahoo/schema/processing/IndexingOutputs.java b/config-model/src/main/java/com/yahoo/schema/processing/IndexingOutputs.java index 071c2878ae8..b5d1cf71809 100644 --- a/config-model/src/main/java/com/yahoo/schema/processing/IndexingOutputs.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/IndexingOutputs.java @@ -37,7 +37,7 @@ public class IndexingOutputs extends Processor { Set<String> summaryFields = new TreeSet<>(); findSummaryTo(schema, field, summaryFields, summaryFields); MyConverter converter = new MyConverter(schema, field, summaryFields, validate); - field.setIndexingScript((ScriptExpression)converter.convert(script)); + field.setIndexingScript(schema.getName(), (ScriptExpression)converter.convert(script)); } } @@ -106,7 +106,7 @@ public class IndexingOutputs extends Processor { MyConverter(Schema schema, Field field, Set<String> summaryFields, boolean validate) { this.schema = schema; this.field = field; - this.summaryFields = summaryFields.isEmpty() ? Collections.singleton(field.getName()) : summaryFields; + this.summaryFields = summaryFields.isEmpty() ? Set.of(field.getName()) : summaryFields; this.validate = validate; } diff --git a/config-model/src/main/java/com/yahoo/schema/processing/IntegerIndex2Attribute.java b/config-model/src/main/java/com/yahoo/schema/processing/IntegerIndex2Attribute.java index 0d296783cfb..37815ef5396 100644 --- a/config-model/src/main/java/com/yahoo/schema/processing/IntegerIndex2Attribute.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/IntegerIndex2Attribute.java @@ -38,7 +38,7 @@ public class IntegerIndex2Attribute extends Processor { ScriptExpression script = field.getIndexingScript(); Set<String> attributeNames = new HashSet<>(); new MyVisitor(attributeNames).visit(script); - field.setIndexingScript((ScriptExpression)new MyConverter(attributeNames).convert(script)); + field.setIndexingScript(schema.getName(), (ScriptExpression)new MyConverter(attributeNames).convert(script)); warn(schema, field, "Changed to attribute because numerical indexes (field has type " + field.getDataType().getName() + ") is not currently supported." + " Index-only settings may fail. Ignore this warning for streaming search."); diff --git a/config-model/src/main/java/com/yahoo/schema/processing/NGramMatch.java b/config-model/src/main/java/com/yahoo/schema/processing/NGramMatch.java index 2ec5c03e04c..c06e0565109 100644 --- a/config-model/src/main/java/com/yahoo/schema/processing/NGramMatch.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/NGramMatch.java @@ -46,7 +46,7 @@ public class NGramMatch extends Processor { field.getNormalizing().inferCodepoint(); field.setStemming(Stemming.NONE); // not compatible with stemming and normalizing field.addQueryCommand("ngram " + n); - field.setIndexingScript((ScriptExpression)new MyProvider(schema, n).convert(field.getIndexingScript())); + field.setIndexingScript(schema.getName(), (ScriptExpression)new MyProvider(schema, n).convert(field.getIndexingScript())); } private static class MyProvider extends TypedTransformProvider { diff --git a/config-model/src/main/java/com/yahoo/schema/processing/OptimizeIlscript.java b/config-model/src/main/java/com/yahoo/schema/processing/OptimizeIlscript.java index b268a7a9c03..fbb49497837 100644 --- a/config-model/src/main/java/com/yahoo/schema/processing/OptimizeIlscript.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/OptimizeIlscript.java @@ -27,7 +27,7 @@ public class OptimizeIlscript extends Processor { ScriptExpression script = field.getIndexingScript(); if (script == null) continue; - field.setIndexingScript((ScriptExpression)new ExpressionOptimizer().convert(script)); + field.setIndexingScript(schema.getName(), (ScriptExpression)new ExpressionOptimizer().convert(script)); if ( ! field.getIndexingScript().toString().equals(script.toString())) { info(schema, field, "Rewrote ilscript from:\n" + script.toString() + "\nto\n" + field.getIndexingScript().toString()); diff --git a/config-model/src/main/java/com/yahoo/schema/processing/PredicateProcessor.java b/config-model/src/main/java/com/yahoo/schema/processing/PredicateProcessor.java index 2a654991835..c451df6370d 100644 --- a/config-model/src/main/java/com/yahoo/schema/processing/PredicateProcessor.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/PredicateProcessor.java @@ -107,7 +107,7 @@ public class PredicateProcessor extends Processor { script = new StatementExpression(makeSetPredicateVariablesScript(booleanIndexDefiniton), script); ExpressionConverter converter = new PredicateOutputTransformer(schema); - field.setIndexingScript(new ScriptExpression((StatementExpression)converter.convert(script))); + field.setIndexingScript(schema.getName(), new ScriptExpression((StatementExpression)converter.convert(script))); } private Expression makeSetPredicateVariablesScript(BooleanIndexDefinition options) { diff --git a/config-model/src/main/java/com/yahoo/schema/processing/Processing.java b/config-model/src/main/java/com/yahoo/schema/processing/Processing.java index c23d87e9eba..52078326aec 100644 --- a/config-model/src/main/java/com/yahoo/schema/processing/Processing.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/Processing.java @@ -9,8 +9,8 @@ import com.yahoo.vespa.model.container.search.QueryProfiles; import com.yahoo.config.model.api.ModelContext; import com.yahoo.config.model.deploy.TestProperties; -import java.util.Arrays; import java.util.Collection; +import java.util.List; import java.util.Set; /** @@ -28,7 +28,7 @@ public class Processing { public Processing(ModelContext.Properties properties) { this.properties = properties; } private Collection<ProcessorFactory> processors() { - return Arrays.asList( + return List.<ProcessorFactory>of( SearchMustHaveDocument::new, UrlFieldValidator::new, BuiltInFieldSets::new, @@ -99,7 +99,7 @@ public class Processing { /** Processors of rank profiles only (those who tolerate and do something useful when the search field is null) */ private Collection<ProcessorFactory> rankProfileProcessors() { - return Arrays.asList( + return List.of( RankProfileTypeSettingsProcessor::new, ReservedFunctionNames::new, RankingExpressionTypeResolver::new); diff --git a/config-model/src/main/java/com/yahoo/schema/processing/Processor.java b/config-model/src/main/java/com/yahoo/schema/processing/Processor.java index beaff13c613..dd36bbb3b61 100644 --- a/config-model/src/main/java/com/yahoo/schema/processing/Processor.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/Processor.java @@ -87,7 +87,7 @@ public abstract class Processor { implementationField.setRankType(RankType.EMPTY); implementationField.setStemming(Stemming.NONE); implementationField.getNormalizing().inferCodepoint(); - implementationField.parseIndexingScript(indexing); + implementationField.parseIndexingScript(schema.getName(), indexing); String indexName = field.getName(); String implementationIndexName = indexName + "_" + suffix; Index implementationIndex = new Index(implementationIndexName); diff --git a/config-model/src/main/java/com/yahoo/schema/processing/TextMatch.java b/config-model/src/main/java/com/yahoo/schema/processing/TextMatch.java index e6fed35b821..e29f683761f 100644 --- a/config-model/src/main/java/com/yahoo/schema/processing/TextMatch.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/TextMatch.java @@ -48,7 +48,7 @@ public class TextMatch extends Processor { if ( ! visitor.requiresTokenize) continue; ExpressionConverter converter = new MyStringTokenizer(schema, findAnnotatorConfig(schema, field)); - field.setIndexingScript((ScriptExpression)converter.convert(script)); + field.setIndexingScript(schema.getName(), (ScriptExpression)converter.convert(script)); } } diff --git a/config-model/src/main/java/com/yahoo/schema/processing/UriHack.java b/config-model/src/main/java/com/yahoo/schema/processing/UriHack.java index e4e2ac9f5be..9a4c4038953 100644 --- a/config-model/src/main/java/com/yahoo/schema/processing/UriHack.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/UriHack.java @@ -12,7 +12,6 @@ import com.yahoo.schema.document.SDField; import com.yahoo.schema.document.Stemming; import com.yahoo.vespa.model.container.search.QueryProfiles; -import java.util.Arrays; import java.util.List; /** @@ -21,7 +20,7 @@ import java.util.List; public class UriHack extends Processor { private static final List<String> URL_SUFFIX = - Arrays.asList("scheme", "host", "port", "path", "query", "fragment", "hostname"); + List.of("scheme", "host", "port", "path", "query", "fragment", "hostname"); UriHack(Schema schema, DeployLogger deployLogger, @@ -52,8 +51,7 @@ public class UriHack extends Processor { if (uriField.getDataType() instanceof ArrayDataType) { generatedType = new ArrayDataType(DataType.STRING); } - else if (uriField.getDataType() instanceof WeightedSetDataType) { - WeightedSetDataType wdt = (WeightedSetDataType) uriField.getDataType(); + else if (uriField.getDataType() instanceof WeightedSetDataType wdt) { generatedType = new WeightedSetDataType(DataType.STRING, wdt.createIfNonExistent(), wdt.removeIfZero()); } diff --git a/config-model/src/main/java/com/yahoo/schema/processing/multifieldresolver/IndexCommandResolver.java b/config-model/src/main/java/com/yahoo/schema/processing/multifieldresolver/IndexCommandResolver.java index 111f5384c41..2854824e5b7 100644 --- a/config-model/src/main/java/com/yahoo/schema/processing/multifieldresolver/IndexCommandResolver.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/multifieldresolver/IndexCommandResolver.java @@ -5,8 +5,6 @@ import com.yahoo.config.application.api.DeployLogger; import com.yahoo.schema.document.SDField; import com.yahoo.schema.Schema; -import java.util.ArrayList; -import java.util.Arrays; import java.util.List; import java.util.logging.Level; @@ -16,17 +14,10 @@ import java.util.logging.Level; public class IndexCommandResolver extends MultiFieldResolver { /** Commands which don't have to be harmonized between fields */ - private static List<String> ignoredCommands = new ArrayList<>(); + private static final List<String> ignoredCommands = List.of( "complete-boost", "literal-boost", "highlight"); /** Commands which must be harmonized between fields */ - private static List<String> harmonizedCommands = new ArrayList<>(); - - static { - String[] ignore = { "complete-boost", "literal-boost", "highlight" }; - ignoredCommands.addAll(Arrays.asList(ignore)); - String[] harmonize = { "stemming", "normalizing" }; - harmonizedCommands.addAll(Arrays.asList(harmonize)); - } + private static final List<String> harmonizedCommands = List.of("stemming", "normalizing"); public IndexCommandResolver(String indexName, List<SDField> fields, Schema schema, DeployLogger logger) { super(indexName, fields, schema, logger); @@ -53,8 +44,8 @@ public class IndexCommandResolver extends MultiFieldResolver { ", adding to field " + field.getName()); field.addQueryCommand(command); } else { - deployLogger.logApplicationPackage(Level.WARNING, "All fields going to the same index should have the same query-commands. Field \'" + field.getName() + - "\' doesn't contain command \'" + command+"\'"); + deployLogger.logApplicationPackage(Level.WARNING, "All fields going to the same index should have the same query-commands. Field '" + field.getName() + + "' doesn't contain command '" + command+"'"); } } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/HostPorts.java b/config-model/src/main/java/com/yahoo/vespa/model/HostPorts.java index f1d3b38e8ff..348a5d575eb 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/HostPorts.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/HostPorts.java @@ -4,7 +4,6 @@ package com.yahoo.vespa.model; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.provision.NetworkPorts; -import java.util.Collections; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -28,7 +27,7 @@ public class HostPorts { private int allocatedPorts = 0; - private PortFinder portFinder = new PortFinder(Collections.emptyList()); + private PortFinder portFinder = new PortFinder(List.of()); private boolean flushed = false; private Optional<NetworkPorts> networkPortsList = Optional.empty(); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/VespaModelFactory.java b/config-model/src/main/java/com/yahoo/vespa/model/VespaModelFactory.java index 1c4b0d31ab2..c876976917b 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/VespaModelFactory.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/VespaModelFactory.java @@ -36,7 +36,6 @@ import java.io.IOException; import java.time.Clock; import java.util.ArrayList; import java.util.Collection; -import java.util.Collections; import java.util.List; import java.util.Optional; import java.util.logging.Level; @@ -96,7 +95,7 @@ public class VespaModelFactory implements ModelFactory { } else { this.configModelRegistry = configModelRegistry; } - this.modelImporters = Collections.emptyList(); + this.modelImporters = List.of(); this.additionalValidators = List.of(); this.zone = zone; this.clock = clock; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainerCluster.java index 78ef6826d26..c19d186df42 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainerCluster.java @@ -10,7 +10,6 @@ import com.yahoo.vespa.model.container.ContainerCluster; import com.yahoo.vespa.model.container.PlatformBundles; import java.nio.file.Path; -import java.util.Collections; import java.util.Optional; import java.util.Set; @@ -22,7 +21,7 @@ import java.util.Set; */ public class ClusterControllerContainerCluster extends ContainerCluster<ClusterControllerContainer> { - private static final Set<Path> UNNECESSARY_BUNDLES = Collections.unmodifiableSet(PlatformBundles.VESPA_SECURITY_BUNDLES); + private static final Set<Path> UNNECESSARY_BUNDLES = Set.copyOf(PlatformBundles.VESPA_SECURITY_BUNDLES); private final ReindexingContext reindexingContext; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java index 0d85696d503..913ae0e1a00 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java @@ -36,7 +36,6 @@ import com.yahoo.vespa.model.container.component.Handler; import com.yahoo.vespa.model.container.component.SystemBindingPattern; import java.nio.file.Path; -import java.util.Collections; import java.util.LinkedHashMap; import java.util.Map; import java.util.Optional; @@ -198,7 +197,7 @@ public class MetricsProxyContainerCluster extends ContainerCluster<MetricsProxyC private Map<String, MetricsConsumer> getUserMetricsConsumers() { return getAdmin() .map(admin -> admin.getUserMetrics().getConsumers()) - .orElse(Collections.emptyMap()); + .orElse(Map.of()); } private Optional<String> getSystemName() { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/MetricsConsumer.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/MetricsConsumer.java index aa476b1ae39..0a933e7d5ed 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/MetricsConsumer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/MetricsConsumer.java @@ -7,7 +7,6 @@ import ai.vespa.metrics.set.Vespa9VespaMetricSet; import ai.vespa.metricsproxy.core.VespaMetrics; import ai.vespa.metricsproxy.http.ValuesFetcher; -import java.util.Arrays; import java.util.List; import java.util.Map; import java.util.Objects; @@ -71,7 +70,7 @@ public class MetricsConsumer { } public static MetricsConsumer consumer(String id, MetricSet ... metricSets) { - return new MetricsConsumer(id, new MetricSet(id + "-consumer-metrics", List.of(), Arrays.asList(metricSets))); + return new MetricsConsumer(id, new MetricSet(id + "-consumer-metrics", List.of(), List.of(metricSets))); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/otel/OpenTelemetryCollector.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/otel/OpenTelemetryCollector.java new file mode 100644 index 00000000000..fcd587622da --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/otel/OpenTelemetryCollector.java @@ -0,0 +1,67 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.admin.otel; + +import com.yahoo.cloud.config.OpenTelemetryConfig; +import com.yahoo.config.model.ApplicationConfigProducerRoot; +import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.config.model.producer.TreeConfigProducer; +import com.yahoo.config.provision.Zone; +import com.yahoo.vespa.model.AbstractService; +import com.yahoo.vespa.model.PortAllocBridge; + +import com.yahoo.cloud.config.ModelConfig; +import com.yahoo.config.model.producer.AnyConfigProducer; + +import java.util.Optional; + +public class OpenTelemetryCollector extends AbstractService implements OpenTelemetryConfig.Producer { + + private final Zone zone; + + public OpenTelemetryCollector(TreeConfigProducer<?> parent) { + super(parent, "otelcol"); + this.zone = null; + setProp("clustertype", "admin"); + setProp("clustername", "admin"); + } + + public OpenTelemetryCollector(TreeConfigProducer<?> parent, DeployState deployState) { + super(parent, "otelcol"); + this.zone = deployState.zone(); + setProp("clustertype", "admin"); + setProp("clustername", "admin"); + } + + /** + * @return the startup command for the otelcol wrapper + */ + @Override + public Optional<String> getStartupCommand() { + return Optional.of("exec $ROOT/bin/vespa-otelcol-start -c " + getConfigId()); + } + + @Override + public void allocatePorts(int start, PortAllocBridge from) {} + + @Override + public int getPortCount() { + return 0; + } + + @Override + public void getConfig(OpenTelemetryConfig.Builder builder) { + var generator = new OpenTelemetryConfigGenerator(zone); + AnyConfigProducer pp = this; + AnyConfigProducer p = pp.getParent(); + while (p != null && p != pp) { + if (pp instanceof ApplicationConfigProducerRoot root) { + generator.addStatePorts(root.getStatePorts()); + break; + } + pp = p; + p = pp.getParent(); + } + builder.yaml(generator.generate()); + builder.refPaths(generator.referencedPaths()); + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/otel/OpenTelemetryConfigGenerator.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/otel/OpenTelemetryConfigGenerator.java new file mode 100644 index 00000000000..36eab6a04b3 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/otel/OpenTelemetryConfigGenerator.java @@ -0,0 +1,206 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.admin.otel; + +import com.fasterxml.jackson.core.JsonEncoding; +import com.fasterxml.jackson.core.JsonFactory; +import com.fasterxml.jackson.core.JsonGenerator; +import com.yahoo.config.model.ApplicationConfigProducerRoot.StatePortInfo; +import com.yahoo.config.provision.Zone; + +import java.io.ByteArrayOutputStream; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; + +import static com.yahoo.vespa.defaults.Defaults.getDefaults; + +/** + * @author olaa + */ +public class OpenTelemetryConfigGenerator { + + private final boolean useTls; + private final String ca_file; + private final String cert_file; + private final String key_file; + private List<StatePortInfo> statePorts = new ArrayList<>(); + + OpenTelemetryConfigGenerator(Zone zone) { + boolean isCd = true; + boolean isPublic = true; + if (zone != null) { + isCd = zone.system().isCd(); + isPublic = zone.system().isPublic(); + this.useTls = true; + } else { + // for manual testing + this.useTls = false; + } + if (isCd) { + if (isPublic) { + this.ca_file = "/opt/vespa/var/vespa/trust-store.pem"; + this.cert_file = "/var/lib/sia/certs/vespa.external.cd.tenant.cert.pem"; + this.key_file = "/var/lib/sia/keys/vespa.external.cd.tenant.key.pem"; + } else { + this.ca_file = "/opt/yahoo/share/ssl/certs/athenz_certificate_bundle.pem"; + this.cert_file = "/var/lib/sia/certs/vespa.vespa.cd.tenant.cert.pem"; + this.key_file = "/var/lib/sia/keys/vespa.vespa.cd.tenant.key.pem"; + } + } else { + if (isPublic) { + this.ca_file = "/opt/vespa/var/vespa/trust-store.pem"; + this.cert_file = "/var/lib/sia/certs/vespa.external.tenant.cert.pem"; + this.key_file = "/var/lib/sia/keys/vespa.external.tenant.key.pem"; + } else { + this.ca_file = "/opt/yahoo/share/ssl/certs/athenz_certificate_bundle.pem"; + this.cert_file = "/var/lib/sia/certs/vespa.vespa.tenant.cert.pem"; + this.key_file = "/var/lib/sia/keys/vespa.vespa.tenant.key.pem"; + } + } + } + + String receiverName(int index) { + return "prometheus_simple/s" + index; + } + + private void addReceivers(JsonGenerator g) throws java.io.IOException { + g.writeFieldName("receivers"); + g.writeStartObject(); + int counter = 0; + for (var statePort : statePorts) { + addReceiver(g, ++counter, statePort); + } + g.writeEndObject(); // receivers + } + private void addReceiver(JsonGenerator g, int index, StatePortInfo statePort) throws java.io.IOException { + g.writeFieldName(receiverName(index)); + g.writeStartObject(); + g.writeStringField("collection_interval", "60s"); + g.writeStringField("endpoint", statePort.hostName() + ":" + statePort.portNumber()); + addUrlInfo(g); + if (useTls) addTls(g); + { + g.writeFieldName("labels"); + g.writeStartObject(); + g.writeStringField("service_type", statePort.serviceType()); + g.writeEndObject(); + } + g.writeEndObject(); + } + private void addTls(JsonGenerator g) throws java.io.IOException { + g.writeFieldName("tls"); + g.writeStartObject(); + g.writeStringField("ca_file", ca_file); + g.writeStringField("cert_file", cert_file); + g.writeBooleanField("insecure_skip_verify", true); + g.writeStringField("key_file", key_file); + g.writeEndObject(); // tls + } + private void addUrlInfo(JsonGenerator g) throws java.io.IOException { + g.writeStringField("metrics_path", "/state/v1/metrics"); + g.writeFieldName("params"); + g.writeStartObject(); + g.writeStringField("format", "prometheus"); + g.writeEndObject(); + } + private void addExporters(JsonGenerator g) throws java.io.IOException { + g.writeFieldName("exporters"); + g.writeStartObject(); + addFileExporter(g); + g.writeEndObject(); // exporters + } + private void addFileExporter(JsonGenerator g) throws java.io.IOException { + g.writeFieldName("file"); + g.writeStartObject(); + g.writeStringField("path", getDefaults().underVespaHome("logs/vespa/otel-test.json")); + { + g.writeFieldName("rotation"); + g.writeStartObject(); + g.writeNumberField("max_megabytes", 10); + g.writeNumberField("max_days", 3); + g.writeNumberField("max_backups", 1); + g.writeEndObject(); // rotation + } + g.writeEndObject(); // file + } + private void addServiceBlock(JsonGenerator g) throws java.io.IOException { + g.writeFieldName("service"); + g.writeStartObject(); + { + g.writeFieldName("telemetry"); + g.writeStartObject(); + { + g.writeFieldName("logs"); + g.writeStartObject(); + g.writeStringField("level", "debug"); + g.writeEndObject(); + } + g.writeEndObject(); + } + { + g.writeFieldName("pipelines"); + g.writeStartObject(); + addMetricsPipelines(g); + g.writeEndObject(); // pipelines + } + g.writeEndObject(); // service + } + private void addMetricsPipelines(JsonGenerator g) throws java.io.IOException { + g.writeFieldName("metrics"); + g.writeStartObject(); + { + g.writeFieldName("receivers"); + g.writeStartArray(); + int counter = 0; + for (var statePort : statePorts) { + g.writeString(receiverName(++counter)); + } + g.writeEndArray(); + } + g.writeFieldName("processors"); + g.writeStartArray(); + g.writeEndArray(); + { + g.writeFieldName("exporters"); + g.writeStartArray(); + g.writeString("file"); + g.writeEndArray(); + } + g.writeEndObject(); // metrics + } + + // For now - mostly dummy config + /* + TODO: Create config + 1. polling /state/v1 handler of every service (mostly done) + 2. Processing with mapping/filtering from metric sets + 3. Exporter to correct endpoint (alternatively amended) + */ + public String generate() { + if (statePorts.isEmpty()) { + return ""; + } + ByteArrayOutputStream out = new ByteArrayOutputStream(); + try { + JsonGenerator g = new JsonFactory().createGenerator(out, JsonEncoding.UTF8); + g.writeStartObject(); + addReceivers(g); + addExporters(g); + addServiceBlock(g); + g.writeEndObject(); // root + g.close(); + } catch (java.io.IOException e) { + System.err.println("unexpected error: " + e); + return ""; + } + return out.toString(StandardCharsets.UTF_8); + } + + void addStatePorts(List<StatePortInfo> portList) { + this.statePorts = portList; + } + + List<String> referencedPaths() { + return List.of(ca_file, cert_file, key_file); + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/BundleValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/BundleValidator.java index 0a23e25e432..4b4cb1530d4 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/BundleValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/BundleValidator.java @@ -5,7 +5,6 @@ import com.yahoo.config.model.deploy.DeployState; import org.w3c.dom.Document; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collection; import java.util.HashMap; import java.util.HashSet; @@ -35,7 +34,7 @@ public class BundleValidator extends AbstractBundleValidator { for (Map.Entry<Object,Object> entry : attributes.entrySet()) { mfAttributes.add(entry.getKey().toString()); } - List<String> requiredOSGIHeaders = Arrays.asList( + List<String> requiredOSGIHeaders = List.of( "Bundle-ManifestVersion", "Bundle-Name", "Bundle-SymbolicName", "Bundle-Version"); for (String header : requiredOSGIHeaders) { if (!mfAttributes.contains(header)) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/PublicApiBundleValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/PublicApiBundleValidator.java index e0a43f0988a..ddffe20f725 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/PublicApiBundleValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/PublicApiBundleValidator.java @@ -3,7 +3,7 @@ package com.yahoo.vespa.model.application.validation; import org.w3c.dom.Document; -import java.util.Arrays; +import java.util.List; import java.util.jar.JarFile; import java.util.jar.Manifest; import java.util.logging.Level; @@ -18,7 +18,7 @@ public class PublicApiBundleValidator extends AbstractBundleValidator { String nonPublicApiAttribute = mf.getMainAttributes().getValue("X-JDisc-Non-PublicApi-Import-Package"); if (nonPublicApiAttribute == null) return; - var nonPublicApisUsed = Arrays.asList(nonPublicApiAttribute.split(",")); + var nonPublicApisUsed = List.of(nonPublicApiAttribute.split(",")); if (! nonPublicApisUsed.isEmpty()) { log(context.deployState(), Level.WARNING, "Jar file '%s' uses non-public Vespa APIs: %s", filename(jar), nonPublicApisUsed); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2Builder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2Builder.java index 7f02ecacc18..692de1769d3 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2Builder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2Builder.java @@ -7,6 +7,7 @@ import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.producer.AnyConfigProducer; import com.yahoo.config.model.producer.TreeConfigProducer; import com.yahoo.text.XML; +import com.yahoo.vespa.model.HostResource; import com.yahoo.vespa.model.SimpleConfigProducer; import com.yahoo.vespa.model.admin.Admin; import com.yahoo.vespa.model.admin.Configserver; @@ -15,6 +16,8 @@ import com.yahoo.vespa.model.admin.Slobrok; import com.yahoo.vespa.model.admin.clustercontroller.ClusterControllerCluster; import com.yahoo.vespa.model.admin.clustercontroller.ClusterControllerContainer; import com.yahoo.vespa.model.admin.clustercontroller.ClusterControllerContainerCluster; +import com.yahoo.vespa.model.admin.otel.OpenTelemetryCollector; +import com.yahoo.vespa.model.admin.otel.OpenTelemetryConfigGenerator; import com.yahoo.vespa.model.builder.xml.dom.VespaDomBuilder.DomConfigProducerBuilderBase; import com.yahoo.vespa.model.container.Container; import org.w3c.dom.Element; @@ -38,10 +41,21 @@ public class DomAdminV2Builder extends DomAdminBuilderBase { super(applicationType, multitenant, configServerSpecs); } + private void addOtelcol(TreeConfigProducer<?> parent, DeployState deployState, HostResource hostResource) { + var otelcol = new OpenTelemetryCollector(parent); + otelcol.setHostResource(hostResource); + otelcol.initService(deployState); + } + @Override protected void doBuildAdmin(DeployState deployState, Admin admin, Element adminE) { List<Configserver> configservers = parseConfigservers(deployState, admin, adminE); - admin.setLogserver(parseLogserver(deployState, admin, adminE)); + var logserver = parseLogserver(deployState, admin, adminE); + admin.setLogserver(logserver); + if (deployState.featureFlags().logserverOtelCol()) { + // for manual testing + addOtelcol(admin, deployState, logserver.getHostResource()); + } admin.addConfigservers(configservers); admin.addSlobroks(getSlobroks(deployState, admin, XML.getChild(adminE, "slobroks"))); if ( ! admin.multitenant()) diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java index f722cf375f3..347bb504857 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.model.builder.xml.dom; import com.yahoo.config.model.ConfigModelContext; import com.yahoo.config.model.api.ConfigServerSpec; import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.config.model.producer.TreeConfigProducer; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.vespa.model.HostResource; import com.yahoo.vespa.model.HostSystem; @@ -12,6 +13,8 @@ import com.yahoo.vespa.model.admin.Logserver; import com.yahoo.vespa.model.admin.LogserverContainer; import com.yahoo.vespa.model.admin.LogserverContainerCluster; import com.yahoo.vespa.model.admin.Slobrok; +import com.yahoo.vespa.model.admin.otel.OpenTelemetryCollector; +import com.yahoo.vespa.model.admin.otel.OpenTelemetryConfigGenerator; import com.yahoo.vespa.model.container.Container; import com.yahoo.vespa.model.container.ContainerModel; import org.w3c.dom.Element; @@ -113,9 +116,18 @@ public class DomAdminV4Builder extends DomAdminBuilderBase { logServerCluster.addContainer(container); admin.addAndInitializeService(deployState, hostResource, container); admin.setLogserverContainerCluster(logServerCluster); + if (deployState.featureFlags().logserverOtelCol()) + addOtelcol(admin, deployState, hostResource); context.getConfigModelRepoAdder().add(logserverClusterModel); } + + private void addOtelcol(TreeConfigProducer<?> parent, DeployState deployState, HostResource hostResource) { + var otelcol = new OpenTelemetryCollector(parent, deployState); + otelcol.setHostResource(hostResource); + otelcol.initService(deployState); + } + private Collection<HostResource> allocateHosts(HostSystem hostSystem, String clusterId, NodesSpecification nodesSpecification) { return nodesSpecification.provision(hostSystem, ClusterSpec.Type.admin, diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/ComponentsBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/ComponentsBuilder.java index 046980087ed..bde12bb1d5a 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/ComponentsBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/ComponentsBuilder.java @@ -151,7 +151,7 @@ public class ComponentsBuilder<T extends ChainedComponent<?>> { private Map<String, ComponentType<?>> unmodifiable(Map<String, ComponentType<?>> outerComponentTypeByComponentName) { return (outerComponentTypeByComponentName != null)? Collections.unmodifiableMap(outerComponentTypeByComponentName): - Collections.emptyMap(); + Map.of(); } public Collection<T> getComponentDefinitions() { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/DomGenericTargetBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/DomGenericTargetBuilder.java index 565cdf3a5f5..b8de223c0cb 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/DomGenericTargetBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/DomGenericTargetBuilder.java @@ -9,7 +9,7 @@ import com.yahoo.vespa.model.container.search.searchchain.GenericTarget; import com.yahoo.vespa.model.container.search.searchchain.Searcher; import org.w3c.dom.Element; -import java.util.Arrays; +import java.util.List; import java.util.Map; /** @@ -20,7 +20,7 @@ import java.util.Map; abstract public class DomGenericTargetBuilder<T extends GenericTarget> extends DomChainBuilderBase<Searcher<?>, T> { DomGenericTargetBuilder(Map<String, ComponentsBuilder.ComponentType<?>> outerSearcherTypeByComponentName) { - super(Arrays.asList(ComponentsBuilder.ComponentType.searcher, ComponentsBuilder.ComponentType.federation), + super(List.of(ComponentsBuilder.ComponentType.searcher, ComponentsBuilder.ComponentType.federation), outerSearcherTypeByComponentName); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/DomSearchChainBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/DomSearchChainBuilder.java index 2eb1127d75b..4ada9e99e5e 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/DomSearchChainBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/DomSearchChainBuilder.java @@ -11,7 +11,7 @@ import com.yahoo.vespa.model.container.search.searchchain.SearchChain; import com.yahoo.vespa.model.container.search.searchchain.Searcher; import org.w3c.dom.Element; -import java.util.Arrays; +import java.util.List; import java.util.Map; /** @@ -21,7 +21,7 @@ import java.util.Map; public class DomSearchChainBuilder extends DomChainBuilderBase<Searcher<?>, SearchChain> { public DomSearchChainBuilder(Map<String, ComponentsBuilder.ComponentType<?>> outerSearcherTypeByComponentName) { - super(Arrays.asList(ComponentsBuilder.ComponentType.searcher, ComponentsBuilder.ComponentType.federation), + super(List.of(ComponentsBuilder.ComponentType.searcher, ComponentsBuilder.ComponentType.federation), outerSearcherTypeByComponentName); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/DomSearchChainsBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/DomSearchChainsBuilder.java index e05d762d7bc..b48df948358 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/DomSearchChainsBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/DomSearchChainsBuilder.java @@ -11,7 +11,6 @@ import com.yahoo.vespa.model.container.search.searchchain.SearchChains; import com.yahoo.vespa.model.container.search.searchchain.Searcher; import org.w3c.dom.Element; -import java.util.Arrays; import java.util.List; import java.util.Map; @@ -23,7 +22,7 @@ import java.util.Map; public class DomSearchChainsBuilder extends DomChainsBuilder<Searcher<?>, SearchChain, SearchChains> { public DomSearchChainsBuilder() { - super(Arrays.asList(ComponentType.searcher, ComponentType.federation)); + super(List.of(ComponentType.searcher, ComponentType.federation)); } @Override diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerModelEvaluation.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerModelEvaluation.java index 62979404025..5be1690f0dc 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerModelEvaluation.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerModelEvaluation.java @@ -31,6 +31,7 @@ public class ContainerModelEvaluation implements public final static String EVALUATION_BUNDLE_NAME = "model-evaluation"; public final static String INTEGRATION_BUNDLE_NAME = "model-integration"; public final static String ONNXRUNTIME_BUNDLE_NAME = "container-onnxruntime.jar"; + public final static String LLAMA_BUNDLE_NAME = "container-llama.jar"; public final static String ONNX_RUNTIME_CLASS = "ai.vespa.modelintegration.evaluator.OnnxRuntime"; private final static String EVALUATOR_NAME = ModelsEvaluator.class.getName(); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/PlatformBundles.java b/config-model/src/main/java/com/yahoo/vespa/model/container/PlatformBundles.java index baa9c4b9a48..468cf8dd961 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/PlatformBundles.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/PlatformBundles.java @@ -14,6 +14,7 @@ import java.util.stream.Stream; import static com.yahoo.vespa.model.container.ContainerModelEvaluation.EVALUATION_BUNDLE_NAME; import static com.yahoo.vespa.model.container.ContainerModelEvaluation.INTEGRATION_BUNDLE_NAME; import static com.yahoo.vespa.model.container.ContainerModelEvaluation.LINGUISTICS_BUNDLE_NAME; +import static com.yahoo.vespa.model.container.ContainerModelEvaluation.LLAMA_BUNDLE_NAME; import static com.yahoo.vespa.model.container.ContainerModelEvaluation.ONNXRUNTIME_BUNDLE_NAME; /** @@ -37,6 +38,7 @@ public class PlatformBundles { public static final Path LIBRARY_PATH = Paths.get(Defaults.getDefaults().underVespaHome("lib/jars")); public static final String SEARCH_AND_DOCPROC_BUNDLE = BundleInstantiationSpecification.CONTAINER_SEARCH_AND_DOCPROC; + public static final String MODEL_INTEGRATION_BUNDLE = BundleInstantiationSpecification.MODEL_INTEGRATION; // Bundles that must be loaded for all container types. public static final Set<Path> COMMON_VESPA_BUNDLES = toBundlePaths( @@ -63,13 +65,14 @@ public class PlatformBundles { "lucene-linguistics", EVALUATION_BUNDLE_NAME, INTEGRATION_BUNDLE_NAME, - ONNXRUNTIME_BUNDLE_NAME + ONNXRUNTIME_BUNDLE_NAME, + LLAMA_BUNDLE_NAME ); private static Set<Path> toBundlePaths(String... bundleNames) { return Stream.of(bundleNames) .map(PlatformBundles::absoluteBundlePath) - .collect(Collectors.toSet()); + .collect(Collectors.toUnmodifiableSet()); } public static Path absoluteBundlePath(String fileName) { @@ -86,6 +89,10 @@ public class PlatformBundles { return searchAndDocprocComponents.contains(className); } + public static boolean isModelIntegrationClass(String className) { + return modelIntegrationComponents.contains(className); + } + // This is a hack to allow users to declare components from the search-and-docproc bundle without naming the bundle. private static final Set<String> searchAndDocprocComponents = Set.of( com.yahoo.docproc.AbstractConcreteDocumentFactory.class.getName(), @@ -147,8 +154,13 @@ public class PlatformBundles { com.yahoo.vespa.streamingvisitors.MetricsSearcher.class.getName(), com.yahoo.vespa.streamingvisitors.StreamingBackend.class.getName(), ai.vespa.search.llm.LLMSearcher.class.getName(), - ai.vespa.search.llm.RAGSearcher.class.getName(), - ai.vespa.llm.clients.OpenAI.class.getName() + ai.vespa.search.llm.RAGSearcher.class.getName() + ); + + // This is a hack to allow users to declare components from the model-integration bundle without naming the bundle. + private static final Set<String> modelIntegrationComponents = Set.of( + ai.vespa.llm.clients.OpenAI.class.getName(), + ai.vespa.llm.clients.LocalLLM.class.getName() ); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/component/DiscBindingsConfigGenerator.java b/config-model/src/main/java/com/yahoo/vespa/model/container/component/DiscBindingsConfigGenerator.java index ed626f98aa0..dafac2c1af9 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/component/DiscBindingsConfigGenerator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/component/DiscBindingsConfigGenerator.java @@ -2,7 +2,6 @@ package com.yahoo.vespa.model.container.component; import java.util.Collection; -import java.util.Collections; import java.util.LinkedHashMap; import java.util.Map; @@ -24,9 +23,9 @@ public class DiscBindingsConfigGenerator { public static <T extends Handler> Map<String, Handlers.Builder> generate(T handler) { if (handler.getServerBindings().isEmpty() && handler.getClientBindings().isEmpty()) - return Collections.emptyMap(); + return Map.of(); - return Collections.singletonMap(handler.model.getComponentId().stringValue(), + return Map.of(handler.model.getComponentId().stringValue(), new Handlers.Builder() .serverBindings(toStrings(handler.getServerBindings())) .clientBindings(toStrings(handler.getClientBindings()))); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/component/Handler.java b/config-model/src/main/java/com/yahoo/vespa/model/container/component/Handler.java index 099255975b6..0d3d8462376 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/component/Handler.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/component/Handler.java @@ -8,7 +8,6 @@ import com.yahoo.vespa.model.container.ContainerThreadpool; import org.w3c.dom.Element; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.LinkedHashSet; @@ -51,7 +50,7 @@ public class Handler extends Component<Component<?, ?>, ComponentModel> { } public void addServerBindings(BindingPattern... bindings) { - serverBindings.addAll(Arrays.asList(bindings)); + serverBindings.addAll(List.of(bindings)); } public void addServerBindings(Collection<BindingPattern> bps) { serverBindings.addAll(bps); } @@ -61,7 +60,7 @@ public class Handler extends Component<Component<?, ?>, ComponentModel> { } public void addClientBindings(BindingPattern... bindings) { - clientBindings.addAll(Arrays.asList(bindings)); + clientBindings.addAll(List.of(bindings)); } public final Collection<BindingPattern> getServerBindings() { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/component/Model.java b/config-model/src/main/java/com/yahoo/vespa/model/container/component/Model.java index 7d6285d00c1..3a8e52c92a8 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/component/Model.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/component/Model.java @@ -13,6 +13,7 @@ import com.yahoo.vespa.model.container.xml.ModelIdResolver; import org.w3c.dom.Element; import java.net.URI; +import java.net.URISyntaxException; import java.util.Objects; import java.util.Optional; import java.util.Set; @@ -51,11 +52,22 @@ class Model { static Model fromXml(DeployState ds, Element model, Set<String> requiredTags) { var modelId = XmlHelper.getOptionalAttribute(model, "model-id").orElse(null); - var url = XmlHelper.getOptionalAttribute(model, "url").map(URI::create).orElse(null); + var url = XmlHelper.getOptionalAttribute(model, "url").map(Model::parseUrl).orElse(null); var path = XmlHelper.getOptionalAttribute(model, "path").map(Path::fromString).orElse(null); return new Model(ds, model.getTagName(), modelId, url, path, requiredTags); } + private static URI parseUrl(String url) { + try { + var uri = new URI(url); + if ( ! uri.isAbsolute()) + throw new IllegalArgumentException("Invalid url '" + url + "': url has no 'scheme' component"); + return uri; + } catch (URISyntaxException e) { + throw new IllegalArgumentException("Invalid url '" + url + "':" + e.getMessage()); + } + } + /** Return tokenizer model from XML if specified, alternatively use model id for ONNX model with suffix '-vocab' appended */ static Model fromXmlOrImplicitlyFromOnnxModel( DeployState ds, Element parent, Model onnxModel, String paramName, Set<String> requiredTags) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/AccessControl.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/AccessControl.java index 8c1b868465a..19c45bba2ef 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/http/AccessControl.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/AccessControl.java @@ -48,7 +48,7 @@ public class AccessControl { private final String domain; private ClientAuthentication clientAuthentication = ClientAuthentication.need; private final Set<BindingPattern> excludeBindings = new LinkedHashSet<>(); - private Collection<Handler> handlers = Collections.emptyList(); + private Collection<Handler> handlers = List.of(); public Builder(String domain) { this.domain = domain; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/FilterChainBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/FilterChainBuilder.java index bc234546625..82a8a01871e 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/FilterChainBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/FilterChainBuilder.java @@ -11,7 +11,7 @@ import com.yahoo.vespa.model.container.http.HttpFilterChain; import org.w3c.dom.Element; import java.util.Collection; -import java.util.Collections; +import java.util.List; import java.util.Map; import static com.yahoo.vespa.model.builder.xml.dom.chains.ComponentsBuilder.ComponentType; @@ -21,7 +21,7 @@ import static com.yahoo.vespa.model.builder.xml.dom.chains.ComponentsBuilder.Com */ public class FilterChainBuilder extends DomChainBuilderBase<Filter, HttpFilterChain> { - private static final Collection<ComponentType<Filter>> allowedComponentTypes = Collections.singleton(ComponentType.filter); + private static final Collection<ComponentType<Filter>> allowedComponentTypes = List.of(ComponentType.filter); public FilterChainBuilder(Map<String, ComponentType<?>> outerFilterTypeByComponentName) { super(allowedComponentTypes, outerFilterTypeByComponentName); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/FilterChainsBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/FilterChainsBuilder.java index 878859a01bf..f1cdbfb3b04 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/FilterChainsBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/FilterChainsBuilder.java @@ -15,7 +15,6 @@ import com.yahoo.vespa.model.container.http.HttpFilterChain; import org.w3c.dom.Element; import java.util.Collection; -import java.util.Collections; import java.util.List; import java.util.Map; @@ -24,7 +23,7 @@ import java.util.Map; */ public class FilterChainsBuilder extends DomChainsBuilder<Filter, HttpFilterChain, FilterChains> { private static final Collection<ComponentType<Filter>> allowedComponentTypes = - Collections.singleton(ComponentType.filter); + List.of(ComponentType.filter); private static final Map<String, Class<? extends DomChainBuilderBase<? extends Filter, ? extends HttpFilterChain>>> chainType2BuilderClass = Map.of( diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/Source.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/Source.java index 29e587c6453..f589884cccd 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/Source.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/Source.java @@ -4,9 +4,8 @@ package com.yahoo.vespa.model.container.search.searchchain; import com.yahoo.component.ComponentId; import com.yahoo.component.chain.model.ChainSpecification; import com.yahoo.search.searchchain.model.federation.FederationOptions; -import com.yahoo.config.model.producer.TreeConfigProducer; -import java.util.Arrays; +import java.util.List; /** @@ -51,7 +50,7 @@ public class Source extends GenericTarget { @Override public ChainSpecification getChainSpecification() { return super.getChainSpecification().addInherits( - Arrays.asList(getParentProvider().getComponentId().toSpecification())); + List.of(getParentProvider().getComponentId().toSpecification())); } public ComponentId getGlobalComponentId() { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/BundleInstantiationSpecificationBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/BundleInstantiationSpecificationBuilder.java index 8a15e61495b..1323506eaeb 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/BundleInstantiationSpecificationBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/BundleInstantiationSpecificationBuilder.java @@ -7,7 +7,6 @@ import com.yahoo.component.ComponentSpecification; import com.yahoo.vespa.model.container.PlatformBundles; import org.w3c.dom.Element; -import java.util.Arrays; import java.util.List; import static com.yahoo.vespa.model.container.component.chain.ProcessingHandler.PROCESSING_HANDLER_CLASS; @@ -27,21 +26,20 @@ public class BundleInstantiationSpecificationBuilder { BundleInstantiationSpecification instSpec = new BundleInstantiationSpecification(id, classId, bundle); validate(instSpec); - return bundle == null ? setBundleForSearchAndDocprocComponents(instSpec) : instSpec; + return bundle == null ? setBundleForComponent(instSpec) : instSpec; } - private static BundleInstantiationSpecification setBundleForSearchAndDocprocComponents(BundleInstantiationSpecification spec) { + private static BundleInstantiationSpecification setBundleForComponent(BundleInstantiationSpecification spec) { if (PlatformBundles.isSearchAndDocprocClass(spec.getClassName())) return spec.inBundle(PlatformBundles.SEARCH_AND_DOCPROC_BUNDLE); + else if (PlatformBundles.isModelIntegrationClass(spec.getClassName())) + return spec.inBundle(PlatformBundles.MODEL_INTEGRATION_BUNDLE); else return spec; } - private static void validate(BundleInstantiationSpecification instSpec) { - List<String> forbiddenClasses = Arrays.asList( - SearchHandler.HANDLER_CLASSNAME, - PROCESSING_HANDLER_CLASS); + List<String> forbiddenClasses = List.of(SearchHandler.HANDLER_CLASSNAME, PROCESSING_HANDLER_CLASS); for (String forbiddenClass: forbiddenClasses) { if (forbiddenClass.equals(instSpec.getClassName())) { @@ -50,7 +48,7 @@ public class BundleInstantiationSpecificationBuilder { } } - //null if missing + // null if missing private static ComponentSpecification getComponentSpecification(Element spec, String attributeName) { return (spec.hasAttribute(attributeName)) ? new ComponentSpecification(spec.getAttribute(attributeName)) : diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java index 456ce8fdbc2..c6fca8d32c6 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java @@ -114,7 +114,6 @@ import java.security.cert.X509Certificate; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; -import java.util.Collections; import java.util.LinkedHashMap; import java.util.LinkedHashSet; import java.util.List; @@ -507,6 +506,13 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { boolean atLeastOneClientWithCertificate = clients.stream().anyMatch(client -> !client.certificates().isEmpty()); if (!atLeastOneClientWithCertificate) throw new IllegalArgumentException("At least one client must require a certificate"); + + List<String> duplicates = clients.stream().collect(Collectors.groupingBy(Client::id)) + .entrySet().stream().filter(entry -> entry.getValue().size() > 1) + .map(Map.Entry::getKey).sorted().toList(); + if (! duplicates.isEmpty()) { + throw new IllegalArgumentException("Duplicate client ids: " + duplicates); + } } List<X509Certificate> operatorAndTesterCertificates = deployState.getProperties().operatorCertificates(); @@ -893,7 +899,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { private void addStandaloneNode(ApplicationContainerCluster cluster, DeployState deployState) { ApplicationContainer container = new ApplicationContainer(cluster, "standalone", cluster.getContainers().size(), deployState); - cluster.addContainers(Collections.singleton(container)); + cluster.addContainers(List.of(container)); } private static String buildJvmGCOptions(ConfigModelContext context, String jvmGCOptions) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ModelIdResolver.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ModelIdResolver.java index f9993b770e5..867ac86f8d5 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ModelIdResolver.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ModelIdResolver.java @@ -26,6 +26,7 @@ public class ModelIdResolver { public static final String ONNX_MODEL = "onnx-model"; public static final String BERT_VOCAB = "bert-vocabulary"; public static final String SIGNIFICANCE_MODEL = "significance-model"; + public static final String GGUF_MODEL = "gguf-model"; private static Map<String, ProvidedModel> setupProvidedModels() { var m = new HashMap<String, ProvidedModel>(); @@ -60,6 +61,9 @@ public class ModelIdResolver { register(m, "e5-large-v2", "https://data.vespa.oath.cloud/onnx_models/e5-large-v2/model.onnx", Set.of(ONNX_MODEL)); register(m, "e5-large-v2-vocab", "https://data.vespa.oath.cloud/onnx_models/e5-large-v2/tokenizer.json", Set.of(HF_TOKENIZER)); + + register(m, "mistral-7b", "https://data.vespa.oath.cloud/gguf_models/mistral-7b-instruct-v0.1.Q6_K.gguf", Set.of(GGUF_MODEL)); + register(m, "mistral-7b-q8", "https://data.vespa.oath.cloud/gguf_models/mistral-7b-instruct-v0.1.Q8_0.gguf", Set.of(GGUF_MODEL)); return Map.copyOf(m); } @@ -124,7 +128,7 @@ public class ModelIdResolver { throw new IllegalArgumentException("Unknown model id '" + modelId + "' on '" + valueName + "'. Available models are [" + providedModels.keySet().stream().sorted().collect(Collectors.joining(", ")) + "]"); var providedModel = providedModels.get(modelId); - if (!providedModel.tags().containsAll(requiredTags)) { + if ( ! providedModel.tags().containsAll(requiredTags)) { throw new IllegalArgumentException( "Model '%s' on '%s' has tags %s but are missing required tags %s" .formatted(modelId, valueName, providedModel.tags(), requiredTags)); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/Content.java b/config-model/src/main/java/com/yahoo/vespa/model/content/Content.java index 9d15eddbef3..6d482a6d4a6 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/Content.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/Content.java @@ -35,7 +35,6 @@ import org.w3c.dom.Element; import java.util.ArrayList; import java.util.Collection; -import java.util.Collections; import java.util.LinkedHashSet; import java.util.List; import java.util.Optional; @@ -195,7 +194,7 @@ public class Content extends ConfigModel { public static class Builder extends ConfigModelBuilder<Content> { - public static final List<ConfigModelId> configModelIds = Collections.singletonList(ConfigModelId.fromName("content")); + public static final List<ConfigModelId> configModelIds = List.of(ConfigModelId.fromName("content")); public Builder() { super(Content.class); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/ReservedDocumentTypeNameValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/content/ReservedDocumentTypeNameValidator.java index 53547a1a15b..6dd7278fde8 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/ReservedDocumentTypeNameValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/ReservedDocumentTypeNameValidator.java @@ -3,9 +3,6 @@ package com.yahoo.vespa.model.content; import com.yahoo.documentmodel.NewDocumentType; -import java.util.Arrays; -import java.util.Collections; -import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; @@ -13,9 +10,8 @@ import java.util.stream.Collectors; public class ReservedDocumentTypeNameValidator { - public static final List<String> ORDERED_RESERVED_NAMES = Collections.unmodifiableList( - Arrays.asList("and", "false", "id", "not", "null", "or", "true")); - public static final Set<String> RESERVED_NAMES = Collections.unmodifiableSet(new HashSet<>(ORDERED_RESERVED_NAMES)); + public static final List<String> ORDERED_RESERVED_NAMES = List.of("and", "false", "id", "not", "null", "or", "true"); + public static final Set<String> RESERVED_NAMES = Set.copyOf(ORDERED_RESERVED_NAMES); public void validate(Map<String, NewDocumentType> documentDefinitions) { List<String> conflictingNames = documentDefinitions.keySet().stream() diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/StorageGroup.java b/config-model/src/main/java/com/yahoo/vespa/model/content/StorageGroup.java index a6f53777c51..a1b9ed67aa7 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/StorageGroup.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/StorageGroup.java @@ -19,7 +19,6 @@ import com.yahoo.vespa.model.content.engines.PersistenceEngine; import java.util.ArrayList; import java.util.Collection; -import java.util.Collections; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -349,7 +348,7 @@ public class StorageGroup { owner.getStorageCluster().getClusterName(), owner.getRoot().hostSystem(), context) : - Collections.emptyMap(); + Map.of(); Map<Optional<ClusterSpec.Group>, Map<HostResource, ClusterMembership>> hostGroups = collectAllocatedSubgroups(hostMapping); if (hostGroups.size() > 1) { @@ -475,7 +474,7 @@ public class StorageGroup { } private List<XmlNodeBuilder> collectExplicitNodes(Optional<ModelElement> groupOrNodesElement) { - if (groupOrNodesElement.isEmpty()) return Collections.emptyList(); + if (groupOrNodesElement.isEmpty()) return List.of(); List<XmlNodeBuilder> nodes = new ArrayList<>(); for (ModelElement n : groupOrNodesElement.get().subElements("node")) nodes.add(new XmlNodeBuilder(clusterElement, n)); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/GlobalDistributionBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/GlobalDistributionBuilder.java index c9b6537434b..07dafda164d 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/GlobalDistributionBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/GlobalDistributionBuilder.java @@ -4,7 +4,6 @@ package com.yahoo.vespa.model.content.cluster; import com.yahoo.documentmodel.NewDocumentType; import com.yahoo.vespa.model.builder.xml.dom.ModelElement; -import java.util.Collections; import java.util.LinkedHashSet; import java.util.Map; import java.util.Set; @@ -20,19 +19,19 @@ public class GlobalDistributionBuilder { private final Map<String, NewDocumentType> documentDefinitions; public GlobalDistributionBuilder(Map<String, NewDocumentType> documentDefinitions) { - this.documentDefinitions = Collections.unmodifiableMap(documentDefinitions); + this.documentDefinitions = Map.copyOf(documentDefinitions); } public Set<NewDocumentType> build(ModelElement documentsElement) { if (documentsElement == null || documentsElement.subElements("document").isEmpty()) - return Collections.emptySet(); + return Set.of(); return documentsElement.subElements("document") .stream() .filter(GlobalDistributionBuilder::isGloballyDistributed) .map(GlobalDistributionBuilder::getDocumentName) .map(this::getDocumentType) - .collect(Collectors.toCollection(() -> new LinkedHashSet<>())); + .collect(Collectors.toCollection(LinkedHashSet::new)); } private static boolean isGloballyDistributed(ModelElement e) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/FileStorProducer.java b/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/FileStorProducer.java index 6fd917c393d..ae84978793a 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/FileStorProducer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/FileStorProducer.java @@ -47,6 +47,7 @@ public class FileStorProducer implements StorFilestorConfig.Producer { private final int responseNumThreads; private final StorFilestorConfig.Response_sequencer_type.Enum responseSequencerType; private final boolean useAsyncMessageHandlingOnSchedule; + private final int persistenceThreadMaxFeedOpBatchSize; private static StorFilestorConfig.Response_sequencer_type.Enum convertResponseSequencerType(String sequencerType) { try { @@ -62,6 +63,7 @@ public class FileStorProducer implements StorFilestorConfig.Producer { this.responseNumThreads = featureFlags.defaultNumResponseThreads(); this.responseSequencerType = convertResponseSequencerType(featureFlags.responseSequencerType()); this.useAsyncMessageHandlingOnSchedule = featureFlags.useAsyncMessageHandlingOnSchedule(); + this.persistenceThreadMaxFeedOpBatchSize = featureFlags.persistenceThreadMaxFeedOpBatchSize(); } @@ -75,6 +77,7 @@ public class FileStorProducer implements StorFilestorConfig.Producer { builder.use_async_message_handling_on_schedule(useAsyncMessageHandlingOnSchedule); var throttleBuilder = new StorFilestorConfig.Async_operation_throttler.Builder(); builder.async_operation_throttler(throttleBuilder); + builder.max_feed_op_batch_size(persistenceThreadMaxFeedOpBatchSize); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/ml/ConvertedModel.java b/config-model/src/main/java/com/yahoo/vespa/model/ml/ConvertedModel.java index 621377e0606..78ed10a583e 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/ml/ConvertedModel.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/ml/ConvertedModel.java @@ -32,7 +32,6 @@ import java.io.StringReader; import java.io.UncheckedIOException; import java.nio.file.Files; import java.util.ArrayList; -import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.LinkedHashMap; @@ -177,7 +176,7 @@ public class ConvertedModel { if (arguments.output().isEmpty()) { List<Map.Entry<String, ExpressionFunction>> entriesWithTheRightPrefix = expressions.entrySet().stream().filter(entry -> entry.getKey().startsWith(arguments.signature().get() + ".")).toList(); - if (entriesWithTheRightPrefix.size() < 1) + if (entriesWithTheRightPrefix.isEmpty()) throw new IllegalArgumentException("No expressions named '" + arguments.signature().get() + missingExpressionMessageSuffix()); if (entriesWithTheRightPrefix.size() > 1) @@ -191,7 +190,7 @@ public class ConvertedModel { private String missingExpressionMessageSuffix() { return "' in model '" + modelDescription + "'. " + - "Available expressions: " + expressions.keySet().stream().collect(Collectors.joining(", ")); + "Available expressions: " + String.join(", ", expressions.keySet()); } // ----------------------- Static model conversion/storage below here @@ -425,8 +424,7 @@ public class ConvertedModel { } private static void addFunctionNamesIn(ExpressionNode node, Set<String> names, ImportedMlModel model) { - if (node instanceof ReferenceNode) { - ReferenceNode referenceNode = (ReferenceNode)node; + if (node instanceof ReferenceNode referenceNode) { if (referenceNode.getOutput() == null) { // function references cannot specify outputs if (names.add(referenceNode.getName())) { if (model.functions().containsKey(referenceNode.getName())) { @@ -485,7 +483,7 @@ public class ConvertedModel { List<Pair<String, ExpressionFunction>> readExpressions() { List<Pair<String, ExpressionFunction>> expressions = new ArrayList<>(); ApplicationFile expressionPath = application.getFile(modelFiles.expressionsPath()); - if ( ! expressionPath.exists() || ! expressionPath.isDirectory()) return Collections.emptyList(); + if ( ! expressionPath.exists() || ! expressionPath.isDirectory()) return List.of(); for (ApplicationFile expressionFile : expressionPath.listFiles()) { try (BufferedReader reader = new BufferedReader(expressionFile.createReader())) { String name = expressionFile.getPath().getName(); @@ -525,7 +523,7 @@ public class ConvertedModel { List<Pair<String, RankingExpression>> readFunctions() { try { ApplicationFile file = application.getFile(modelFiles.functionsPath()); - if ( ! file.exists()) return Collections.emptyList(); + if ( ! file.exists()) return List.of(); List<Pair<String, RankingExpression>> functions = new ArrayList<>(); try (BufferedReader reader = new BufferedReader(file.createReader())) { @@ -597,7 +595,7 @@ public class ConvertedModel { private List<Pair<String, Tensor>> readSmallConstants() { try { ApplicationFile file = application.getFile(modelFiles.smallConstantsPath()); - if ( ! file.exists()) return Collections.emptyList(); + if ( ! file.exists()) return List.of(); List<Pair<String, Tensor>> constants = new ArrayList<>(); BufferedReader reader = new BufferedReader(file.createReader()); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/ml/OnnxModelInfo.java b/config-model/src/main/java/com/yahoo/vespa/model/ml/OnnxModelInfo.java index 502924ed31a..f47a48b09d7 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/ml/OnnxModelInfo.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/ml/OnnxModelInfo.java @@ -17,7 +17,6 @@ import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; import java.util.ArrayList; -import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -51,8 +50,8 @@ public class OnnxModelInfo { Map<String, OnnxTypeInfo> outputs, Set<String> initializers, String defaultOutput) { this.app = app; this.modelPath = path; - this.inputs = Collections.unmodifiableMap(inputs); - this.outputs = Collections.unmodifiableMap(outputs); + this.inputs = Map.copyOf(inputs); + this.outputs = Map.copyOf(outputs); this.defaultOutput = defaultOutput; this.initializers = Set.copyOf(initializers); } |