diff options
Diffstat (limited to 'config-model/src/main/java/com/yahoo')
81 files changed, 823 insertions, 520 deletions
diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java index 3da85a83119..400aea86834 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java @@ -78,6 +78,13 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea private boolean enableBitVectors = false; private boolean loadCodeAsHugePages = false; private boolean sharedStringRepoNoReclaim = false; + private int mbus_java_num_targets = 1; + private int mbus_java_events_before_wakeup = 1; + private int mbus_cpp_num_targets = 1; + private int mbus_cpp_events_before_wakeup = 1; + private int rpc_num_targets = 1; + private int rpc_events_before_wakeup = 1; + private int mbus_network_threads = 1; private Architecture adminClusterNodeResourcesArchitecture = Architecture.getDefault(); @Override public ModelContext.FeatureFlags featureFlags() { return this; } @@ -101,9 +108,6 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea @Override public Optional<AthenzDomain> athenzDomain() { return Optional.ofNullable(athenzDomain); } @Override public String responseSequencerType() { return responseSequencerType; } @Override public int defaultNumResponseThreads() { return responseNumThreads; } - @Override public boolean skipCommunicationManagerThread() { return false; } - @Override public boolean skipMbusRequestThread() { return false; } - @Override public boolean skipMbusReplyThread() { return false; } @Override public Quota quota() { return quota; } @Override public boolean useAsyncMessageHandlingOnSchedule() { return useAsyncMessageHandlingOnSchedule; } @Override public double feedConcurrency() { return feedConcurrency; } @@ -136,6 +140,14 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea @Override public Architecture adminClusterArchitecture() { return adminClusterNodeResourcesArchitecture; } @Override public boolean sharedStringRepoNoReclaim() { return sharedStringRepoNoReclaim; } @Override public boolean loadCodeAsHugePages() { return loadCodeAsHugePages; } + @Override public int mbusNetworkThreads() { return mbus_network_threads; } + @Override public int mbusJavaRpcNumTargets() { return mbus_java_num_targets; } + @Override public int mbusJavaEventsBeforeWakeup() { return mbus_java_events_before_wakeup; } + @Override public int mbusCppRpcNumTargets() { return mbus_cpp_num_targets; } + @Override public int mbusCppEventsBeforeWakeup() { return mbus_cpp_events_before_wakeup; } + @Override public int rpcNumTargets() { return rpc_num_targets; } + @Override public int rpcEventsBeforeWakeup() { return rpc_events_before_wakeup; } + public TestProperties sharedStringRepoNoReclaim(boolean sharedStringRepoNoReclaim) { this.sharedStringRepoNoReclaim = sharedStringRepoNoReclaim; @@ -361,6 +373,35 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea return this; } + public TestProperties setMbusNetworkThreads(int value) { + this.mbus_network_threads = value; + return this; + } + public TestProperties setMbusJavaRpcNumTargets(int value) { + this.mbus_java_num_targets = value; + return this; + } + public TestProperties setMbusJavaEventsBeforeWakeup(int value) { + this.mbus_java_events_before_wakeup = value; + return this; + } + public TestProperties setMbusCppEventsBeforeWakeup(int value) { + this.mbus_cpp_events_before_wakeup = value; + return this; + } + public TestProperties setMbusCppRpcNumTargets(int value) { + this.mbus_cpp_num_targets = value; + return this; + } + public TestProperties setRpcNumTargets(int value) { + this.rpc_num_targets = value; + return this; + } + public TestProperties setRpcEventsBeforeWakeup(int value) { + this.rpc_events_before_wakeup = value; + return this; + } + public TestProperties setAdminClusterNodeResourcesArchitecture(Architecture architecture) { this.adminClusterNodeResourcesArchitecture = architecture; return this; @@ -386,12 +427,11 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea @Override public boolean equals(Object o) { - if (o instanceof ConfigServerSpec) { - ConfigServerSpec other = (ConfigServerSpec)o; + if (o instanceof ConfigServerSpec rhsSpec) { - return hostName.equals(other.getHostName()) && - configServerPort == other.getConfigServerPort() && - zooKeeperPort == other.getZooKeeperPort(); + return hostName.equals(rhsSpec.getHostName()) && + configServerPort == rhsSpec.getConfigServerPort() && + zooKeeperPort == rhsSpec.getZooKeeperPort(); } else { return false; } diff --git a/config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java b/config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java index 102023ba1fd..1c93ce4bd6d 100644 --- a/config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java +++ b/config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java @@ -237,7 +237,7 @@ public class InMemoryProvisioner implements HostProvisioner { .findFirst(); if (hostResources.isEmpty()) { if (canFail) - throw new IllegalArgumentException("Insufficient capacity for " + requestedResources); + throw new IllegalArgumentException("Insufficient capacity for " + requestedResources + " in cluster " + clusterGroup); else break; // ¯\_(ツ)_/¯ } diff --git a/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java b/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java index abd7ba3be9f..9ee279c68d3 100644 --- a/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java +++ b/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java @@ -1,22 +1,22 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.config.model.test; +import com.yahoo.component.Version; +import com.yahoo.config.application.api.ApplicationFile; import com.yahoo.config.application.api.ApplicationMetaData; +import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.application.api.ComponentInfo; import com.yahoo.config.application.api.UnparsedConfigDefinition; -import com.yahoo.config.application.api.ApplicationFile; -import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.TenantName; import com.yahoo.io.IOUtils; -import com.yahoo.path.Path; import com.yahoo.io.reader.NamedReader; +import com.yahoo.path.Path; import com.yahoo.search.query.profile.QueryProfileRegistry; import com.yahoo.search.query.profile.config.QueryProfileXMLReader; import com.yahoo.vespa.config.ConfigDefinitionKey; -import com.yahoo.config.application.api.ApplicationPackage; import java.io.BufferedInputStream; import java.io.File; @@ -46,7 +46,6 @@ import java.util.stream.Collectors; */ public class MockApplicationPackage implements ApplicationPackage { - public static final String DEPLOYED_BY_USER = "user"; public static final String APPLICATION_NAME = "application"; public static final long APPLICATION_GENERATION = 1L; public static final String MUSIC_SCHEMA = createSchema("music", "foo"); @@ -80,8 +79,7 @@ public class MockApplicationPackage implements ApplicationPackage { this.failOnValidateXml = failOnValidateXml; queryProfileRegistry = new QueryProfileXMLReader().read(asNamedReaderList(queryProfileType), asNamedReaderList(queryProfile)); - applicationMetaData = new ApplicationMetaData(DEPLOYED_BY_USER, - "dir", + applicationMetaData = new ApplicationMetaData("dir", 0L, false, ApplicationId.from(TenantName.defaultName(), @@ -165,7 +163,7 @@ public class MockApplicationPackage implements ApplicationPackage { @Override public ApplicationFile getFile(Path file) { if (files.containsKey(file)) return files.get(file); - return new MockApplicationFile(file, Path.fromString(root.toString())); + return new MockApplicationFile(file, root); } @Override @@ -288,7 +286,7 @@ public class MockApplicationPackage implements ApplicationPackage { Map<Path, MockApplicationFile> mockFiles = new HashMap<>(); for (var file : files.entrySet()) mockFiles.put(file.getKey(), new MockApplicationFile(file.getKey(), - Path.fromString(root.toString()), file.getValue())); + root, file.getValue())); this.files = mockFiles; return this; } @@ -374,8 +372,8 @@ public class MockApplicationPackage implements ApplicationPackage { public static class MockApplicationFile extends ApplicationFile { - /** The path to the application package root */ - private final Path root; + /** The application package root */ + private final File root; /** The File pointing to the actual file represented by this */ private final File file; @@ -383,14 +381,14 @@ public class MockApplicationPackage implements ApplicationPackage { /** The content of this file, or null to read it from the file system. */ private final String content; - public MockApplicationFile(Path filePath, Path applicationPackagePath) { - this(filePath, applicationPackagePath, null); + public MockApplicationFile(Path relativeFile, File root) { + this(relativeFile, root, null); } - private MockApplicationFile(Path filePath, Path applicationPackagePath, String content) { - super(filePath); - this.root = applicationPackagePath; - file = applicationPackagePath.append(filePath).toFile(); + private MockApplicationFile(Path relativeFile, File root, String content) { + super(relativeFile); + this.root = root; + this.file = root.toPath().resolve(relativeFile.toString()).toFile(); this.content = content; } @@ -491,7 +489,7 @@ public class MockApplicationPackage implements ApplicationPackage { Iterator<String> pathIterator = path.iterator(); // Skip the path elements this shares with the root - for (Iterator<String> rootIterator = root.iterator(); rootIterator.hasNext(); ) { + for (Iterator<String> rootIterator = Path.fromString(root.toString()).iterator(); rootIterator.hasNext(); ) { String rootElement = rootIterator.next(); String pathElement = pathIterator.next(); if ( ! rootElement.equals(pathElement)) throw new RuntimeException("Assumption broken"); diff --git a/config-model/src/main/java/com/yahoo/config/model/test/MockRoot.java b/config-model/src/main/java/com/yahoo/config/model/test/MockRoot.java index 7e2953e6606..74a5e92d2ba 100644 --- a/config-model/src/main/java/com/yahoo/config/model/test/MockRoot.java +++ b/config-model/src/main/java/com/yahoo/config/model/test/MockRoot.java @@ -61,7 +61,7 @@ public class MockRoot extends AbstractConfigProducerRoot { public MockRoot(String rootConfigId, DeployState deployState) { super(rootConfigId); - hostSystem = new HostSystem(this, "hostsystem", deployState.getProvisioner(), deployState.getDeployLogger()); + hostSystem = new HostSystem(this, "hostsystem", deployState.getProvisioner(), deployState.getDeployLogger(), deployState.isHosted()); this.deployState = deployState; fileReferencesRepository = new FileReferencesRepository(deployState.getFileRegistry()); } diff --git a/config-model/src/main/java/com/yahoo/schema/ApplicationBuilder.java b/config-model/src/main/java/com/yahoo/schema/ApplicationBuilder.java index d98709569b1..894415091e4 100644 --- a/config-model/src/main/java/com/yahoo/schema/ApplicationBuilder.java +++ b/config-model/src/main/java/com/yahoo/schema/ApplicationBuilder.java @@ -107,7 +107,7 @@ public class ApplicationBuilder { this(MockApplicationPackage.createEmpty(), new MockFileRegistry(), new BaseDeployLogger(), properties, rankProfileRegistry, queryProfileRegistry); } - /** normal constructor */ + /** Regular constructor */ public ApplicationBuilder(ApplicationPackage app, FileRegistry fileRegistry, DeployLogger deployLogger, diff --git a/config-model/src/main/java/com/yahoo/schema/derived/IndexInfo.java b/config-model/src/main/java/com/yahoo/schema/derived/IndexInfo.java index 4887ad52974..277858bed26 100644 --- a/config-model/src/main/java/com/yahoo/schema/derived/IndexInfo.java +++ b/config-model/src/main/java/com/yahoo/schema/derived/IndexInfo.java @@ -178,13 +178,11 @@ public class IndexInfo extends Derived implements IndexInfoConfig.Producer { PrimitiveDataType primitive = dataType.getPrimitiveType(); if (primitive == PrimitiveDataType.STRING) return true; if (primitive != null) return false; - if (dataType instanceof StructuredDataType) { - StructuredDataType structured = (StructuredDataType) dataType; + if (dataType instanceof StructuredDataType structured) { for (Field field : structured.getFields()) { if (isAnyChildString(field.getDataType())) return true; } - } else if (dataType instanceof MapDataType) { - MapDataType mapType = (MapDataType) dataType; + } else if (dataType instanceof MapDataType mapType) { return isAnyChildString(mapType.getKeyType()) || isAnyChildString(mapType.getValueType()); } return false; @@ -494,9 +492,9 @@ public class IndexInfo extends Derived implements IndexInfoConfig.Producer { */ public static class IndexCommand { - private String index; + private final String index; - private String command; + private final String command; public IndexCommand(String index, String command) { this.index = index; @@ -523,14 +521,12 @@ public class IndexInfo extends Derived implements IndexInfoConfig.Producer { } public boolean equals(Object object) { - if (!(object instanceof IndexCommand)) { + if (!(object instanceof IndexCommand other)) { return false; } - IndexCommand other = (IndexCommand)object; - return - other.index.equals(this.index) && - other.command.equals(this.command); + return other.index.equals(this.index) && + other.command.equals(this.command); } public String toString() { @@ -544,7 +540,7 @@ public class IndexInfo extends Derived implements IndexInfoConfig.Producer { */ private static abstract class IndexOverrider { - protected IndexInfo owner; + protected final IndexInfo owner; public IndexOverrider(IndexInfo owner) { this.owner = owner; @@ -560,7 +556,7 @@ public class IndexInfo extends Derived implements IndexInfoConfig.Producer { private static class StemmingOverrider extends IndexOverrider { - private Schema schema; + private final Schema schema; public StemmingOverrider(IndexInfo owner, Schema schema) { super(owner); diff --git a/config-model/src/main/java/com/yahoo/schema/derived/SummaryClass.java b/config-model/src/main/java/com/yahoo/schema/derived/SummaryClass.java index 193c6893203..efc64a5aa40 100644 --- a/config-model/src/main/java/com/yahoo/schema/derived/SummaryClass.java +++ b/config-model/src/main/java/com/yahoo/schema/derived/SummaryClass.java @@ -59,7 +59,7 @@ public class SummaryClass extends Derived { /** MUST be called after all other fields are added */ private void deriveImplicitFields(DocumentSummary summary, Map<String, SummaryClassField> fields) { if (summary.getName().equals("default")) { - addField(SummaryClass.DOCUMENT_ID_FIELD, DataType.STRING, fields); + addField(SummaryClass.DOCUMENT_ID_FIELD, DataType.STRING, SummaryTransform.DOCUMENT_ID, fields); } } @@ -72,10 +72,6 @@ public class SummaryClass extends Derived { } } - private void addField(String name, DataType type, Map<String, SummaryClassField> fields) { - addField(name, type, null, fields); - } - private void addField(String name, DataType type, SummaryTransform transform, Map<String, SummaryClassField> fields) { diff --git a/config-model/src/main/java/com/yahoo/schema/derived/SummaryMap.java b/config-model/src/main/java/com/yahoo/schema/derived/SummaryMap.java index 40d763eb897..9d3d00f1481 100644 --- a/config-model/src/main/java/com/yahoo/schema/derived/SummaryMap.java +++ b/config-model/src/main/java/com/yahoo/schema/derived/SummaryMap.java @@ -30,6 +30,7 @@ public class SummaryMap extends Derived implements SummarymapConfig.Producer { for (DocumentSummary documentSummary : schema.getSummaries().values()) { derive(documentSummary); } + addSummaryTransformForDocumentId(); super.derive(schema); } @@ -37,11 +38,22 @@ public class SummaryMap extends Derived implements SummarymapConfig.Producer { protected void derive(ImmutableSDField field, Schema schema) { } + private void addSummaryTransformForDocumentId() { + // The 'documentid' field is added to the 'default' summary class in SummaryClass.deriveImplicitFields(). + // This ensures the corresponding transform is added as well. + if (!resultTransforms.containsKey(SummaryClass.DOCUMENT_ID_FIELD)) { + resultTransforms.put(SummaryClass.DOCUMENT_ID_FIELD, + new FieldResultTransform(SummaryClass.DOCUMENT_ID_FIELD, SummaryTransform.DOCUMENT_ID, "")); + } + } + private void derive(DocumentSummary documentSummary) { for (SummaryField summaryField : documentSummary.getSummaryFields().values()) { if (summaryField.getTransform()== SummaryTransform.NONE) continue; if (summaryField.getTransform()==SummaryTransform.ATTRIBUTE || + (summaryField.getTransform()==SummaryTransform.ATTRIBUTECOMBINER && summaryField.hasExplicitSingleSource()) || + summaryField.getTransform()==SummaryTransform.COPY || summaryField.getTransform()==SummaryTransform.DISTANCE || summaryField.getTransform()==SummaryTransform.GEOPOS || summaryField.getTransform()==SummaryTransform.POSITIONS || @@ -57,7 +69,7 @@ public class SummaryMap extends Derived implements SummarymapConfig.Producer { // This works, but is suboptimal. We could consolidate to a minimal set and // use the right value from the minimal set as the third parameter here, // and add "override" commands to multiple static values - boolean useFieldNameAsArgument = summaryField.getTransform().isDynamic() || summaryField.getTransform() == SummaryTransform.TEXTEXTRACTOR; + boolean useFieldNameAsArgument = summaryField.getTransform().isDynamic(); resultTransforms.put(summaryField.getName(), new FieldResultTransform(summaryField.getName(), summaryField.getTransform(), useFieldNameAsArgument ? summaryField.getName() : "")); diff --git a/config-model/src/main/java/com/yahoo/schema/document/ImmutableImportedSDField.java b/config-model/src/main/java/com/yahoo/schema/document/ImmutableImportedSDField.java index 335942de99d..221280dc1e4 100644 --- a/config-model/src/main/java/com/yahoo/schema/document/ImmutableImportedSDField.java +++ b/config-model/src/main/java/com/yahoo/schema/document/ImmutableImportedSDField.java @@ -83,6 +83,11 @@ public class ImmutableImportedSDField implements ImmutableSDField { } @Override + public boolean wasConfiguredToDoIndexing() { + return importedField.targetField().wasConfiguredToDoIndexing(); + } + + @Override public DataType getDataType() { return importedField.targetField().getDataType(); } diff --git a/config-model/src/main/java/com/yahoo/schema/document/ImmutableSDField.java b/config-model/src/main/java/com/yahoo/schema/document/ImmutableSDField.java index 44e442811ba..37cbb3a8171 100644 --- a/config-model/src/main/java/com/yahoo/schema/document/ImmutableSDField.java +++ b/config-model/src/main/java/com/yahoo/schema/document/ImmutableSDField.java @@ -45,6 +45,14 @@ public interface ImmutableSDField { */ boolean wasConfiguredToDoAttributing(); + /** + * Whether this field at some time was configured to do indexing. + * + * This function can typically return a different value than doesIndexing(), + * which uses the final state of the underlying indexing script instead. + */ + boolean wasConfiguredToDoIndexing(); + DataType getDataType(); Index getIndex(String name); diff --git a/config-model/src/main/java/com/yahoo/schema/document/SDField.java b/config-model/src/main/java/com/yahoo/schema/document/SDField.java index 943d6c6fc14..2d79d89a2a0 100644 --- a/config-model/src/main/java/com/yahoo/schema/document/SDField.java +++ b/config-model/src/main/java/com/yahoo/schema/document/SDField.java @@ -117,6 +117,7 @@ public class SDField extends Field implements TypedKey, ImmutableSDField { private boolean isExtraField = false; private boolean wasConfiguredToDoAttributing = false; + private boolean wasConfiguredToDoIndexing = false; /** * Creates a new field. This method is only used to create reserved fields. @@ -381,6 +382,11 @@ public class SDField extends Field implements TypedKey, ImmutableSDField { return wasConfiguredToDoAttributing; } + @Override + public boolean wasConfiguredToDoIndexing() { + return wasConfiguredToDoIndexing; + } + /** Parse an indexing expression which will use the simple linguistics implementation suitable for testing */ public void parseIndexingScript(String script) { parseIndexingScript(script, new SimpleLinguistics(), Embedder.throwsOnUse.asMap()); @@ -408,6 +414,9 @@ public class SDField extends Field implements TypedKey, ImmutableSDField { if (!wasConfiguredToDoAttributing()) { wasConfiguredToDoAttributing = doesAttributing(); } + if (!wasConfiguredToDoIndexing()) { + wasConfiguredToDoIndexing = doesIndexing(); + } if (!usesStructOrMap()) { new ExpressionVisitor() { diff --git a/config-model/src/main/java/com/yahoo/schema/parser/IntermediateCollection.java b/config-model/src/main/java/com/yahoo/schema/parser/IntermediateCollection.java index 8bb9bca3249..139d20aac82 100644 --- a/config-model/src/main/java/com/yahoo/schema/parser/IntermediateCollection.java +++ b/config-model/src/main/java/com/yahoo/schema/parser/IntermediateCollection.java @@ -26,7 +26,7 @@ public class IntermediateCollection { private final DeployLogger deployLogger; private final ModelContext.Properties modelProperties; - private Map<String, ParsedSchema> parsedSchemas = new LinkedHashMap<>(); + private final Map<String, ParsedSchema> parsedSchemas = new LinkedHashMap<>(); IntermediateCollection() { this.deployLogger = new BaseDeployLogger(); diff --git a/config-model/src/main/java/com/yahoo/schema/processing/AddExtraFieldsToDocument.java b/config-model/src/main/java/com/yahoo/schema/processing/AddExtraFieldsToDocument.java index ca81301da73..3afc25131c0 100644 --- a/config-model/src/main/java/com/yahoo/schema/processing/AddExtraFieldsToDocument.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/AddExtraFieldsToDocument.java @@ -38,7 +38,7 @@ public class AddExtraFieldsToDocument extends Processor { case BOLDED: case DYNAMICBOLDED: case DYNAMICTEASER: - case TEXTEXTRACTOR: + case DOCUMENT_ID: // TODO: Adding the 'documentid' field should no longer be needed when the docsum framework in the backend has been simplified and the transform is always used. addSummaryField(schema, document, summaryField, validate); break; default: diff --git a/config-model/src/main/java/com/yahoo/schema/processing/CreatePositionZCurve.java b/config-model/src/main/java/com/yahoo/schema/processing/CreatePositionZCurve.java index e39b78d0d9f..648ed085a54 100644 --- a/config-model/src/main/java/com/yahoo/schema/processing/CreatePositionZCurve.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/CreatePositionZCurve.java @@ -36,6 +36,7 @@ import java.util.logging.Level; */ public class CreatePositionZCurve extends Processor { + private boolean useV8GeoPositions = false; private final SDDocumentType repo; public CreatePositionZCurve(Schema schema, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { @@ -43,8 +44,6 @@ public class CreatePositionZCurve extends Processor { this.repo = schema.getDocument(); } - private boolean useV8GeoPositions = false; - @Override public void process(boolean validate, boolean documentsOnly, ModelContext.Properties properties) { this.useV8GeoPositions = properties.featureFlags().useV8GeoPositions(); diff --git a/config-model/src/main/java/com/yahoo/schema/processing/Processing.java b/config-model/src/main/java/com/yahoo/schema/processing/Processing.java index 63eca2121c1..8f7e8daeed0 100644 --- a/config-model/src/main/java/com/yahoo/schema/processing/Processing.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/Processing.java @@ -52,6 +52,7 @@ public class Processing { ImplicitSummaries::new, ImplicitSummaryFields::new, AdjustPositionSummaryFields::new, + SummaryTransformForDocumentId::new, SummaryConsistency::new, SummaryNamesFieldCollisions::new, SummaryFieldsMustHaveValidSource::new, diff --git a/config-model/src/main/java/com/yahoo/schema/processing/RankingExpressionTypeResolver.java b/config-model/src/main/java/com/yahoo/schema/processing/RankingExpressionTypeResolver.java index 07f79f16334..d985089b2cb 100644 --- a/config-model/src/main/java/com/yahoo/schema/processing/RankingExpressionTypeResolver.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/RankingExpressionTypeResolver.java @@ -93,9 +93,9 @@ public class RankingExpressionTypeResolver extends Processor { throw new IllegalArgumentException(profile + " is strict but is missing a query profile type " + "declaration of features " + context.queryFeaturesNotDeclared()); else - deployLogger.logApplicationPackage(Level.WARNING, "The following query features used in " + profile + - " are not declared in query profile " + - "types and will be interpreted as scalars, not tensors: " + + deployLogger.logApplicationPackage(Level.WARNING, "The following query features used in " + + profile + " are not declared " + + "and will be interpreted as scalars, not tensors: " + context.queryFeaturesNotDeclared()); warnedAbout.addAll(context.queryFeaturesNotDeclared()); } diff --git a/config-model/src/main/java/com/yahoo/schema/processing/SummaryConsistency.java b/config-model/src/main/java/com/yahoo/schema/processing/SummaryConsistency.java index 4fb45c3c68f..4aa8f6f0e37 100644 --- a/config-model/src/main/java/com/yahoo/schema/processing/SummaryConsistency.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/SummaryConsistency.java @@ -37,6 +37,7 @@ public class SummaryConsistency extends Processor { assertConsistency(summaryField, schema, validate); makeAttributeTransformIfAppropriate(summaryField, schema); makeAttributeCombinerTransformIfAppropriate(summaryField, schema); + makeCopyTransformIfAppropriate(summaryField, schema); } } } @@ -70,14 +71,27 @@ public class SummaryConsistency extends Processor { /** If the source is a complex field with only struct field attributes then make this use the attribute combiner transform */ private void makeAttributeCombinerTransformIfAppropriate(SummaryField summaryField, Schema schema) { if (summaryField.getTransform() == SummaryTransform.NONE) { - String source_field_name = summaryField.getSingleSource(); - ImmutableSDField source = schema.getField(source_field_name); + String sourceFieldName = summaryField.getSingleSource(); + ImmutableSDField source = schema.getField(sourceFieldName); if (source != null && isComplexFieldWithOnlyStructFieldAttributes(source)) { summaryField.setTransform(SummaryTransform.ATTRIBUTECOMBINER); } } } + /* + * This function must be called after makeAttributeCombinerTransformIfAppropriate(). + */ + private void makeCopyTransformIfAppropriate(SummaryField summaryField, Schema schema) { + if (summaryField.getTransform() == SummaryTransform.NONE) { + String sourceFieldName = summaryField.getSingleSource(); + ImmutableSDField source = schema.getField(sourceFieldName); + if (source != null && source.usesStructOrMap() && summaryField.hasExplicitSingleSource()) { + summaryField.setTransform(SummaryTransform.COPY); + } + } + } + private void assertConsistentTypes(SummaryField existing, SummaryField seen) { if (existing.getDataType() instanceof WeightedSetDataType && seen.getDataType() instanceof WeightedSetDataType && ((WeightedSetDataType)existing.getDataType()).getNestedType().equals(((WeightedSetDataType)seen.getDataType()).getNestedType())) diff --git a/config-model/src/main/java/com/yahoo/schema/processing/SummaryTransformForDocumentId.java b/config-model/src/main/java/com/yahoo/schema/processing/SummaryTransformForDocumentId.java new file mode 100644 index 00000000000..99419ecd526 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/processing/SummaryTransformForDocumentId.java @@ -0,0 +1,32 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing; + +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.schema.Schema; +import com.yahoo.schema.derived.SummaryClass; +import com.yahoo.vespa.documentmodel.SummaryTransform; +import com.yahoo.vespa.model.container.search.QueryProfiles; + +/** + * Adds the corresponding summary transform for all "documentid" summary fields. + * + * @author geirst + */ +public class SummaryTransformForDocumentId extends Processor { + + public SummaryTransformForDocumentId(Schema schema, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { + super(schema, deployLogger, rankProfileRegistry, queryProfiles); + } + + @Override + public void process(boolean validate, boolean documentsOnly) { + for (var summary : schema.getSummaries().values()) { + for (var summaryField : summary.getSummaryFields().values()) { + if (summaryField.getName().equals(SummaryClass.DOCUMENT_ID_FIELD)) { + summaryField.setTransform(SummaryTransform.DOCUMENT_ID); + } + } + } + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentManager.java b/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentManager.java index e54ed1f1a8b..c6fcd4c115c 100644 --- a/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentManager.java +++ b/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentManager.java @@ -36,14 +36,13 @@ public class DocumentManager { private boolean useV8GeoPositions = false; - public DocumentManager useV8GeoPositions(boolean value) { - this.useV8GeoPositions = value; + public DocumentManager useV8GeoPositions(boolean useV8GeoPositions) { + this.useV8GeoPositions = useV8GeoPositions; return this; } public DocumentmanagerConfig.Builder produce(DocumentModel model, - DocumentmanagerConfig.Builder documentConfigBuilder) - { + DocumentmanagerConfig.Builder documentConfigBuilder) { return produceDocTypes(model, documentConfigBuilder); } @@ -75,6 +74,7 @@ public class DocumentManager { } static private class IdxMap { + private final Map<Integer, Boolean> doneMap = new HashMap<>(); private final Map<String, Integer> map = new HashMap<>(); private final DataTypeRecognizer recognizer = new DataTypeRecognizer(); diff --git a/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentTypes.java b/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentTypes.java index 549b94adc02..d62270034f0 100644 --- a/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentTypes.java +++ b/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentTypes.java @@ -33,6 +33,7 @@ import java.util.Set; * @author baldersheim */ public class DocumentTypes { + private boolean useV8GeoPositions = false; public DocumentTypes useV8GeoPositions(boolean value) { diff --git a/config-model/src/main/java/com/yahoo/vespa/documentmodel/DocumentModel.java b/config-model/src/main/java/com/yahoo/vespa/documentmodel/DocumentModel.java index 15599c567ab..76f0bbd854e 100644 --- a/config-model/src/main/java/com/yahoo/vespa/documentmodel/DocumentModel.java +++ b/config-model/src/main/java/com/yahoo/vespa/documentmodel/DocumentModel.java @@ -4,20 +4,18 @@ package com.yahoo.vespa.documentmodel; import com.yahoo.documentmodel.DocumentTypeRepo; /** - * DocumentModel represents everything derived from a set of search definitions. - * It contains a document manager managing all defined document types. - * It contains a search manager managing all specified search definitions. - * It contains a storage manager managing all specified storage definitions. + * DocumentModel represents everything derived from a set of schemas. + * It contains a document manager managing all defined document types, and + * a search manager managing all search aspects of the schemas. * * @author baldersheim */ public class DocumentModel { - private final DocumentTypeRepo documentMan = new DocumentTypeRepo(); - private final SearchManager searchMan = new SearchManager(); + private final DocumentTypeRepo documentManager = new DocumentTypeRepo(); + private final SearchManager searchManager = new SearchManager(); - public DocumentTypeRepo getDocumentManager() { return documentMan; } - - public SearchManager getSearchManager() { return searchMan; } + public DocumentTypeRepo getDocumentManager() { return documentManager; } + public SearchManager getSearchManager() { return searchManager; } } diff --git a/config-model/src/main/java/com/yahoo/vespa/documentmodel/SummaryField.java b/config-model/src/main/java/com/yahoo/vespa/documentmodel/SummaryField.java index ed6668f0d0d..58044163885 100644 --- a/config-model/src/main/java/com/yahoo/vespa/documentmodel/SummaryField.java +++ b/config-model/src/main/java/com/yahoo/vespa/documentmodel/SummaryField.java @@ -263,6 +263,22 @@ public class SummaryField extends Field implements Cloneable, TypedKey { } } + /** + * Returns true if the summary field uses an explicit source, i.e. + * a field with different name that is not a nested field. + */ + public boolean hasExplicitSingleSource() { + String fieldName = getName(); + String sourceName = getSingleSource(); + if (fieldName.equals(sourceName)) { + return false; + } + if (sourceName.contains(".")) { + return false; + } + return true; + } + public VsmCommand getVsmCommand() { return vsmCommand; } diff --git a/config-model/src/main/java/com/yahoo/vespa/documentmodel/SummaryTransform.java b/config-model/src/main/java/com/yahoo/vespa/documentmodel/SummaryTransform.java index c50766a2585..e94518c988d 100644 --- a/config-model/src/main/java/com/yahoo/vespa/documentmodel/SummaryTransform.java +++ b/config-model/src/main/java/com/yahoo/vespa/documentmodel/SummaryTransform.java @@ -18,11 +18,12 @@ public enum SummaryTransform { POSITIONS("positions"), RANKFEATURES("rankfeatures"), SUMMARYFEATURES("summaryfeatures"), - TEXTEXTRACTOR("textextractor"), GEOPOS("geopos"), ATTRIBUTECOMBINER("attributecombiner"), MATCHED_ELEMENTS_FILTER("matchedelementsfilter"), - MATCHED_ATTRIBUTE_ELEMENTS_FILTER("matchedattributeelementsfilter"); + MATCHED_ATTRIBUTE_ELEMENTS_FILTER("matchedattributeelementsfilter"), + COPY("copy"), + DOCUMENT_ID("documentid"); private final String name; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/HostSystem.java b/config-model/src/main/java/com/yahoo/vespa/model/HostSystem.java index 6e63a1d9e68..fcad1e20c16 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/HostSystem.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/HostSystem.java @@ -18,6 +18,7 @@ import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; +import java.util.function.BiConsumer; import java.util.logging.Level; import java.util.logging.Logger; import java.util.stream.Collectors; @@ -39,30 +40,34 @@ public class HostSystem extends AbstractConfigProducer<Host> { private final Map<String, HostResource> hostname2host = new LinkedHashMap<>(); private final HostProvisioner provisioner; private final DeployLogger deployLogger; + private final boolean isHosted; static { String checkIpProperty = System.getProperty("config_model.ip_check", "true"); doCheckIp = ! checkIpProperty.equalsIgnoreCase("false"); } - public HostSystem(AbstractConfigProducer<?> parent, String name, HostProvisioner provisioner, DeployLogger deployLogger) { + public HostSystem(AbstractConfigProducer<?> parent, String name, HostProvisioner provisioner, DeployLogger deployLogger, boolean isHosted) { super(parent, name); this.provisioner = provisioner; this.deployLogger = deployLogger; + this.isHosted = isHosted; } void checkName(String hostname) { if (doCheckIp) { + // Bad DNS config in a hosted system isn't actionable by the tenant, so we log any warnings internally + BiConsumer<Level, String> logFunction = isHosted ? deployLogger::log : deployLogger::logApplicationPackage; // Give a warning if the host does not exist try { var inetAddr = java.net.InetAddress.getByName(hostname); String canonical = inetAddr.getCanonicalHostName(); if (!hostname.equals(canonical)) { - deployLogger.logApplicationPackage(Level.WARNING, "Host named '" + hostname + "' may not receive any config " + - "since it differs from its canonical hostname '" + canonical + "' (check DNS and /etc/hosts)."); + logFunction.accept(Level.WARNING, "Host named '" + hostname + "' may not receive any config " + + "since it differs from its canonical hostname '" + canonical + "' (check DNS and /etc/hosts)."); } } catch (UnknownHostException e) { - deployLogger.logApplicationPackage(Level.WARNING, "Unable to lookup IP address of host: " + hostname); + logFunction.accept(Level.WARNING, "Unable to lookup IP address of host: " + hostname); } } } @@ -139,7 +144,7 @@ public class HostSystem extends AbstractConfigProducer<Host> { private Optional<HostResource> getExistingHost(HostSpec key) { List<HostResource> hosts = hostname2host.values().stream() .filter(resource -> resource.getHostname().equals(key.hostname())) - .collect(Collectors.toList()); + .toList(); if (hosts.isEmpty()) { return Optional.empty(); } else { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/Logserver.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/Logserver.java index d5a0b97a416..6f0e5a775e9 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/Logserver.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/Logserver.java @@ -7,8 +7,7 @@ import com.yahoo.vespa.model.AbstractService; import com.yahoo.vespa.model.PortAllocBridge; /** - * Represents the Logserver. There is exactly one logserver in a Vespa - * system. + * Represents the Logserver. There is exactly one logserver in a Vespa system. * * @author gjoranv * @author bjorncs @@ -42,9 +41,7 @@ public class Logserver extends AbstractService { return "exec $ROOT/bin/vespa-logserver-start " + getMyJVMArgs() + " " + getJvmOptions(); } - /** - * @return the jvm args to be used by the logserver. - */ + /** Returns the jvm args to be used by the logserver. */ private String getMyJVMArgs() { StringBuilder sb = new StringBuilder(); sb.append("--add-opens=java.base/java.io=ALL-UNNAMED"); @@ -57,9 +54,7 @@ public class Logserver extends AbstractService { return sb.toString(); } - /** - * Returns the desired base port for this service. - */ + /** Returns the desired base port for this service. */ public int getWantedPort() { return 19080; } @@ -73,9 +68,7 @@ public class Logserver extends AbstractService { return true; // TODO Support dynamic port allocation for logserver } - /** - * @return the number of ports needed by the logserver. - */ + /** Returns the number of ports needed by the logserver. */ public int getPortCount() { return 4; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/LogserverContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/LogserverContainerCluster.java index 75b13a89e83..c50b9b7f842 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/LogserverContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/LogserverContainerCluster.java @@ -36,7 +36,7 @@ public class LogserverContainerCluster extends ContainerCluster<LogserverContain protected boolean messageBusEnabled() { return false; } private void addLogHandler() { - Handler<?> logHandler = Handler.fromClassName(ContainerCluster.LOG_HANDLER_CLASS); + Handler logHandler = Handler.fromClassName(ContainerCluster.LOG_HANDLER_CLASS); logHandler.addServerBindings(SystemBindingPattern.fromHttpPath("/logs")); addComponent(logHandler); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerCluster.java index bf3ba084091..88acf6cafb8 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerCluster.java @@ -49,6 +49,7 @@ public class ClusterControllerCluster extends AbstractConfigProducer<ClusterCont public void getConfig(ZookeeperServerConfig.Builder builder) { builder.clientPort(ZK_CLIENT_PORT); builder.juteMaxBuffer(1024 * 1024); // 1 Mb should be more than enough for cluster controller + builder.snapshotCount(1000); // Use a low value, few transactions per time unit in cluster controller for (ClusterControllerContainer container : containerCluster.getContainers()) { ZookeeperServerConfig.Server.Builder serverBuilder = new ZookeeperServerConfig.Server.Builder(); serverBuilder.hostname(container.getHostName()); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainer.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainer.java index f303da6c9f0..5d60cec0679 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainer.java @@ -49,7 +49,7 @@ public class ClusterControllerContainer extends Container implements private static final int DEFAULT_NETTY_NUM_DIRECT_ARENAS = 1; // Reduced from nettys default of 2*cores private static final int DEFAULT_NETTY_NUM_HEAP_ARENAS = 1; // Reduced from nettys default of 2*cores - private final Set<String> bundles = new TreeSet<>(); + private final Set<String> bundles = new TreeSet<>(); // Ensure stable ordering public ClusterControllerContainer(AbstractConfigProducer<?> parent, int index, @@ -118,7 +118,7 @@ public class ClusterControllerContainer extends Container implements ZOOKEEPER_SERVER_BUNDLE); } - private void addHandler(Handler<?> h, String path) { + private void addHandler(Handler h, String path) { h.addServerBindings(SystemBindingPattern.fromHttpPath(path)); super.addHandler(h); } @@ -138,7 +138,7 @@ public class ClusterControllerContainer extends Container implements } private void addHandler(String id, String className, String path, ComponentSpecification bundle) { - addHandler(new Handler<>(createComponentModel(id, className, bundle)), path); + addHandler(new Handler(createComponentModel(id, className, bundle)), path); } private ReindexingContext reindexingContext() { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainerCluster.java index a7f3a6224f2..f7007fec181 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainerCluster.java @@ -7,8 +7,12 @@ import com.yahoo.config.model.producer.AbstractConfigProducer; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.search.config.QrStartConfig; import com.yahoo.vespa.model.container.ContainerCluster; +import com.yahoo.vespa.model.container.PlatformBundles; +import java.nio.file.Path; +import java.util.Collections; import java.util.Optional; +import java.util.Set; /** * Container cluster for cluster-controller containers. @@ -18,6 +22,8 @@ import java.util.Optional; */ public class ClusterControllerContainerCluster extends ContainerCluster<ClusterControllerContainer> { + private static final Set<Path> UNNECESSARY_BUNDLES = Collections.unmodifiableSet(PlatformBundles.VESPA_SECURITY_BUNDLES); + private final ReindexingContext reindexingContext; public ClusterControllerContainerCluster( @@ -29,6 +35,9 @@ public class ClusterControllerContainerCluster extends ContainerCluster<ClusterC } @Override + protected Set<Path> unnecessaryPlatformBundles() { return UNNECESSARY_BUNDLES; } + + @Override protected void doPrepare(DeployState deployState) { } @Override protected boolean messageBusEnabled() { return false; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java index a29647b062a..17f169033d3 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java @@ -36,16 +36,19 @@ import com.yahoo.vespa.model.admin.monitoring.MetricSet; import com.yahoo.vespa.model.admin.monitoring.MetricsConsumer; import com.yahoo.vespa.model.admin.monitoring.Monitoring; import com.yahoo.vespa.model.container.ContainerCluster; +import com.yahoo.vespa.model.container.PlatformBundles; import com.yahoo.vespa.model.container.component.Handler; import com.yahoo.vespa.model.container.component.SystemBindingPattern; -import com.yahoo.vespa.model.container.PlatformBundles; import java.nio.file.Path; import java.util.Collections; import java.util.LinkedHashMap; import java.util.Map; import java.util.Optional; +import java.util.Set; import java.util.logging.Logger; +import java.util.stream.Collectors; +import java.util.stream.Stream; import static com.yahoo.vespa.model.admin.metricsproxy.ConsumersConfigGenerator.addMetrics; import static com.yahoo.vespa.model.admin.metricsproxy.ConsumersConfigGenerator.generateConsumers; @@ -75,6 +78,12 @@ public class MetricsProxyContainerCluster extends ContainerCluster<MetricsProxyC static final Path METRICS_PROXY_BUNDLE_FILE = PlatformBundles.absoluteBundlePath(METRICS_PROXY_NAME); static final String METRICS_PROXY_BUNDLE_NAME = "com.yahoo.vespa." + METRICS_PROXY_NAME; + private static final Set<Path> UNNECESSARY_BUNDLES = Stream.concat + ( + PlatformBundles.VESPA_SECURITY_BUNDLES.stream(), + PlatformBundles.VESPA_ZK_BUNDLES.stream() + ).collect(Collectors.toSet()); + static final class AppDimensionNames { static final String SYSTEM = "system"; static final String TENANT = "tenantName"; @@ -98,6 +107,9 @@ public class MetricsProxyContainerCluster extends ContainerCluster<MetricsProxyC addClusterComponents(); } + @Override + protected Set<Path> unnecessaryPlatformBundles() { return UNNECESSARY_BUNDLES; } + private void addClusterComponents() { addMetricsProxyComponent(ApplicationDimensions.class); addMetricsProxyComponent(ConfigSentinelClient.class); @@ -119,12 +131,12 @@ public class MetricsProxyContainerCluster extends ContainerCluster<MetricsProxyC } private void addHttpHandler(Class<? extends ThreadedHttpRequestHandler> clazz, String bindingPath) { - Handler<AbstractConfigProducer<?>> metricsHandler = createMetricsHandler(clazz, bindingPath); + Handler metricsHandler = createMetricsHandler(clazz, bindingPath); addComponent(metricsHandler); } - static Handler<AbstractConfigProducer<?>> createMetricsHandler(Class<? extends ThreadedHttpRequestHandler> clazz, String bindingPath) { - Handler<AbstractConfigProducer<?>> metricsHandler = new Handler<>( + static Handler createMetricsHandler(Class<? extends ThreadedHttpRequestHandler> clazz, String bindingPath) { + Handler metricsHandler = new Handler( new ComponentModel(clazz.getName(), null, METRICS_PROXY_BUNDLE_NAME, null)); metricsHandler.addServerBindings( SystemBindingPattern.fromHttpPath(bindingPath), diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/DefaultMetrics.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/DefaultMetrics.java index 5e10001baf6..2348970ed1a 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/DefaultMetrics.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/DefaultMetrics.java @@ -33,7 +33,7 @@ public class DefaultMetrics { return ImmutableSet.<Metric>builder() .addAll(getContentMetrics()) .addAll(getContainerMetrics()) - .addAll(getQrserverMetrics()) + .addAll(getSearchChainMetrics()) .build(); } @@ -51,15 +51,24 @@ public class DefaultMetrics { return metrics; } - private static Set<Metric> getQrserverMetrics() { + private static Set<Metric> getSearchChainMetrics() { Set<Metric> metrics = new LinkedHashSet<>(); metrics.add(new Metric("queries.rate")); - metrics.add(new Metric("query_latency.average")); + metrics.add(new Metric("query_latency.sum")); + metrics.add(new Metric("query_latency.count")); + metrics.add(new Metric("query_latency.max")); + metrics.add(new Metric("query_latency.average")); // TODO: Remove with Vespa 9 metrics.add(new Metric("query_latency.95percentile")); metrics.add(new Metric("query_latency.99percentile")); - metrics.add(new Metric("hits_per_query.average")); - metrics.add(new Metric("totalhits_per_query.average")); + metrics.add(new Metric("hits_per_query.sum")); + metrics.add(new Metric("hits_per_query.count")); + metrics.add(new Metric("hits_per_query.max")); + metrics.add(new Metric("hits_per_query.average")); // TODO: Remove with Vespa 9 + metrics.add(new Metric("totalhits_per_query.sum")); + metrics.add(new Metric("totalhits_per_query.count")); + metrics.add(new Metric("totalhits_per_query.max")); + metrics.add(new Metric("totalhits_per_query.average")); // TODO: Remove with Vespa 9 metrics.add(new Metric("degraded_queries.rate")); metrics.add(new Metric("failed_queries.rate")); metrics.add(new Metric("serverActiveThreads.average")); @@ -71,8 +80,14 @@ public class DefaultMetrics { Set<Metric> metrics = new LinkedHashSet<>(); metrics.add(new Metric("content.proton.search_protocol.docsum.requested_documents.rate")); - metrics.add(new Metric("content.proton.search_protocol.docsum.latency.average")); - metrics.add(new Metric("content.proton.search_protocol.query.latency.average")); + metrics.add(new Metric("content.proton.search_protocol.docsum.latency.sum")); + metrics.add(new Metric("content.proton.search_protocol.docsum.latency.count")); + metrics.add(new Metric("content.proton.search_protocol.docsum.latency.max")); + metrics.add(new Metric("content.proton.search_protocol.docsum.latency.average")); // TODO: Remove with Vespa 9 + metrics.add(new Metric("content.proton.search_protocol.query.latency.sum")); + metrics.add(new Metric("content.proton.search_protocol.query.latency.count")); + metrics.add(new Metric("content.proton.search_protocol.query.latency.max")); + metrics.add(new Metric("content.proton.search_protocol.query.latency.average")); // TODO: Remove with Vespa 9 metrics.add(new Metric("content.proton.documentdb.documents.total.last")); metrics.add(new Metric("content.proton.documentdb.documents.ready.last")); @@ -85,9 +100,18 @@ public class DefaultMetrics { metrics.add(new Metric("content.proton.documentdb.matching.docs_matched.rate")); metrics.add(new Metric("content.proton.documentdb.matching.docs_reranked.rate")); - metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.query_setup_time.average")); - metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.query_latency.average")); - metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.rerank_time.average")); + metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.query_setup_time.sum")); + metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.query_setup_time.count")); + metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.query_setup_time.max")); + metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.query_setup_time.average")); // TODO: Remove with Vespa 9 + metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.query_latency.sum")); + metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.query_latency.count")); + metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.query_latency.max")); + metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.query_latency.average")); // TODO: Remove with Vespa 9 + metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.rerank_time.sum")); + metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.rerank_time.count")); + metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.rerank_time.max")); + metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.rerank_time.average")); // TODO: Remove with Vespa 9 metrics.add(new Metric("content.proton.transactionlog.disk_usage.last")); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java index 6ce8c54f98c..b25e8aaf2c0 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java @@ -28,7 +28,7 @@ public class VespaMetricSet { metrics.addAll(getDistributorMetrics()); metrics.addAll(getDocprocMetrics()); metrics.addAll(getClusterControllerMetrics()); - metrics.addAll(getQrserverMetrics()); + metrics.addAll(getSearchChainMetrics()); metrics.addAll(getContainerMetrics()); metrics.addAll(getConfigServerMetrics()); metrics.addAll(getSentinelMetrics()); @@ -297,7 +297,7 @@ public class VespaMetricSet { return metrics; } - private static Set<Metric> getQrserverMetrics() { + private static Set<Metric> getSearchChainMetrics() { Set<Metric> metrics = new LinkedHashSet<>(); metrics.add(new Metric("peak_qps.max")); @@ -351,7 +351,7 @@ public class VespaMetricSet { metrics.add(new Metric("relevance.at_10.sum")); metrics.add(new Metric("relevance.at_10.count")); - // Errors from qrserver + // Errors from search container metrics.add(new Metric("error.timeout.rate")); metrics.add(new Metric("error.backends_oos.rate")); metrics.add(new Metric("error.plugin_failure.rate")); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/AbstractBundleValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/AbstractBundleValidator.java index 1642a8cb3cc..ea6a081bf6a 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/AbstractBundleValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/AbstractBundleValidator.java @@ -9,6 +9,7 @@ import com.yahoo.config.application.api.ComponentInfo; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.path.Path; +import com.yahoo.text.XML; import com.yahoo.vespa.model.VespaModel; import org.w3c.dom.Document; import org.w3c.dom.Element; @@ -16,8 +17,6 @@ import org.w3c.dom.NodeList; import org.xml.sax.InputSource; import org.xml.sax.SAXException; -import javax.xml.parsers.DocumentBuilderFactory; -import javax.xml.parsers.ParserConfigurationException; import javax.xml.xpath.XPathConstants; import javax.xml.xpath.XPathExpressionException; import javax.xml.xpath.XPathFactory; @@ -96,20 +95,19 @@ public abstract class AbstractBundleValidator extends Validator { } private static final Pattern POM_FILE_LOCATION = Pattern.compile("META-INF/maven/.+?/.+?/pom.xml"); - private Optional<Document> getPomXmlContent(DeployLogger deployLogger, JarFile jar) { + public Optional<Document> getPomXmlContent(DeployLogger deployLogger, JarFile jar) { return jar.stream() .filter(f -> POM_FILE_LOCATION.matcher(f.getName()).matches()) .findFirst() .map(f -> { try { String text = new String(jar.getInputStream(f).readAllBytes()); - return DocumentBuilderFactory.newDefaultInstance().newDocumentBuilder() + return XML.getDocumentBuilder(false) .parse(new InputSource(new StringReader(text))); - } catch (ParserConfigurationException e) { - throw new RuntimeException(e); } catch (SAXException e) { - deployLogger.log(Level.INFO, String.format("Unable to parse pom.xml from %s", filename(jar))); - return null; + String message = String.format("Unable to parse pom.xml from %s", filename(jar)); + deployLogger.log(Level.SEVERE, message); + throw new RuntimeException(message, e); } catch (IOException e) { deployLogger.log(Level.INFO, String.format("Unable to read '%s' from '%s'", f.getName(), jar.getName())); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ComplexAttributeFieldsValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ComplexFieldsWithStructFieldAttributesValidator.java index 01bf846d4cb..8515c34a377 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ComplexAttributeFieldsValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ComplexFieldsWithStructFieldAttributesValidator.java @@ -22,7 +22,7 @@ import java.util.stream.Collectors; * * @author geirst */ -public class ComplexAttributeFieldsValidator extends Validator { +public class ComplexFieldsWithStructFieldAttributesValidator extends Validator { @Override public void validate(VespaModel model, DeployState deployState) { @@ -39,7 +39,7 @@ public class ComplexAttributeFieldsValidator extends Validator { private static void validateComplexFields(String clusterName, Schema schema) { String unsupportedFields = schema.allFields() .filter(field -> isUnsupportedComplexField(field)) - .map(ComplexAttributeFieldsValidator::toString) + .map(ComplexFieldsWithStructFieldAttributesValidator::toString) .collect(Collectors.joining(", ")); if (!unsupportedFields.isEmpty()) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ComplexFieldsWithStructFieldIndexesValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ComplexFieldsWithStructFieldIndexesValidator.java new file mode 100644 index 00000000000..fdd71ebccc5 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ComplexFieldsWithStructFieldIndexesValidator.java @@ -0,0 +1,75 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.application.validation; + +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.schema.Schema; +import com.yahoo.schema.document.ImmutableSDField; +import com.yahoo.vespa.model.VespaModel; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.logging.Level; +import java.util.stream.Collectors; + +/** + * Validates that complex fields (of type struct or map) do not have any struct fields with 'indexing: index'. + * This is not supported and will confuse the user if not validated. + * + * Only applies for indexed search clusters. + * + * @author geirst + */ +public class ComplexFieldsWithStructFieldIndexesValidator extends Validator { + + @Override + public void validate(VespaModel model, DeployState deployState) { + for (var cluster : model.getSearchClusters()) { + if (cluster.isStreaming()) { + continue; + } + for (var spec : cluster.schemas().values()) { + validateComplexFields(cluster.getClusterName(), spec.fullSchema(), deployState.getDeployLogger()); + } + } + } + + private static void validateComplexFields(String clusterName, Schema schema, DeployLogger logger) { + String unsupportedFields = schema.allFields() + .filter(field -> hasStructFieldsWithIndex(field)) + .map(ComplexFieldsWithStructFieldIndexesValidator::toString) + .collect(Collectors.joining(", ")); + + if (!unsupportedFields.isEmpty()) { + // TODO (Vespa 9 or before): Change back to an exception when no applications are using it wrong. + logger.logApplicationPackage(Level.WARNING, + String.format("For cluster '%s', schema '%s': The following complex fields have struct fields with 'indexing: index' which is not supported and has no effect: %s. " + + "Remove setting or change to 'indexing: attribute' if needed for matching.", + clusterName, schema.getName(), unsupportedFields)); + } + } + + private static boolean hasStructFieldsWithIndex(ImmutableSDField field) { + return (!field.isImportedField() && field.usesStructOrMap() && hasStructFieldsWithIndex(field.getStructFields())); + } + + private static String toString(ImmutableSDField field) { + return field.getName() + " (" + String.join(", ", getStructFieldsWithIndex(field.getStructFields())) + ")"; + } + + private static boolean hasStructFieldsWithIndex(Collection<? extends ImmutableSDField> structFields) { + return !getStructFieldsWithIndex(structFields).isEmpty(); + } + + private static List<String> getStructFieldsWithIndex(Collection<? extends ImmutableSDField> structFields) { + var result = new ArrayList<String>(); + for (var structField : structFields) { + if (structField.wasConfiguredToDoIndexing()) { + result.add(structField.getName()); + } + result.addAll(getStructFieldsWithIndex(structField.getStructFields())); + } + return result; + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/EndpointCertificateSecretsValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/EndpointCertificateSecretsValidator.java index 338c3a71e44..21025c38c13 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/EndpointCertificateSecretsValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/EndpointCertificateSecretsValidator.java @@ -1,7 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.application.validation; -import com.yahoo.config.model.api.EndpointCertificateSecrets; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.provision.CertificateNotReadyException; import com.yahoo.vespa.model.VespaModel; @@ -11,9 +10,9 @@ public class EndpointCertificateSecretsValidator extends Validator { /** This check is delayed until validation to allow node provisioning to complete while we are waiting for cert */ @Override public void validate(VespaModel model, DeployState deployState) { - if (deployState.endpointCertificateSecrets().isPresent() && deployState.endpointCertificateSecrets().get() == EndpointCertificateSecrets.MISSING) { - throw new CertificateNotReadyException("TLS enabled, but could not yet retrieve certificate for application " + deployState.getProperties().applicationId().serializedForm()); + if (deployState.endpointCertificateSecrets().isPresent() && deployState.endpointCertificateSecrets().get().isMissing()) { + throw new CertificateNotReadyException("TLS enabled, but could not yet retrieve certificate version %s for application %s" + .formatted(deployState.endpointCertificateSecrets().get().version(), deployState.getProperties().applicationId().serializedForm())); } } - } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/QuotaValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/QuotaValidator.java index e4a64e8d476..7ea582b99e6 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/QuotaValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/QuotaValidator.java @@ -84,22 +84,21 @@ public class QuotaValidator extends Validator { } } - private void throwIfBudgetNegative(double spend, BigDecimal budget, SystemName systemName) { + private static void throwIfBudgetNegative(double spend, BigDecimal budget, SystemName systemName) { if (budget.doubleValue() < 0) { - throwBudgetException("Please free up some capacity! This deployment's quota use is ($%.2f) and reserved quota is below zero! ($%.2f)", spend, budget, systemName); + throw new IllegalArgumentException(quotaMessage("Please free up some capacity", systemName, spend, budget)); } } - private void throwIfBudgetExceeded(double spend, BigDecimal budget, SystemName systemName) { + private static void throwIfBudgetExceeded(double spend, BigDecimal budget, SystemName systemName) { if (budget.doubleValue() < spend) { - throw new IllegalArgumentException((systemName.equals(SystemName.Public) ? "" : systemName.value() + ": ") + - "Deployment would make your tenant exceed its quota and has been blocked! Please contact support to update your plan."); + throw new IllegalArgumentException(quotaMessage("Deployment exceeds its quota and has been blocked! Please contact support to update your plan", systemName, spend, budget)); } } - private void throwBudgetException(String formatMessage, double spend, BigDecimal budget, SystemName systemName) { - var message = String.format(Locale.US, formatMessage, spend, budget); - var messageWithSystem = (systemName.equals(SystemName.Public) ? "" : systemName.value() + ": ") + message; - throw new IllegalArgumentException(messageWithSystem); + private static String quotaMessage(String message, SystemName system, double spend, BigDecimal budget) { + String quotaDescription = String.format(Locale.ENGLISH, "Quota is $%.2f, but at least $%.2f is required", budget, spend); + return (system == SystemName.Public ? "" : system.value() + ": ") + message + ": " + quotaDescription; } + } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/UriBindingsValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/UriBindingsValidator.java index 4f322578b1c..718f1646126 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/UriBindingsValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/UriBindingsValidator.java @@ -24,7 +24,7 @@ class UriBindingsValidator extends Validator { @Override public void validate(VespaModel model, DeployState deployState) { for (ApplicationContainerCluster cluster : model.getContainerClusters().values()) { - for (Handler<?> handler : cluster.getHandlers()) { + for (Handler handler : cluster.getHandlers()) { for (BindingPattern binding : handler.getServerBindings()) { validateUserBinding(binding, model, deployState); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java index ce61d3edc3b..dab1eeccc96 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java @@ -71,7 +71,8 @@ public class Validation { new SchemasDirValidator().validate(model, deployState); new BundleValidator().validate(model, deployState); new SearchDataTypeValidator().validate(model, deployState); - new ComplexAttributeFieldsValidator().validate(model, deployState); + new ComplexFieldsWithStructFieldAttributesValidator().validate(model, deployState); + new ComplexFieldsWithStructFieldIndexesValidator().validate(model, deployState); new StreamingValidator().validate(model, deployState); new RankSetupValidator(validationParameters.ignoreValidationErrors()).validate(model, deployState); new NoPrefixForIndexes().validate(model, deployState); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomClientProviderBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomClientProviderBuilder.java index 69613944e74..170e8940787 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomClientProviderBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomClientProviderBuilder.java @@ -5,7 +5,6 @@ import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.producer.AbstractConfigProducer; import com.yahoo.text.XML; import com.yahoo.vespa.model.container.ApplicationContainerCluster; -import com.yahoo.vespa.model.container.component.Component; import com.yahoo.vespa.model.container.component.Handler; import com.yahoo.vespa.model.container.component.UserBindingPattern; import org.w3c.dom.Element; @@ -22,7 +21,7 @@ public class DomClientProviderBuilder extends DomHandlerBuilder { @Override protected Handler doBuild(DeployState deployState, AbstractConfigProducer parent, Element clientElement) { - Handler<? super Component<?, ?>> client = createHandler(clientElement); + Handler client = createHandler(clientElement); for (Element binding : XML.getChildren(clientElement, "binding")) client.addClientBindings(UserBindingPattern.fromPattern(XML.getValue(binding))); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomHandlerBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomHandlerBuilder.java index 025a6377b04..7bfe971981e 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomHandlerBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomHandlerBuilder.java @@ -9,15 +9,12 @@ import com.yahoo.osgi.provider.model.ComponentModel; import com.yahoo.text.XML; import com.yahoo.vespa.model.container.ApplicationContainerCluster; import com.yahoo.vespa.model.container.component.BindingPattern; -import com.yahoo.vespa.model.container.component.Component; import com.yahoo.vespa.model.container.component.Handler; import com.yahoo.vespa.model.container.component.UserBindingPattern; import com.yahoo.vespa.model.container.xml.BundleInstantiationSpecificationBuilder; import org.w3c.dom.Element; -import java.util.List; import java.util.Set; -import java.util.logging.Level; import static com.yahoo.vespa.model.container.ApplicationContainerCluster.METRICS_V2_HANDLER_BINDING_1; import static com.yahoo.vespa.model.container.ApplicationContainerCluster.METRICS_V2_HANDLER_BINDING_2; @@ -29,7 +26,7 @@ import static java.util.logging.Level.INFO; /** * @author gjoranv */ -public class DomHandlerBuilder extends VespaDomBuilder.DomConfigProducerBuilder<Handler<?>> { +public class DomHandlerBuilder extends VespaDomBuilder.DomConfigProducerBuilder<Handler> { private static final Set<BindingPattern> reservedBindings = Set.of(METRICS_V2_HANDLER_BINDING_1, @@ -45,8 +42,8 @@ public class DomHandlerBuilder extends VespaDomBuilder.DomConfigProducerBuilder< } @Override - protected Handler<?> doBuild(DeployState deployState, AbstractConfigProducer<?> parent, Element handlerElement) { - Handler<? super Component<?, ?>> handler = createHandler(handlerElement); + protected Handler doBuild(DeployState deployState, AbstractConfigProducer<?> parent, Element handlerElement) { + Handler handler = createHandler(handlerElement); for (Element binding : XML.getChildren(handlerElement, "binding")) addServerBinding(handler, UserBindingPattern.fromPattern(XML.getValue(binding)), deployState.getDeployLogger()); @@ -56,18 +53,18 @@ public class DomHandlerBuilder extends VespaDomBuilder.DomConfigProducerBuilder< return handler; } - Handler<? super Component<?, ?>> createHandler(Element handlerElement) { + Handler createHandler(Element handlerElement) { BundleInstantiationSpecification bundleSpec = BundleInstantiationSpecificationBuilder.build(handlerElement); - return new Handler<>(new ComponentModel(bundleSpec)); + return new Handler(new ComponentModel(bundleSpec)); } - private void addServerBinding(Handler<? super Component<?, ?>> handler, BindingPattern binding, DeployLogger log) { + private void addServerBinding(Handler handler, BindingPattern binding, DeployLogger log) { throwIfBindingIsReserved(binding, handler); handler.addServerBindings(binding); removeExistingServerBinding(binding, handler, log); } - private void throwIfBindingIsReserved(BindingPattern binding, Handler<?> newHandler) { + private void throwIfBindingIsReserved(BindingPattern binding, Handler newHandler) { for (var reserved : reservedBindings) { if (binding.hasSamePattern(reserved)) { throw new IllegalArgumentException("Binding '" + binding.patternString() + "' is a reserved Vespa binding and " + @@ -76,7 +73,7 @@ public class DomHandlerBuilder extends VespaDomBuilder.DomConfigProducerBuilder< } } - private void removeExistingServerBinding(BindingPattern binding, Handler<?> newHandler, DeployLogger log) { + private void removeExistingServerBinding(BindingPattern binding, Handler newHandler, DeployLogger log) { for (var handler : cluster.getHandlers()) { for (BindingPattern serverBinding : handler.getServerBindings()) { if (serverBinding.hasSamePattern(binding)) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/VespaDomBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/VespaDomBuilder.java index e2bfcb66f37..c6fa7b8a69d 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/VespaDomBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/VespaDomBuilder.java @@ -193,7 +193,7 @@ public class VespaDomBuilder extends VespaModelBuilder { } /** - * The SimpleConfigProducer is the producer for elements such as qrservers, gateways. + * The SimpleConfigProducer is the producer for elements such as container. * Must support overrides for that too, hence this builder * * @author vegardh @@ -230,7 +230,9 @@ public class VespaDomBuilder extends VespaModelBuilder { deployState.getDocumentModel(), deployState.getVespaVersion(), deployState.getProperties().applicationId()); - root.setHostSystem(new HostSystem(root, "hosts", deployState.getProvisioner(), deployState.getDeployLogger())); + root.setHostSystem(new HostSystem(root, "hosts", deployState.getProvisioner(), + deployState.getDeployLogger(), + deployState.isHosted())); new Client(root); return root; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/clients/ContainerDocumentApi.java b/config-model/src/main/java/com/yahoo/vespa/model/clients/ContainerDocumentApi.java index 8534e1f65a7..f23f27c0d8e 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/clients/ContainerDocumentApi.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/clients/ContainerDocumentApi.java @@ -1,16 +1,16 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.clients; -import com.yahoo.config.model.producer.AbstractConfigProducer; -import com.yahoo.container.bundle.BundleInstantiationSpecification; import com.yahoo.container.handler.threadpool.ContainerThreadpoolConfig; import com.yahoo.osgi.provider.model.ComponentModel; import com.yahoo.vespa.model.container.ContainerCluster; import com.yahoo.vespa.model.container.ContainerThreadpool; +import com.yahoo.vespa.model.container.PlatformBundles; import com.yahoo.vespa.model.container.component.Handler; import com.yahoo.vespa.model.container.component.SystemBindingPattern; import com.yahoo.vespa.model.container.component.UserBindingPattern; +import java.nio.file.Path; import java.util.Collection; import java.util.Collections; @@ -21,43 +21,55 @@ import java.util.Collections; public class ContainerDocumentApi { public static final String DOCUMENT_V1_PREFIX = "/document/v1"; + public static final Path VESPACLIENT_CONTAINER_BUNDLE = + PlatformBundles.absoluteBundlePath("vespaclient-container-plugin"); - public ContainerDocumentApi(ContainerCluster<?> cluster, Options options) { - addRestApiHandler(cluster, options); - addFeedHandler(cluster, options); + private final boolean ignoreUndefinedFields; + + public ContainerDocumentApi(ContainerCluster<?> cluster, HandlerOptions handlerOptions, boolean ignoreUndefinedFields) { + this.ignoreUndefinedFields = ignoreUndefinedFields; + addRestApiHandler(cluster, handlerOptions); + addFeedHandler(cluster, handlerOptions); + addVespaClientContainerBundle(cluster); + } + + public static void addVespaClientContainerBundle(ContainerCluster<?> c) { + c.addPlatformBundle(VESPACLIENT_CONTAINER_BUNDLE); } - private static void addFeedHandler(ContainerCluster<?> cluster, Options options) { + private static void addFeedHandler(ContainerCluster<?> cluster, HandlerOptions handlerOptions) { String bindingSuffix = ContainerCluster.RESERVED_URI_PREFIX + "/feedapi"; - var handler = newVespaClientHandler("com.yahoo.vespa.http.server.FeedHandler", bindingSuffix, options); + var executor = new Threadpool("feedapi-handler", handlerOptions.feedApiThreadpoolOptions); + var handler = newVespaClientHandler("com.yahoo.vespa.http.server.FeedHandler", + bindingSuffix, handlerOptions, executor); cluster.addComponent(handler); - var executor = new Threadpool("feedapi-handler", cluster, options.feedApiThreadpoolOptions); - handler.inject(executor); - handler.addComponent(executor); } - private static void addRestApiHandler(ContainerCluster<?> cluster, Options options) { - var handler = newVespaClientHandler("com.yahoo.document.restapi.resource.DocumentV1ApiHandler", DOCUMENT_V1_PREFIX + "/*", options); + private static void addRestApiHandler(ContainerCluster<?> cluster, HandlerOptions handlerOptions) { + var handler = newVespaClientHandler("com.yahoo.document.restapi.resource.DocumentV1ApiHandler", + DOCUMENT_V1_PREFIX + "/*", handlerOptions, null); cluster.addComponent(handler); // We need to include a dummy implementation of the previous restapi handler (using the same class name). // The internal legacy test framework requires that the name of the old handler is listed in /ApplicationStatus. - var oldHandlerDummy = handlerComponentSpecification("com.yahoo.document.restapi.resource.RestApi"); + var oldHandlerDummy = createHandler("com.yahoo.document.restapi.resource.RestApi", null); cluster.addComponent(oldHandlerDummy); } - private static Handler<AbstractConfigProducer<?>> newVespaClientHandler( - String componentId, - String bindingSuffix, - Options options) { - Handler<AbstractConfigProducer<?>> handler = handlerComponentSpecification(componentId); - if (options.bindings.isEmpty()) { + public boolean ignoreUndefinedFields() { return ignoreUndefinedFields; } + + private static Handler newVespaClientHandler(String componentId, + String bindingSuffix, + HandlerOptions handlerOptions, + Threadpool executor) { + Handler handler = createHandler(componentId, executor); + if (handlerOptions.bindings.isEmpty()) { handler.addServerBindings( SystemBindingPattern.fromHttpPath(bindingSuffix), SystemBindingPattern.fromHttpPath(bindingSuffix + '/')); } else { - for (String rootBinding : options.bindings) { + for (String rootBinding : handlerOptions.bindings) { String pathWithoutLeadingSlash = bindingSuffix.substring(1); handler.addServerBindings( UserBindingPattern.fromPattern(rootBinding + pathWithoutLeadingSlash), @@ -67,16 +79,17 @@ public class ContainerDocumentApi { return handler; } - private static Handler<AbstractConfigProducer<?>> handlerComponentSpecification(String className) { - return new Handler<>(new ComponentModel( - BundleInstantiationSpecification.getFromStrings(className, null, "vespaclient-container-plugin"), "")); + private static Handler createHandler(String className, Threadpool executor) { + return new Handler(new ComponentModel(className, null, "vespaclient-container-plugin"), + executor); } - public static final class Options { + public static final class HandlerOptions { + private final Collection<String> bindings; private final ContainerThreadpool.UserOptions feedApiThreadpoolOptions; - public Options(Collection<String> bindings, ContainerThreadpool.UserOptions feedApiThreadpoolOptions) { + public HandlerOptions(Collection<String> bindings, ContainerThreadpool.UserOptions feedApiThreadpoolOptions) { this.bindings = Collections.unmodifiableCollection(bindings); this.feedApiThreadpoolOptions = feedApiThreadpoolOptions; } @@ -84,22 +97,15 @@ public class ContainerDocumentApi { private static class Threadpool extends ContainerThreadpool { - private final ContainerCluster<?> cluster; - - Threadpool(String name, - ContainerCluster<?> cluster, - ContainerThreadpool.UserOptions threadpoolOptions) { + Threadpool(String name, ContainerThreadpool.UserOptions threadpoolOptions) { super(name, threadpoolOptions); - this.cluster = cluster; } @Override - public void getConfig(ContainerThreadpoolConfig.Builder builder) { - super.getConfig(builder); - - // User options overrides below configuration - if (hasUserOptions()) return; - builder.maxThreads(-4).minThreads(-4).queueSize(500); + protected void setDefaultConfigValues(ContainerThreadpoolConfig.Builder builder) { + builder.maxThreads(-4) + .minThreads(-4) + .queueSize(500); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainer.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainer.java index 47dab37cc14..9997b20d205 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainer.java @@ -27,7 +27,6 @@ public final class ApplicationContainer extends Container implements private static final String defaultHostedJVMArgs = "-XX:+SuppressFatalErrorMessage"; private final boolean isHostedVespa; - private final boolean enableServerOcspStapling; public ApplicationContainer(AbstractConfigProducer<?> parent, String name, int index, DeployState deployState) { this(parent, name, false, index, deployState); @@ -36,7 +35,6 @@ public final class ApplicationContainer extends Container implements public ApplicationContainer(AbstractConfigProducer<?> parent, String name, boolean retired, int index, DeployState deployState) { super(parent, name, retired, index, deployState); this.isHostedVespa = deployState.isHosted(); - this.enableServerOcspStapling = deployState.featureFlags().enableServerOcspStapling(); addComponent(new SimpleComponent("com.yahoo.container.jdisc.messagebus.NetworkMultiplexerHolder")); addComponent(new SimpleComponent("com.yahoo.container.jdisc.messagebus.NetworkMultiplexerProvider")); @@ -68,12 +66,10 @@ public final class ApplicationContainer extends Container implements if (hasDocproc()) { b.append(ApplicationContainer.defaultHostedJVMArgs).append(' '); } - if (enableServerOcspStapling) { - b.append("-Djdk.tls.server.enableStatusRequestExtension=true ") - .append("-Djdk.tls.stapling.responseTimeout=2000 ") - .append("-Djdk.tls.stapling.cacheSize=256 ") - .append("-Djdk.tls.stapling.cacheLifetime=3600 "); - } + b.append("-Djdk.tls.server.enableStatusRequestExtension=true ") + .append("-Djdk.tls.stapling.responseTimeout=2000 ") + .append("-Djdk.tls.stapling.cacheSize=256 ") + .append("-Djdk.tls.stapling.cacheLifetime=3600 "); } String jvmArgs = super.getJvmOptions(); if (!jvmArgs.isBlank()) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java index 2b6efab3389..e316f826ad6 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java @@ -36,7 +36,6 @@ import com.yahoo.vespa.model.container.component.Component; import com.yahoo.vespa.model.container.component.Handler; import com.yahoo.vespa.model.container.component.SystemBindingPattern; import com.yahoo.vespa.model.container.configserver.ConfigserverCluster; -import com.yahoo.vespa.model.container.docproc.DocprocChains; import com.yahoo.vespa.model.utils.FileSender; import java.util.ArrayList; @@ -79,7 +78,6 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat public static final int heapSizePercentageOfTotalNodeMemory = 70; public static final int heapSizePercentageOfTotalNodeMemoryWhenCombinedCluster = 18; - private final Set<FileReference> applicationBundles = new LinkedHashSet<>(); private final Set<String> previousHosts; @@ -90,6 +88,8 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat private MbusParams mbusParams; private boolean messageBusEnabled = true; + private final int transport_events_before_wakeup; + private final int transport_connections_per_target; private Integer memoryPercentage = null; @@ -115,6 +115,8 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat addMetricsHandlers(); addTestrunnerComponentsIfTester(deployState); + transport_connections_per_target = deployState.featureFlags().mbusJavaRpcNumTargets(); + transport_events_before_wakeup = deployState.featureFlags().mbusJavaEventsBeforeWakeup(); } @Override @@ -145,7 +147,7 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat } private void addMetricsHandler(String handlerClass, BindingPattern rootBinding, BindingPattern innerBinding) { - Handler<AbstractConfigProducer<?>> handler = new Handler<>( + Handler handler = new Handler( new ComponentModel(handlerClass, null, null, null)); handler.addServerBindings(rootBinding, innerBinding); addComponent(handler); @@ -262,6 +264,8 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat } if (getDocproc() != null) getDocproc().getConfig(builder); + builder.transport_events_before_wakeup(transport_events_before_wakeup); + builder.numconnectionspertarget(transport_connections_per_target); } @Override diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java b/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java index 4b1c03a170c..3f01ac6a103 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java @@ -1,7 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.container; -import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.model.api.ModelContext; import com.yahoo.config.model.api.container.ContainerServiceType; import com.yahoo.config.model.deploy.DeployState; @@ -77,7 +76,7 @@ public abstract class Container extends AbstractService implements private final boolean dumpHeapOnShutdownTimeout; private final double shutdownTimeoutS; - private final ComponentGroup<Handler<?>> handlers = new ComponentGroup<>(this, "handler"); + private final ComponentGroup<Handler> handlers = new ComponentGroup<>(this, "handler"); private final ComponentGroup<Component<?, ?>> components = new ComponentGroup<>(this, "components"); private final JettyHttpServer defaultHttpServer; @@ -114,7 +113,7 @@ public abstract class Container extends AbstractService implements /** True if this container is retired (slated for removal) */ public boolean isRetired() { return retired; } - public ComponentGroup<Handler<?>> getHandlers() { + public ComponentGroup<Handler> getHandlers() { return handlers; } @@ -130,7 +129,7 @@ public abstract class Container extends AbstractService implements addComponent(new SimpleComponent(new ComponentModel(idSpec, classSpec, bundleSpec))); } - public final void addHandler(Handler<?> h) { + public final void addHandler(Handler h) { handlers.addComponent(h); } @@ -148,19 +147,11 @@ public abstract class Container extends AbstractService implements return (parent instanceof ContainerCluster) ? ((ContainerCluster<?>) parent).getHttp() : null; } + @SuppressWarnings("unused") // used by amenders public JettyHttpServer getDefaultHttpServer() { return defaultHttpServer; } - public JettyHttpServer getHttpServer() { - Http http = getHttp(); - if (http == null) { - return defaultHttpServer; - } else { - return http.getHttpServer().orElse(null); - } - } - /** Returns the index of this node. The index of a given node is stable through changes with best effort. */ public final int index() { return index; } @@ -216,7 +207,7 @@ public abstract class Container extends AbstractService implements } /** - * First Qrserver or container must run on ports familiar to the user. + * First container must run on ports familiar to the user. */ @Override public boolean requiresWantedPort() { @@ -381,7 +372,7 @@ public abstract class Container extends AbstractService implements @Override public void getConfig(ContainerMbusConfig.Builder builder) { - builder.enabled(messageBusEnabled()).port(getMessagingPort()); + builder.port(getMessagingPort()); } @Override diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java index 4755f674f69..7e375951c7f 100755 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java @@ -29,8 +29,8 @@ import com.yahoo.metrics.simple.runtime.MetricProperties; import com.yahoo.osgi.provider.model.ComponentModel; import com.yahoo.prelude.semantics.SemanticRulesConfig; import com.yahoo.search.config.IndexInfoConfig; -import com.yahoo.search.config.SchemaInfoConfig; import com.yahoo.search.config.QrStartConfig; +import com.yahoo.search.config.SchemaInfoConfig; import com.yahoo.search.pagetemplates.PageTemplatesConfig; import com.yahoo.search.query.profile.config.QueryProfilesConfig; import com.yahoo.vespa.configdefinition.IlscriptsConfig; @@ -64,12 +64,14 @@ import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.LinkedHashMap; -import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Optional; import java.util.Set; +import java.util.TreeSet; + +import static com.yahoo.vespa.model.container.component.chain.ProcessingHandler.PROCESSING_HANDLER_CLASS; /** * Parent class for all container cluster types. @@ -105,7 +107,7 @@ public abstract class ContainerCluster<CONTAINER extends Container> /** * URI prefix used for internal, usually programmatic, APIs. URIs using this - * prefix should never considered available for direct use by customers, and + * prefix should never be considered available for direct use by customers, and * normal compatibility concerns only applies to libraries using the URIs in * question, not contents served from the URIs themselves. */ @@ -136,11 +138,12 @@ public abstract class ContainerCluster<CONTAINER extends Container> private ContainerDocproc containerDocproc; private ContainerDocumentApi containerDocumentApi; private SecretStore secretStore; + private final ContainerThreadpool defaultHandlerThreadpool = new Handler.DefaultHandlerThreadpool(); private boolean rpcServerEnabled = true; private boolean httpServerEnabled = true; - private final Set<Path> platformBundles = new LinkedHashSet<>(); + private final Set<Path> platformBundles = new TreeSet<>(); // Ensure stable ordering private final ComponentGroup<Component<?, ?>> componentGroup; private final boolean isHostedVespa; @@ -175,6 +178,7 @@ public abstract class ContainerCluster<CONTAINER extends Container> addCommonVespaBundles(); addSimpleComponent(AccessLog.class); addComponent(new DefaultThreadpoolProvider(this, defaultPoolNumThreads)); + addComponent(defaultHandlerThreadpool); addSimpleComponent(com.yahoo.concurrent.classlock.ClassLocking.class); addSimpleComponent("com.yahoo.container.jdisc.metric.MetricConsumerProviderProvider"); addSimpleComponent("com.yahoo.container.jdisc.metric.MetricProvider"); @@ -190,6 +194,8 @@ public abstract class ContainerCluster<CONTAINER extends Container> addSimpleComponent(com.yahoo.jdisc.http.server.jetty.Janitor.class); } + protected abstract boolean messageBusEnabled(); + public ClusterSpec.Id id() { return ClusterSpec.Id.from(getName()); } public void setZone(Zone zone) { @@ -211,36 +217,45 @@ public abstract class ContainerCluster<CONTAINER extends Container> } public void addMetricStateHandler() { - Handler<AbstractConfigProducer<?>> stateHandler = new Handler<>( + Handler stateHandler = new Handler( new ComponentModel(STATE_HANDLER_CLASS, null, null, null)); stateHandler.addServerBindings(STATE_HANDLER_BINDING_1, STATE_HANDLER_BINDING_2); addComponent(stateHandler); } public void addDefaultRootHandler() { - Handler<AbstractConfigProducer<?>> handler = new Handler<>( - new ComponentModel(BundleInstantiationSpecification.getFromStrings( + Handler handler = new Handler( + new ComponentModel(BundleInstantiationSpecification.fromStrings( BINDINGS_OVERVIEW_HANDLER_CLASS, null, null), null)); // null bundle, as the handler is in container-disc handler.addServerBindings(ROOT_HANDLER_BINDING); addComponent(handler); } public void addApplicationStatusHandler() { - Handler<AbstractConfigProducer<?>> statusHandler = new Handler<>( - new ComponentModel(BundleInstantiationSpecification.getInternalHandlerSpecificationFromStrings( - APPLICATION_STATUS_HANDLER_CLASS, null), null)); + Handler statusHandler = new Handler( + new ComponentModel(BundleInstantiationSpecification.fromStrings( + APPLICATION_STATUS_HANDLER_CLASS, null, null), null)); // null bundle, as the handler is in container-disc statusHandler.addServerBindings(SystemBindingPattern.fromHttpPath("/ApplicationStatus")); addComponent(statusHandler); } public void addVipHandler() { - Handler<?> vipHandler = Handler.fromClassName(FileStatusHandlerComponent.CLASS); + Handler vipHandler = Handler.fromClassName(FileStatusHandlerComponent.CLASS); vipHandler.addServerBindings(VIP_HANDLER_BINDING); addComponent(vipHandler); } public final void addComponent(Component<?, ?> component) { componentGroup.addComponent(component); + if (component instanceof Handler handler) { + ensureHandlerHasThreadpool(handler); + } + } + + private void ensureHandlerHasThreadpool(Handler handler) { + if (! handler.hasCustomThreadPool) { + handler.inject(defaultHandlerThreadpool); + } } public final void addSimpleComponent(String idSpec, String classSpec, String bundleSpec) { @@ -253,7 +268,7 @@ public abstract class ContainerCluster<CONTAINER extends Container> * @return the removed component, or null if it was not present */ @SuppressWarnings("unused") // Used from other repositories - public Component removeComponent(ComponentId componentId) { + public Component<?, ?> removeComponent(ComponentId componentId) { return componentGroup.removeComponent(componentId); } @@ -299,10 +314,9 @@ public abstract class ContainerCluster<CONTAINER extends Container> this.processingChains = processingChains; - // Cannot use the class object for ProcessingHandler, because its superclass is not accessible ProcessingHandler<?> processingHandler = new ProcessingHandler<>( processingChains, - "com.yahoo.processing.handler.ProcessingHandler"); + BundleInstantiationSpecification.fromStrings(PROCESSING_HANDLER_CLASS, null, null)); for (BindingPattern binding: serverBindings) processingHandler.addServerBindings(binding); @@ -361,9 +375,8 @@ public abstract class ContainerCluster<CONTAINER extends Container> return containerDocproc.getChains(); } - @SuppressWarnings("unchecked") - public Collection<Handler<?>> getHandlers() { - return (Collection<Handler<?>>)(Collection)componentGroup.getComponents(Handler.class); + public Collection<Handler> getHandlers() { + return componentGroup.getComponents(Handler.class); } public void setSecretStore(SecretStore secretStore) { @@ -411,8 +424,8 @@ public abstract class ContainerCluster<CONTAINER extends Container> @Override public void getConfig(DocumentmanagerConfig.Builder builder) { - if (containerDocproc != null && containerDocproc.isCompressDocuments()) - builder.enablecompression(true); + if (containerDocumentApi != null) + builder.ignoreundefinedfields(containerDocumentApi.ignoreUndefinedFields()); } @Override @@ -435,23 +448,37 @@ public abstract class ContainerCluster<CONTAINER extends Container> @Override public void getConfig(ApplicationMetadataConfig.Builder builder) { - if (applicationMetaData != null) { + if (applicationMetaData != null) builder.name(applicationMetaData.getApplicationId().application().value()). - user(applicationMetaData.getDeployedByUser()). path(applicationMetaData.getDeployPath()). timestamp(applicationMetaData.getDeployTimestamp()). checksum(applicationMetaData.getChecksum()). generation(applicationMetaData.getGeneration()); - } } /** - * Adds the Vespa bundles that are necessary for all container types. + * Adds the Vespa bundles that are necessary for most container types. + * Note that some of these can be removed later by the individual cluster types. */ public void addCommonVespaBundles() { - PlatformBundles.commonVespaBundles().forEach(this::addPlatformBundle); + PlatformBundles.COMMON_VESPA_BUNDLES.forEach(this::addPlatformBundle); + PlatformBundles.VESPA_SECURITY_BUNDLES.forEach(this::addPlatformBundle); + PlatformBundles.VESPA_ZK_BUNDLES.forEach(this::addPlatformBundle); + } + + /** + * Add all search/docproc/feed related platform bundles. + * These are only required for application configured containers as the platform bundle set is not allowed to change + * between config generations. For standalone container platform bundles can be added on features enabled as an + * update of application package requires restart. + */ + public void addAllPlatformBundles() { + ContainerDocumentApi.addVespaClientContainerBundle(this); + addSearchAndDocprocBundles(); } + public void addSearchAndDocprocBundles() { PlatformBundles.SEARCH_AND_DOCPROC_BUNDLES.forEach(this::addPlatformBundle); } + /** * Adds a bundle present at a known location at the target container nodes. * Note that the set of platform bundles cannot change during the jdisc container's lifetime. @@ -459,9 +486,19 @@ public abstract class ContainerCluster<CONTAINER extends Container> * @param bundlePath usually an absolute path, e.g. '$VESPA_HOME/lib/jars/foo.jar' */ public final void addPlatformBundle(Path bundlePath) { - platformBundles.add(bundlePath); + if (! unnecessaryPlatformBundles().contains(bundlePath)) { + platformBundles.add(bundlePath); + } else { + log.fine(() -> "Not installing bundle " + bundlePath + " for cluster " + getName()); + } } + /** + * Implement in subclasses to avoid installing unnecessary bundles, see {@link PlatformBundles} + * Should only return constant values, as there is no guarantee for when this is called. + */ + protected Set<Path> unnecessaryPlatformBundles() { return Set.of(); } + @Override public void getConfig(PlatformBundlesConfig.Builder builder) { platformBundles.stream() @@ -536,9 +573,7 @@ public abstract class ContainerCluster<CONTAINER extends Container> @Override public void getConfig(IlscriptsConfig.Builder builder) { - List<SearchCluster> searchClusters = new ArrayList<>(); - searchClusters.addAll(Content.getSearchClusters(getRoot().configModelRepo())); - for (SearchCluster searchCluster : searchClusters) { + for (SearchCluster searchCluster : Content.getSearchClusters(getRoot().configModelRepo())) { searchCluster.getConfig(builder); } } @@ -610,8 +645,6 @@ public abstract class ContainerCluster<CONTAINER extends Container> public void setEnvironmentVars(String environmentVars) { this.environmentVars = environmentVars; } - public String getEnvironmentVars() { return environmentVars; } - public Optional<String> getJvmGCOptions() { return Optional.ofNullable(jvmGCOptions); } public final void setRpcServerEnabled(boolean rpcServerEnabled) { this.rpcServerEnabled = rpcServerEnabled; } @@ -627,8 +660,6 @@ public abstract class ContainerCluster<CONTAINER extends Container> return "container cluster '" + getName() + "'"; } - protected abstract boolean messageBusEnabled(); - /** * Mark whether the config emitted by this cluster currently should be applied by clients already running with * a previous generation of it only by restarting the consuming processes. diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerModelEvaluation.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerModelEvaluation.java index cb8e6ba85ff..088465f56b1 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerModelEvaluation.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerModelEvaluation.java @@ -63,8 +63,8 @@ public class ContainerModelEvaluation implements rankProfileList.getConfig(builder); } - public static Handler<?> getHandler() { - Handler<?> handler = new Handler<>(new ComponentModel(REST_HANDLER_NAME, null, EVALUATION_BUNDLE_NAME)); + public static Handler getHandler() { + Handler handler = new Handler(new ComponentModel(REST_HANDLER_NAME, null, EVALUATION_BUNDLE_NAME)); handler.addServerBindings( SystemBindingPattern.fromHttpPath(REST_BINDING_PATH), SystemBindingPattern.fromHttpPath(REST_BINDING_PATH + "/*")); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerThreadpool.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerThreadpool.java index 489e4cc135a..fb4e62f5cd1 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerThreadpool.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerThreadpool.java @@ -17,14 +17,14 @@ import java.util.Optional; * * @author bjorncs */ -public class ContainerThreadpool extends SimpleComponent implements ContainerThreadpoolConfig.Producer { +public abstract class ContainerThreadpool extends SimpleComponent implements ContainerThreadpoolConfig.Producer { private final String name; private final UserOptions userOptions; public ContainerThreadpool(String name, UserOptions userOptions) { super(new ComponentModel( - BundleInstantiationSpecification.getFromStrings( + BundleInstantiationSpecification.fromStrings( "threadpool@" + name, ContainerThreadpoolImpl.class.getName(), null))); @@ -32,8 +32,13 @@ public class ContainerThreadpool extends SimpleComponent implements ContainerThr this.userOptions = userOptions; } + // Must be implemented by subclasses to set values that may be overridden by user options. + protected abstract void setDefaultConfigValues(ContainerThreadpoolConfig.Builder builder); + @Override public void getConfig(ContainerThreadpoolConfig.Builder builder) { + setDefaultConfigValues(builder); + builder.name(this.name); if (userOptions != null) { builder.maxThreads(userOptions.maxThreads); @@ -42,9 +47,6 @@ public class ContainerThreadpool extends SimpleComponent implements ContainerThr } } - protected Optional<UserOptions> userOptions() { return Optional.ofNullable(userOptions); } - protected boolean hasUserOptions() { return userOptions().isPresent(); } - public static class UserOptions { private final int maxThreads; private final int minThreads; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/DefaultThreadpoolProvider.java b/config-model/src/main/java/com/yahoo/vespa/model/container/DefaultThreadpoolProvider.java index 0b37abaded9..0fdd36b8811 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/DefaultThreadpoolProvider.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/DefaultThreadpoolProvider.java @@ -19,7 +19,7 @@ class DefaultThreadpoolProvider extends SimpleComponent implements ThreadpoolCon DefaultThreadpoolProvider(ContainerCluster<?> cluster, int defaultWorkerThreads) { super(new ComponentModel( - BundleInstantiationSpecification.getFromStrings( + BundleInstantiationSpecification.fromStrings( "default-threadpool", ThreadPoolProvider.class.getName(), null))); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/IdentityProvider.java b/config-model/src/main/java/com/yahoo/vespa/model/container/IdentityProvider.java index 7ce2425179b..5e8bb85c29d 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/IdentityProvider.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/IdentityProvider.java @@ -33,7 +33,7 @@ public class IdentityProvider extends SimpleComponent implements IdentityConfig. URI ztsUrl, String athenzDnsSuffix, Zone zone) { - super(new ComponentModel(BundleInstantiationSpecification.getFromStrings(CLASS, CLASS, BUNDLE))); + super(new ComponentModel(BundleInstantiationSpecification.fromStrings(CLASS, CLASS, BUNDLE))); this.domain = domain; this.service = service; this.loadBalancerName = loadBalancerName; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/PlatformBundles.java b/config-model/src/main/java/com/yahoo/vespa/model/container/PlatformBundles.java index e5125fe7e1d..7ce82848d09 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/PlatformBundles.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/PlatformBundles.java @@ -1,16 +1,18 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.container; +import com.yahoo.container.bundle.BundleInstantiationSpecification; import com.yahoo.vespa.defaults.Defaults; import java.nio.file.Path; import java.nio.file.Paths; -import java.util.Collections; -import java.util.LinkedHashSet; -import java.util.List; import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; /** + * NOTE: Stable ordering of bundles in config is handled by {@link ContainerCluster#addPlatformBundle(Path)} + * * @author gjoranv * @author Ulf Lilleengen */ @@ -28,12 +30,37 @@ public class PlatformBundles { } public static final Path LIBRARY_PATH = Paths.get(Defaults.getDefaults().underVespaHome("lib/jars")); - public static final String searchAndDocprocBundle = "container-search-and-docproc"; + public static final String SEARCH_AND_DOCPROC_BUNDLE = BundleInstantiationSpecification.CONTAINER_SEARCH_AND_DOCPROC; - public static Set<Path> commonVespaBundles() { - var bundles = new LinkedHashSet<Path>(); - commonVespaBundles.stream().map(PlatformBundles::absoluteBundlePath).forEach(bundles::add); - return Collections.unmodifiableSet(bundles); + // Bundles that must be loaded for all container types. + public static final Set<Path> COMMON_VESPA_BUNDLES = toBundlePaths( + "container-spifly.jar", // Aries SPIFly repackaged + // Used by vespa-athenz, zkfacade, other vespa bundles and nearly all hosted apps. + // TODO Vespa 9: stop installing and providing servlet-api. Seems difficult, though. + "javax.servlet-api-3.1.0.jar" + ); + + public static final Set<Path> VESPA_SECURITY_BUNDLES = toBundlePaths( + "jdisc-security-filters", + "vespa-athenz" + ); + + public static final Set<Path> VESPA_ZK_BUNDLES = toBundlePaths( + "zkfacade", + "zookeeper-server" + ); + + public static final Set<Path> SEARCH_AND_DOCPROC_BUNDLES = toBundlePaths( + SEARCH_AND_DOCPROC_BUNDLE, + "container-search-gui", + "docprocs", + "linguistics-components" + ); + + private static Set<Path> toBundlePaths(String... bundleNames) { + return Stream.of(bundleNames) + .map(PlatformBundles::absoluteBundlePath) + .collect(Collectors.toSet()); } public static Path absoluteBundlePath(String fileName) { @@ -42,19 +69,14 @@ public class PlatformBundles { public static Path absoluteBundlePath(String fileName, JarSuffix jarSuffix) { if (fileName == null) return null; - return LIBRARY_PATH.resolve(Paths.get(fileName + jarSuffix.suffix)); + String fullFilename = fileName.endsWith(".jar") ? fileName : fileName + jarSuffix.suffix; + return LIBRARY_PATH.resolve(Paths.get(fullFilename)); } public static boolean isSearchAndDocprocClass(String className) { return searchAndDocprocComponents.contains(className); } - // Bundles that must be loaded for all container types. - private static final List<String> commonVespaBundles = List.of( - "zkfacade", - "zookeeper-server" // TODO: not necessary in metrics-proxy. - ); - // This is a hack to allow users to declare components from the search-and-docproc bundle without naming the bundle. private static final Set<String> searchAndDocprocComponents = Set.of( "com.yahoo.docproc.AbstractConcreteDocumentFactory", diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/component/DiscBindingsConfigGenerator.java b/config-model/src/main/java/com/yahoo/vespa/model/container/component/DiscBindingsConfigGenerator.java index 76124b14209..e4a5c2cd440 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/component/DiscBindingsConfigGenerator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/component/DiscBindingsConfigGenerator.java @@ -14,16 +14,16 @@ import static java.util.stream.Collectors.toList; */ public class DiscBindingsConfigGenerator { - public static Map<String, Handlers.Builder> generate(Collection<? extends Handler<?>> handlers) { + public static Map<String, Handlers.Builder> generate(Collection<? extends Handler> handlers) { Map<String, Handlers.Builder> handlerBuilders = new LinkedHashMap<>(); - for (Handler<?> handler : handlers) { + for (Handler handler : handlers) { handlerBuilders.putAll(generate(handler)); } return handlerBuilders; } - public static <T extends Handler<?>> Map<String, Handlers.Builder> generate(T handler) { + public static <T extends Handler> Map<String, Handlers.Builder> generate(T handler) { if (handler.getServerBindings().isEmpty() && handler.getClientBindings().isEmpty()) return Collections.emptyMap(); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/component/Handler.java b/config-model/src/main/java/com/yahoo/vespa/model/container/component/Handler.java index 8ffdccae896..9f2bfe9251b 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/component/Handler.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/component/Handler.java @@ -1,8 +1,9 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.container.component; -import com.yahoo.config.model.producer.AbstractConfigProducer; +import com.yahoo.container.handler.threadpool.ContainerThreadpoolConfig; import com.yahoo.osgi.provider.model.ComponentModel; +import com.yahoo.vespa.model.container.ContainerThreadpool; import java.util.ArrayList; import java.util.Arrays; @@ -15,22 +16,35 @@ import java.util.Set; * Models a jdisc RequestHandler (including ClientProvider). * RequestHandlers always have at least one server binding, * while ClientProviders have at least one client binding. - * <p> - * Note that this is also used to model vespa handlers (which do not have any bindings) * * @author gjoranv */ -public class Handler<CHILD extends AbstractConfigProducer<?>> extends Component<CHILD, ComponentModel> { +public class Handler extends Component<Component<?, ?>, ComponentModel> { private final Set<BindingPattern> serverBindings = new LinkedHashSet<>(); private final List<BindingPattern> clientBindings = new ArrayList<>(); + public final boolean hasCustomThreadPool; + public Handler(ComponentModel model) { + this(model, null); + } + + public Handler(ComponentModel model, ContainerThreadpool threadpool) { super(model); + + // The default threadpool is always added to the cluster, so cannot be added here. + if (threadpool != null) { + hasCustomThreadPool = true; + addComponent(threadpool); + inject(threadpool); + } else { + hasCustomThreadPool = false; + } } - public static Handler<AbstractConfigProducer<?>> fromClassName(String className) { - return new Handler<>(new ComponentModel(className, null, null, null)); + public static Handler fromClassName(String className) { + return new Handler(new ComponentModel(className, null, null, null)); } public void addServerBindings(BindingPattern... bindings) { @@ -53,4 +67,24 @@ public class Handler<CHILD extends AbstractConfigProducer<?>> extends Component< return Collections.unmodifiableList(clientBindings); } + + /** + * The default threadpool for all handlers, except those that declare their own, e.g. SearchHandler. + */ + public static class DefaultHandlerThreadpool extends ContainerThreadpool { + + public DefaultHandlerThreadpool() { + super("default-handler-common", null); + } + + @Override + public void setDefaultConfigValues(ContainerThreadpoolConfig.Builder builder) { + builder.maxThreadExecutionTimeSeconds(190) + .keepAliveTime(5.0) + .maxThreads(-2) + .minThreads(-2) + .queueSize(-40); + } + } + } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/component/SimpleComponent.java b/config-model/src/main/java/com/yahoo/vespa/model/container/component/SimpleComponent.java index 2c4dd9a8dda..23915afacdd 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/component/SimpleComponent.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/component/SimpleComponent.java @@ -17,7 +17,7 @@ public class SimpleComponent extends Component<AbstractConfigProducer<?>, Compon } public SimpleComponent(String className) { - this(new ComponentModel(BundleInstantiationSpecification.getFromStrings(className, null, null))); + this(new ComponentModel(BundleInstantiationSpecification.fromStrings(className, null, null))); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/component/chain/ProcessingHandler.java b/config-model/src/main/java/com/yahoo/vespa/model/container/component/chain/ProcessingHandler.java index 71524d7e157..897a1f22f30 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/component/chain/ProcessingHandler.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/component/chain/ProcessingHandler.java @@ -4,7 +4,7 @@ package com.yahoo.vespa.model.container.component.chain; import com.yahoo.container.bundle.BundleInstantiationSpecification; import com.yahoo.container.core.ChainsConfig; import com.yahoo.osgi.provider.model.ComponentModel; -import com.yahoo.config.model.producer.AbstractConfigProducer; +import com.yahoo.vespa.model.container.ContainerThreadpool; import com.yahoo.vespa.model.container.component.Handler; @@ -14,15 +14,22 @@ import com.yahoo.vespa.model.container.component.Handler; * @author gjoranv */ public class ProcessingHandler<CHAINS extends Chains<?>> - extends Handler<AbstractConfigProducer<?>> + extends Handler implements ChainsConfig.Producer { + // Cannot use the class object for ProcessingHandler, because its superclass is not accessible + public static final String PROCESSING_HANDLER_CLASS = "com.yahoo.processing.handler.ProcessingHandler"; + protected final CHAINS chains; - public ProcessingHandler(CHAINS chains, String handlerClass) { - super(new ComponentModel(BundleInstantiationSpecification.getInternalProcessingSpecificationFromStrings(handlerClass, null), null)); - this.chains = chains; + // Create a handler that uses the default threadpool for handlers + public ProcessingHandler(CHAINS chains, BundleInstantiationSpecification spec) { + this(chains, spec, null); + } + public ProcessingHandler(CHAINS chains, BundleInstantiationSpecification spec, ContainerThreadpool threadpool) { + super(new ComponentModel(spec), threadpool); + this.chains = chains; } @Override diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/docproc/ContainerDocproc.java b/config-model/src/main/java/com/yahoo/vespa/model/container/docproc/ContainerDocproc.java index 8be02f77ab3..242f7cbefdb 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/docproc/ContainerDocproc.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/docproc/ContainerDocproc.java @@ -15,41 +15,32 @@ import java.util.HashMap; import java.util.Map; /** - * @author einarmr + * @author Einar M R Rosenvinge * @author gjoranv */ -public class ContainerDocproc extends ContainerSubsystem<DocprocChains> - implements - ContainerMbusConfig.Producer, - SchemamappingConfig.Producer, - DocprocConfig.Producer -{ - public final Options options; - - // Whether or not to prefer sending to a local node. - private boolean preferLocalNode = false; +public class ContainerDocproc extends ContainerSubsystem<DocprocChains> implements + ContainerMbusConfig.Producer, + SchemamappingConfig.Producer, + DocprocConfig.Producer { - // The number of nodes to use per client. - private int numNodesPerClient = 0; - - private Map<Pair<String, String>, String> fieldNameSchemaMap = new HashMap<>(); + public final Options options; + private final Map<Pair<String, String>, String> fieldNameSchemaMap = new HashMap<>(); - public ContainerDocproc(ContainerCluster cluster, DocprocChains chains) { - this(cluster, chains, new Options(false, null, null, null, null, null, null)); + public ContainerDocproc(ContainerCluster<?> cluster, DocprocChains chains) { + this(cluster, chains, Options.empty()); } - public ContainerDocproc(ContainerCluster cluster, DocprocChains chains, Options options) { + public ContainerDocproc(ContainerCluster<?> cluster, DocprocChains chains, Options options) { this(cluster, chains, options, true); } - private void addSource( - final ContainerCluster cluster, final String name, final SessionConfig.Type.Enum type) { + private void addSource(ContainerCluster<?> cluster, String name, SessionConfig.Type.Enum type) { final MbusClient mbusClient = new MbusClient(name, type); mbusClient.addClientBindings(SystemBindingPattern.fromPattern("mbus://*/" + mbusClient.getSessionName())); cluster.addComponent(mbusClient); } - public ContainerDocproc(ContainerCluster cluster, DocprocChains chains, Options options, boolean addSourceClientProvider) { + public ContainerDocproc(ContainerCluster<?> cluster, DocprocChains chains, Options options, boolean addSourceClientProvider) { super(chains); assert (options != null) : "Null Options for " + this + " under cluster " + cluster.getName(); this.options = options; @@ -58,25 +49,12 @@ public class ContainerDocproc extends ContainerSubsystem<DocprocChains> addSource(cluster, "source", SessionConfig.Type.SOURCE); addSource(cluster, MbusRequestContext.internalNoThrottledSource, SessionConfig.Type.INTERNAL); } - } - - public boolean isCompressDocuments() { - return options.compressDocuments; - } - - public boolean isPreferLocalNode() { - return preferLocalNode; - } - - public int getNumNodesPerClient() { - return numNodesPerClient; + cluster.addSearchAndDocprocBundles(); } @Override public void getConfig(ContainerMbusConfig.Builder builder) { builder.maxpendingcount(getMaxMessagesInQueue()); - if (getMaxQueueMbSize() != null) - builder.maxpendingsize(getMaxQueueMbSize()); //yes, this shall be set in megabytes. } private int getMaxMessagesInQueue() { @@ -84,13 +62,9 @@ public class ContainerDocproc extends ContainerSubsystem<DocprocChains> return options.maxMessagesInQueue; } - //maxmessagesinqueue has not been set for this node. let's try to give a good value anyway: + // maxmessagesinqueue has not been set for this node. let's try to give a good value anyway: return 2048 * getChains().allChains().allComponents().size(); - //intentionally high, getMaxQueueMbSize() will probably kick in before this one! - } - - private Integer getMaxQueueMbSize() { - return options.maxQueueMbSize; + // intentionally high, getMaxQueueMbSize() will probably kick in before this one! } private Integer getMaxQueueTimeMs() { @@ -137,26 +111,24 @@ public class ContainerDocproc extends ContainerSubsystem<DocprocChains> } public static class Options { - // Whether or not to compress documents after processing them. - public final boolean compressDocuments; public final Integer maxMessagesInQueue; - public final Integer maxQueueMbSize; public final Integer maxQueueTimeMs; public final Double maxConcurrentFactor; public final Double documentExpansionFactor; public final Integer containerCoreMemory; - public Options(boolean compressDocuments, Integer maxMessagesInQueue, Integer maxQueueMbSize, Integer maxQueueTimeMs, Double maxConcurrentFactor, Double documentExpansionFactor, Integer containerCoreMemory) { - this.compressDocuments = compressDocuments; + public Options(Integer maxMessagesInQueue, Integer maxQueueTimeMs, Double maxConcurrentFactor, Double documentExpansionFactor, Integer containerCoreMemory) { this.maxMessagesInQueue = maxMessagesInQueue; - this.maxQueueMbSize = maxQueueMbSize; this.maxQueueTimeMs = maxQueueTimeMs; this.maxConcurrentFactor = maxConcurrentFactor; this.documentExpansionFactor = documentExpansionFactor; this.containerCoreMemory = containerCoreMemory; } + + static Options empty() { return new Options(null, null, null, null, null); } + } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/docproc/DocprocChains.java b/config-model/src/main/java/com/yahoo/vespa/model/container/docproc/DocprocChains.java index 4b9897d0950..109ab3e806e 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/docproc/DocprocChains.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/docproc/DocprocChains.java @@ -3,9 +3,13 @@ package com.yahoo.vespa.model.container.docproc; import com.yahoo.component.ComponentId; import com.yahoo.config.model.producer.AbstractConfigProducer; +import com.yahoo.container.bundle.BundleInstantiationSpecification; import com.yahoo.container.jdisc.config.SessionConfig; +import com.yahoo.docproc.jdisc.observability.DocprocsStatusExtension; +import com.yahoo.osgi.provider.model.ComponentModel; import com.yahoo.vespa.model.container.ApplicationContainerCluster; import com.yahoo.vespa.model.container.ContainerCluster; +import com.yahoo.vespa.model.container.PlatformBundles; import com.yahoo.vespa.model.container.component.Component; import com.yahoo.vespa.model.container.component.SimpleComponent; import com.yahoo.vespa.model.container.component.SystemBindingPattern; @@ -23,8 +27,12 @@ public class DocprocChains extends Chains<DocprocChain> { public DocprocChains(AbstractConfigProducer<?> parent, String subId) { super(parent, subId); - docprocHandler = new ProcessingHandler<>(this, "com.yahoo.docproc.jdisc.DocumentProcessingHandler"); + docprocHandler = new ProcessingHandler<>( + this, + BundleInstantiationSpecification.fromSearchAndDocproc("com.yahoo.docproc.jdisc.DocumentProcessingHandler")); addComponent(docprocHandler); + addComponent(new SimpleComponent( + new ComponentModel(DocprocsStatusExtension.class.getName(), null, PlatformBundles.SEARCH_AND_DOCPROC_BUNDLE))); if (! (getParent() instanceof ApplicationContainerCluster)) { // All application containers already have a DocumentTypeManager, diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/docproc/MbusClient.java b/config-model/src/main/java/com/yahoo/vespa/model/container/docproc/MbusClient.java index 28a0748be26..0efcd8df37f 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/docproc/MbusClient.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/docproc/MbusClient.java @@ -6,13 +6,12 @@ import com.yahoo.component.ComponentSpecification; import com.yahoo.container.jdisc.config.SessionConfig; import com.yahoo.container.bundle.BundleInstantiationSpecification; import com.yahoo.osgi.provider.model.ComponentModel; -import com.yahoo.config.model.producer.AbstractConfigProducer; import com.yahoo.vespa.model.container.component.Handler; /** * @author Einar M R Rosenvinge */ -public class MbusClient extends Handler<AbstractConfigProducer<?>> implements SessionConfig.Producer { +public class MbusClient extends Handler implements SessionConfig.Producer { private static final ComponentSpecification CLASSNAME = ComponentSpecification.fromString("com.yahoo.container.jdisc.messagebus.MbusClientProvider"); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/AccessControl.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/AccessControl.java index d85f00a5bb2..3ee0414bf32 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/http/AccessControl.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/AccessControl.java @@ -51,7 +51,7 @@ public class AccessControl { private final String domain; private ClientAuthentication clientAuthentication = ClientAuthentication.need; private final Set<BindingPattern> excludeBindings = new LinkedHashSet<>(); - private Collection<Handler<?>> handlers = Collections.emptyList(); + private Collection<Handler> handlers = Collections.emptyList(); public Builder(String domain) { this.domain = domain; } @@ -79,12 +79,12 @@ public class AccessControl { public final String domain; public final ClientAuthentication clientAuthentication; private final Set<BindingPattern> excludedBindings; - private final Collection<Handler<?>> handlers; + private final Collection<Handler> handlers; private AccessControl(String domain, ClientAuthentication clientAuthentication, Set<BindingPattern> excludedBindings, - Collection<Handler<?>> handlers) { + Collection<Handler> handlers) { this.domain = domain; this.clientAuthentication = clientAuthentication; this.excludedBindings = Collections.unmodifiableSet(excludedBindings); @@ -119,7 +119,7 @@ public class AccessControl { public Set<BindingPattern> excludedBindings() { return excludedBindings; } /** all handlers (that are known by the access control components) **/ - public Collection<Handler<?>> handlers() { return handlers; } + public Collection<Handler> handlers() { return handlers; } public static boolean hasHandlerThatNeedsProtection(ApplicationContainerCluster cluster) { return cluster.getHandlers().stream() @@ -135,7 +135,7 @@ public class AccessControl { for (BindingPattern excludedBinding : excludedBindings) { http.getBindings().add(createAccessControlExcludedBinding(excludedBinding)); } - for (Handler<?> handler : handlers) { + for (Handler handler : handlers) { if (isExcludedHandler(handler)) { for (BindingPattern binding : handler.getServerBindings()) { http.getBindings().add(createAccessControlExcludedBinding(binding)); @@ -188,9 +188,9 @@ public class AccessControl { private static Chain<Filter> createChain(ComponentId id) { return new Chain<>(FilterChains.emptyChainSpec(id)); } - private static boolean isExcludedHandler(Handler<?> handler) { return EXCLUDED_HANDLERS.contains(handler.getClassId().getName()); } + private static boolean isExcludedHandler(Handler handler) { return EXCLUDED_HANDLERS.contains(handler.getClassId().getName()); } - private static boolean hasNonMbusBinding(Handler<?> handler) { + private static boolean hasNonMbusBinding(Handler handler) { return handler.getServerBindings().stream().anyMatch(binding -> ! binding.scheme().equals("mbus")); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/BlockFeedGlobalEndpointsFilter.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/BlockFeedGlobalEndpointsFilter.java index 167dac4c57e..039daba8ad0 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/http/BlockFeedGlobalEndpointsFilter.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/BlockFeedGlobalEndpointsFilter.java @@ -7,7 +7,7 @@ import com.yahoo.component.chain.dependencies.Dependencies; import com.yahoo.component.chain.model.ChainedComponentModel; import com.yahoo.config.model.api.ContainerEndpoint; import com.yahoo.container.bundle.BundleInstantiationSpecification; -import com.yahoo.jdisc.http.filter.security.rule.RuleBasedFilterConfig; +import com.yahoo.vespa.config.jdisc.http.filter.RuleBasedFilterConfig; import com.yahoo.vespa.model.clients.ContainerDocumentApi; import com.yahoo.vespa.model.container.ContainerCluster; @@ -17,11 +17,11 @@ import java.util.List; import java.util.Set; import java.util.stream.Collectors; -import static com.yahoo.jdisc.http.filter.security.rule.RuleBasedFilterConfig.DefaultRule.Action.Enum.ALLOW; -import static com.yahoo.jdisc.http.filter.security.rule.RuleBasedFilterConfig.Rule.Action.Enum.BLOCK; -import static com.yahoo.jdisc.http.filter.security.rule.RuleBasedFilterConfig.Rule.Methods.Enum.DELETE; -import static com.yahoo.jdisc.http.filter.security.rule.RuleBasedFilterConfig.Rule.Methods.Enum.POST; -import static com.yahoo.jdisc.http.filter.security.rule.RuleBasedFilterConfig.Rule.Methods.Enum.PUT; +import static com.yahoo.vespa.config.jdisc.http.filter.RuleBasedFilterConfig.DefaultRule.Action.Enum.ALLOW; +import static com.yahoo.vespa.config.jdisc.http.filter.RuleBasedFilterConfig.Rule.Action.Enum.BLOCK; +import static com.yahoo.vespa.config.jdisc.http.filter.RuleBasedFilterConfig.Rule.Methods.Enum.DELETE; +import static com.yahoo.vespa.config.jdisc.http.filter.RuleBasedFilterConfig.Rule.Methods.Enum.POST; +import static com.yahoo.vespa.config.jdisc.http.filter.RuleBasedFilterConfig.Rule.Methods.Enum.PUT; /** * @author mortent diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/FilterChains.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/FilterChains.java index bc9a45da08a..476c6249a1e 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/http/FilterChains.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/FilterChains.java @@ -11,6 +11,7 @@ import com.yahoo.vespa.model.container.component.chain.Chain; import com.yahoo.vespa.model.container.component.chain.Chains; import java.util.Collections; +import java.util.Set; /** * @author Tony Vaagenes @@ -44,8 +45,8 @@ public class FilterChains extends Chains<Chain<Filter>> { public static ChainSpecification emptyChainSpec(ComponentId chainId) { return new ChainSpecification(chainId, new ChainSpecification.Inheritance(null, null), - Collections.<Phase>emptySet(), - Collections.<ComponentSpecification>emptySet()); + Set.of(), + Set.of()); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java index 97c3f85dced..7a0b6c8e023 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java @@ -5,6 +5,7 @@ import com.yahoo.container.QrSearchersConfig; import com.yahoo.prelude.semantics.SemanticRulesConfig; import com.yahoo.search.config.IndexInfoConfig; import com.yahoo.search.config.SchemaInfoConfig; +import com.yahoo.search.handler.observability.SearchStatusExtension; import com.yahoo.search.pagetemplates.PageTemplatesConfig; import com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry; import com.yahoo.search.query.profile.config.QueryProfilesConfig; @@ -24,7 +25,7 @@ import java.util.LinkedList; import java.util.List; import java.util.Map; -import static com.yahoo.vespa.model.container.PlatformBundles.searchAndDocprocBundle; +import static com.yahoo.vespa.model.container.PlatformBundles.SEARCH_AND_DOCPROC_BUNDLE; /** * @author gjoranv @@ -55,8 +56,10 @@ public class ContainerSearch extends ContainerSubsystem<SearchChains> this.owningCluster = cluster; this.options = options; - owningCluster.addComponent(Component.fromClassAndBundle(QUERY_PROFILE_REGISTRY_CLASS, searchAndDocprocBundle)); - owningCluster.addComponent(Component.fromClassAndBundle(com.yahoo.search.schema.SchemaInfo.class.getName(), searchAndDocprocBundle)); + owningCluster.addComponent(Component.fromClassAndBundle(QUERY_PROFILE_REGISTRY_CLASS, SEARCH_AND_DOCPROC_BUNDLE)); + owningCluster.addComponent(Component.fromClassAndBundle(com.yahoo.search.schema.SchemaInfo.class.getName(), SEARCH_AND_DOCPROC_BUNDLE)); + owningCluster.addComponent(Component.fromClassAndBundle(SearchStatusExtension.class.getName(), SEARCH_AND_DOCPROC_BUNDLE)); + cluster.addSearchAndDocprocBundles(); } public void connectSearchClusters(Map<String, SearchCluster> searchClusters) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/DispatcherComponent.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/DispatcherComponent.java index 6edcd8901de..44c60cf0619 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/search/DispatcherComponent.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/DispatcherComponent.java @@ -32,7 +32,7 @@ public class DispatcherComponent extends Component<AbstractConfigProducer<?>, Co String dispatcherComponentId = "dispatcher." + indexedSearchCluster.getClusterName(); // used by ClusterSearcher return new ComponentModel(dispatcherComponentId, com.yahoo.search.dispatch.Dispatcher.class.getName(), - PlatformBundles.searchAndDocprocBundle); + PlatformBundles.SEARCH_AND_DOCPROC_BUNDLE); } @Override diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/GUIHandler.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/GUIHandler.java index 7087cabafc1..b00faa80715 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/search/GUIHandler.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/GUIHandler.java @@ -1,7 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.container.search; -import com.yahoo.config.model.producer.AbstractConfigProducer; import com.yahoo.container.bundle.BundleInstantiationSpecification; import com.yahoo.osgi.provider.model.ComponentModel; import com.yahoo.vespa.model.container.component.Handler; @@ -10,7 +9,7 @@ import com.yahoo.vespa.model.container.component.Handler; /** * @author Henrik Høiness */ -public class GUIHandler extends Handler<AbstractConfigProducer<?>> { +public class GUIHandler extends Handler { public static final String BUNDLE = "container-search-gui"; public static final String CLASS = "com.yahoo.search.query.gui.GUIHandler"; @@ -21,7 +20,7 @@ public class GUIHandler extends Handler<AbstractConfigProducer<?>> { } public static BundleInstantiationSpecification bundleSpec(String className, String bundle) { - return BundleInstantiationSpecification.getFromStrings(className, className, bundle); + return BundleInstantiationSpecification.fromStrings(className, className, bundle); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/RpcResourcePoolComponent.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/RpcResourcePoolComponent.java index 956d551f6b3..b4b2af3c808 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/search/RpcResourcePoolComponent.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/RpcResourcePoolComponent.java @@ -13,6 +13,6 @@ public class RpcResourcePoolComponent extends Component<RpcResourcePoolComponent private static ComponentModel toComponentModel(String clusterName) { String componentId = "rpcresourcepool." + clusterName; - return new ComponentModel(componentId, com.yahoo.search.dispatch.rpc.RpcResourcePool.class.getName(), PlatformBundles.searchAndDocprocBundle); + return new ComponentModel(componentId, com.yahoo.search.dispatch.rpc.RpcResourcePool.class.getName(), PlatformBundles.SEARCH_AND_DOCPROC_BUNDLE); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/FederationSearcher.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/FederationSearcher.java index 1abb62fedab..799309b8ca1 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/FederationSearcher.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/FederationSearcher.java @@ -45,7 +45,7 @@ public class FederationSearcher extends Searcher<FederationSearcherModel> implem this.documentTypes = documentTypes; } - public FederationConfig.Target.SearchChain.Builder getSearchChainConfig() { + FederationConfig.Target.SearchChain.Builder getSearchChainConfig() { FederationConfig.Target.SearchChain.Builder sB = new FederationConfig.Target.SearchChain.Builder(); FederationOptions resolvedOptions = targetOptions.inherit(searchChain.federationOptions()); sB. @@ -70,12 +70,12 @@ public class FederationSearcher extends Searcher<FederationSearcherModel> implem final ComponentId id; final FederationOptions targetOptions; - public Target(ComponentId id, FederationOptions targetOptions) { + Target(ComponentId id, FederationOptions targetOptions) { this.id = id; this.targetOptions = targetOptions; } - public FederationConfig.Target.Builder getTargetConfig() { + FederationConfig.Target.Builder getTargetConfig() { FederationConfig.Target.Builder tb = new FederationConfig.Target.Builder(); tb. id(id.stringValue()). @@ -92,7 +92,7 @@ public class FederationSearcher extends Searcher<FederationSearcherModel> implem private final SearchChainConfig searchChainConfig; - public SearchChainTarget(SearchChain searchChain, FederationOptions targetOptions) { + SearchChainTarget(SearchChain searchChain, FederationOptions targetOptions) { super(searchChain.getComponentId(), targetOptions); searchChainConfig = new SearchChainConfig(searchChain, null, targetOptions, searchChain.getDocumentTypes()); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/defaultsearchchains/LocalClustersCreator.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/defaultsearchchains/LocalClustersCreator.java index 229afd56360..0307d5d5774 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/defaultsearchchains/LocalClustersCreator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/defaultsearchchains/LocalClustersCreator.java @@ -2,15 +2,12 @@ package com.yahoo.vespa.model.container.search.searchchain.defaultsearchchains; import com.yahoo.component.ComponentId; -import com.yahoo.component.ComponentSpecification; -import com.yahoo.component.chain.Phase; import com.yahoo.component.chain.model.ChainSpecification; import com.yahoo.search.searchchain.model.federation.FederationOptions; import com.yahoo.search.searchchain.model.federation.LocalProviderSpec; import com.yahoo.vespa.model.container.search.searchchain.LocalProvider; import com.yahoo.vespa.model.container.search.searchchain.SearchChains; -import java.util.Collections; import java.util.LinkedHashSet; import java.util.List; import java.util.Set; @@ -21,20 +18,20 @@ import java.util.Set; */ public class LocalClustersCreator { - static ChainSpecification emptySearchChainSpecification(String componentName) { + private static ChainSpecification emptySearchChainSpecification(String componentName) { return new ChainSpecification(new ComponentId(componentName), VespaSearchChainsCreator.inheritsVespaPhases(), //TODO: refactor List.of(), Set.of()); } - static LocalProvider createDefaultLocalProvider(String clusterName) { + private static LocalProvider createDefaultLocalProvider(String clusterName) { return new LocalProvider(emptySearchChainSpecification(clusterName), new FederationOptions(), new LocalProviderSpec(clusterName)); } - static Set<String> presentClusters(SearchChains searchChains) { + private static Set<String> presentClusters(SearchChains searchChains) { Set<String> presentClusters = new LinkedHashSet<>(); for (LocalProvider provider : searchChains.localProviders()) { presentClusters.add(provider.getClusterName()); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/defaultsearchchains/VespaSearchChainsCreator.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/defaultsearchchains/VespaSearchChainsCreator.java index eb396c52fca..d9c3beaaf0e 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/defaultsearchchains/VespaSearchChainsCreator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/defaultsearchchains/VespaSearchChainsCreator.java @@ -9,10 +9,18 @@ import com.yahoo.component.chain.model.ChainedComponentModel; import com.yahoo.search.searchchain.PhaseNames; import com.yahoo.search.searchchain.model.VespaSearchers; import com.yahoo.search.searchchain.model.federation.FederationSearcherModel; -import com.yahoo.vespa.model.container.component.Component; -import com.yahoo.vespa.model.container.search.searchchain.*; +import com.yahoo.vespa.model.container.search.searchchain.FederationSearcher; +import com.yahoo.vespa.model.container.search.searchchain.SearchChain; +import com.yahoo.vespa.model.container.search.searchchain.SearchChains; +import com.yahoo.vespa.model.container.search.searchchain.Searcher; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Set; +import java.util.Optional; -import java.util.*; /** * Creates the search chains vespaPhases, vespa and native. @@ -36,17 +44,15 @@ public class VespaSearchChainsCreator { return new Phase(phase, set(before), null); } - public static Collection<Phase> linearPhases(String... phases) { + static Collection<Phase> linearPhases(String... phases) { List<Phase> result = new ArrayList<>(); for (int i=0; i < phases.length - 1; ++i) { - result.add( - createPhase(phases[i], phases[i+1])); + result.add(createPhase(phases[i], phases[i+1])); } if (phases.length > 0) { - result.add( - createPhase(lastElement(phases), null)); + result.add(createPhase(lastElement(phases), null)); } return result; @@ -54,11 +60,11 @@ public class VespaSearchChainsCreator { } private static Set<ComponentSpecification> noSearcherReferences() { - return Collections.emptySet(); + return Set.of(); } private static Collection<Phase> noPhases() { - return Collections.emptySet(); + return Set.of(); } private static ChainSpecification.Inheritance inherits(ComponentId chainId) { @@ -79,7 +85,7 @@ public class VespaSearchChainsCreator { private static Searcher<? extends ChainedComponentModel> createSearcher(ChainedComponentModel searcherModel) { if (searcherModel instanceof FederationSearcherModel) { - return new FederationSearcher((FederationSearcherModel) searcherModel, Optional.<Component>empty()); + return new FederationSearcher((FederationSearcherModel) searcherModel, Optional.empty()); } else { return new Searcher<>(searcherModel); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/BundleInstantiationSpecificationBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/BundleInstantiationSpecificationBuilder.java index fbdefd0afb0..cdea63cfb70 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/BundleInstantiationSpecificationBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/BundleInstantiationSpecificationBuilder.java @@ -10,6 +10,8 @@ import org.w3c.dom.Element; import java.util.Arrays; import java.util.List; +import static com.yahoo.vespa.model.container.component.chain.ProcessingHandler.PROCESSING_HANDLER_CLASS; + /** * This object builds a bundle instantiation spec from an XML element. * @@ -30,7 +32,7 @@ public class BundleInstantiationSpecificationBuilder { private static BundleInstantiationSpecification setBundleForSearchAndDocprocComponents(BundleInstantiationSpecification spec) { if (PlatformBundles.isSearchAndDocprocClass(spec.getClassName())) - return spec.inBundle(PlatformBundles.searchAndDocprocBundle); + return spec.inBundle(PlatformBundles.SEARCH_AND_DOCPROC_BUNDLE); else return spec; } @@ -39,7 +41,7 @@ public class BundleInstantiationSpecificationBuilder { private static void validate(BundleInstantiationSpecification instSpec) { List<String> forbiddenClasses = Arrays.asList( SearchHandler.HANDLER_CLASS, - "com.yahoo.processing.handler.ProcessingHandler"); + PROCESSING_HANDLER_CLASS); for (String forbiddenClass: forbiddenClasses) { if (forbiddenClass.equals(instSpec.getClassName())) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/CloudSecretStore.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/CloudSecretStore.java index 18916f1e09b..373eb9714d3 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/CloudSecretStore.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/CloudSecretStore.java @@ -21,7 +21,7 @@ public class CloudSecretStore extends SimpleComponent implements SecretStoreConf private final List<StoreConfig> configList; public CloudSecretStore() { - super(new ComponentModel(BundleInstantiationSpecification.getFromStrings(CLASS, CLASS, BUNDLE))); + super(new ComponentModel(BundleInstantiationSpecification.fromStrings(CLASS, CLASS, BUNDLE))); configList = new ArrayList<>(); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java index f971aa97e6a..f6174e1740f 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java @@ -62,13 +62,13 @@ import com.yahoo.vespa.model.container.PlatformBundles; import com.yahoo.vespa.model.container.SecretStore; import com.yahoo.vespa.model.container.component.AccessLogComponent; import com.yahoo.vespa.model.container.component.BindingPattern; +import com.yahoo.vespa.model.container.component.Component; import com.yahoo.vespa.model.container.component.ConnectionLogComponent; import com.yahoo.vespa.model.container.component.FileStatusHandlerComponent; import com.yahoo.vespa.model.container.component.Handler; import com.yahoo.vespa.model.container.component.SimpleComponent; import com.yahoo.vespa.model.container.component.SystemBindingPattern; import com.yahoo.vespa.model.container.component.UserBindingPattern; -import com.yahoo.vespa.model.container.component.chain.ProcessingHandler; import com.yahoo.vespa.model.container.docproc.ContainerDocproc; import com.yahoo.vespa.model.container.docproc.DocprocChains; import com.yahoo.vespa.model.container.http.AccessControl; @@ -88,6 +88,7 @@ import com.yahoo.vespa.model.container.xml.embedder.EmbedderConfig; import com.yahoo.vespa.model.content.StorageGroup; import org.w3c.dom.Element; import org.w3c.dom.Node; + import java.net.URI; import java.security.cert.X509Certificate; import java.util.ArrayList; @@ -205,6 +206,8 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { addServerProviders(deployState, spec, cluster); + if (!standaloneBuilder) cluster.addAllPlatformBundles(); + // Must be added after nodes: addDeploymentSpecConfig(cluster, context, deployState.getDeployLogger()); addZooKeeper(cluster, spec); @@ -220,7 +223,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { } if (deployState.zone().system().isPublic()) { BindingPattern bindingPattern = SystemBindingPattern.fromHttpPath("/validate-secret-store"); - Handler<AbstractConfigProducer<?>> handler = new Handler<>( + Handler handler = new Handler( new ComponentModel("com.yahoo.jdisc.cloud.aws.AwsParameterStoreValidationHandler", null, "jdisc-cloud-aws", null)); handler.addServerBindings(bindingPattern); cluster.addComponent(handler); @@ -548,7 +551,10 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { cluster.setSearch(buildSearch(deployState, cluster, searchElement)); addSearchHandler(cluster, searchElement); - addGUIHandler(cluster); + + // Set up GUI handler only on self hosted + if (!deployState.isHosted()) + addGUIHandler(cluster); validateAndAddConfiguredComponents(deployState, cluster, searchElement, "renderer", ContainerModelBuilder::validateRendererElement); } @@ -595,6 +601,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { Element processingElement = XML.getChild(spec, "processing"); if (processingElement == null) return; + cluster.addSearchAndDocprocBundles(); addIncludes(processingElement); cluster.setProcessingChains(new DomProcessingBuilder(null).build(deployState, cluster, processingElement), serverBindings(processingElement, ProcessingChains.defaultBindings).toArray(BindingPattern[]::new)); @@ -732,9 +739,14 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { private List<ApplicationContainer> createNodes(ApplicationContainerCluster cluster, Element containerElement, Element nodesElement, ConfigModelContext context) { if (nodesElement.hasAttribute("type")) // internal use for hosted system infrastructure nodes return createNodesFromNodeType(cluster, nodesElement, context); - else if (nodesElement.hasAttribute("of")) // hosted node spec referencing a content cluster - return createNodesFromContentServiceReference(cluster, nodesElement, context); - else if (nodesElement.hasAttribute("count")) // regular, hosted node spec + else if (nodesElement.hasAttribute("of")) {// hosted node spec referencing a content cluster + // TODO: Remove support for combined clusters in Vespa 9 + List<ApplicationContainer> containers = createNodesFromContentServiceReference(cluster, nodesElement, context); + log.logApplicationPackage(WARNING, "Declaring combined cluster with <nodes of=\"...\"> is deprecated without " + + "replacement, and the feature will be removed in Vespa 9. Use separate container and " + + "content clusters instead"); + return containers; + } else if (nodesElement.hasAttribute("count")) // regular, hosted node spec return createNodesFromNodeCount(cluster, containerElement, nodesElement, context); else if (cluster.isHostedVespa() && cluster.getZone().environment().isManuallyDeployed()) // default to 1 in manual zones return createNodesFromNodeCount(cluster, containerElement, nodesElement, context); @@ -877,19 +889,17 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { } private void addSearchHandler(ApplicationContainerCluster cluster, Element searchElement) { - // Magic spell is needed to receive the chains config :-| - cluster.addComponent(new ProcessingHandler<>(cluster.getSearch().getChains(), - "com.yahoo.search.searchchain.ExecutionFactory")); + SearchHandler searchHandler = new SearchHandler(cluster, + serverBindings(searchElement, SearchHandler.DEFAULT_BINDING), + ContainerThreadpool.UserOptions.fromXml(searchElement).orElse(null)); + cluster.addComponent(searchHandler); - cluster.addComponent( - new SearchHandler( - cluster, - serverBindings(searchElement, SearchHandler.DEFAULT_BINDING), - ContainerThreadpool.UserOptions.fromXml(searchElement).orElse(null))); + // Add as child to SearchHandler to get the correct chains config. + searchHandler.addComponent(Component.fromClassAndBundle(SearchHandler.EXECUTION_FACTORY_CLASS, PlatformBundles.SEARCH_AND_DOCPROC_BUNDLE)); } private void addGUIHandler(ApplicationContainerCluster cluster) { - Handler<?> guiHandler = new GUIHandler(); + Handler guiHandler = new GUIHandler(); guiHandler.addServerBindings(SystemBindingPattern.fromHttpPath(GUIHandler.BINDING_PATH)); cluster.addComponent(guiHandler); } @@ -919,8 +929,10 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { Element documentApiElement = XML.getChild(spec, "document-api"); if (documentApiElement == null) return null; - ContainerDocumentApi.Options documentApiOptions = DocumentApiOptionsBuilder.build(documentApiElement); - return new ContainerDocumentApi(cluster, documentApiOptions); + ContainerDocumentApi.HandlerOptions documentApiOptions = DocumentApiOptionsBuilder.build(documentApiElement); + Element ignoreUndefinedFields = XML.getChild(documentApiElement, "ignore-undefined-fields"); + return new ContainerDocumentApi(cluster, documentApiOptions, + "true".equals(XML.getValue(ignoreUndefinedFields))); } private ContainerDocproc buildDocproc(DeployState deployState, ApplicationContainerCluster cluster, Element spec) { @@ -931,7 +943,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { addIncludes(docprocElement); DocprocChains chains = new DomDocprocChainsBuilder(null, false).build(deployState, cluster, docprocElement); - ContainerDocproc.Options docprocOptions = DocprocOptionsBuilder.build(docprocElement); + ContainerDocproc.Options docprocOptions = DocprocOptionsBuilder.build(docprocElement, deployState.getDeployLogger()); return new ContainerDocproc(cluster, chains, docprocOptions, !standaloneBuilder); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/DocprocOptionsBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/DocprocOptionsBuilder.java index 1239cbf9bdc..faf2d01d385 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/DocprocOptionsBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/DocprocOptionsBuilder.java @@ -1,18 +1,18 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.container.xml; +import com.yahoo.config.application.api.DeployLogger; import com.yahoo.vespa.model.container.docproc.ContainerDocproc; import org.w3c.dom.Element; +import java.util.Set; +import java.util.logging.Level; -/** - * Extracted from DomDocProcClusterBuilder - */ public class DocprocOptionsBuilder { - public static ContainerDocproc.Options build(Element spec) { + public static ContainerDocproc.Options build(Element spec, DeployLogger deployLogger) { + checkForDeprecatedAttributes(spec, Set.of("maxqueuebytesize", "numnodesperclient", "preferlocalnode"), deployLogger); + return new ContainerDocproc.Options( - getCompression(spec), getMaxMessagesInQueue(spec), - getSizeInMegabytes(spec.getAttribute("maxqueuebytesize")), getTime(spec.getAttribute("maxqueuewait")), getFactor(spec.getAttribute("maxconcurrentfactor")), getFactor(spec.getAttribute("documentexpansionfactor")), @@ -20,22 +20,17 @@ public class DocprocOptionsBuilder { } private static Integer getInt(String integer) { - return integer == null || integer.trim().isEmpty() ? - null: - Integer.parseInt(integer); - } - - private static boolean getCompression(Element spec) { - return (spec.hasAttribute("compressdocuments") && spec.getAttribute("compressdocuments").equals("true")); + return integer == null || integer.trim().isEmpty() + ? null + : Integer.parseInt(integer); } private static Double getFactor(String factor) { - return factor == null || factor.trim().isEmpty() ? - null : - Double.parseDouble(factor); + return factor == null || factor.trim().isEmpty() + ? null + : Double.parseDouble(factor); } - private static Integer getMaxMessagesInQueue(Element spec) { // get max queue size (number of messages), if set Integer maxMessagesInQueue = null; @@ -45,28 +40,6 @@ public class DocprocOptionsBuilder { return maxMessagesInQueue; } - private static Integer getSizeInMegabytes(String size) { - if (size == null) { - return null; - } - size = size.trim(); - if (size.isEmpty()) { - return null; - } - - Integer megabyteSize; - if (size.endsWith("m")) { - size = size.substring(0, size.length() - 1); - megabyteSize = Integer.parseInt(size); - } else if (size.endsWith("g")) { - size = size.substring(0, size.length() - 1); - megabyteSize = Integer.parseInt(size) * 1024; - } else { - throw new IllegalArgumentException("Heap sizes for docproc must be set to Xm or Xg, where X is an integer specifying megabytes or gigabytes, respectively."); - } - return megabyteSize; - } - private static Integer getTime(String intStr) { if (intStr == null) { return null; @@ -78,4 +51,13 @@ public class DocprocOptionsBuilder { return 1000 * (int)Double.parseDouble(intStr); } + + private static void checkForDeprecatedAttributes(Element spec, Set<String> names, DeployLogger deployLogger) { + names.forEach(n -> { + if (!spec.getAttribute(n).isEmpty()) + deployLogger.logApplicationPackage(Level.WARNING, "'" + n + "' is ignored, deprecated and will be removed in Vespa 9."); + }); + } + + } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/DocumentApiOptionsBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/DocumentApiOptionsBuilder.java index 2ef31469c45..bb1d0af1db9 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/DocumentApiOptionsBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/DocumentApiOptionsBuilder.java @@ -18,9 +18,8 @@ public class DocumentApiOptionsBuilder { private static final Logger log = Logger.getLogger(DocumentApiOptionsBuilder.class.getName()); - - public static ContainerDocumentApi.Options build(Element spec) { - return new ContainerDocumentApi.Options(getBindings(spec), threadpoolOptions(spec, "http-client-api")); + public static ContainerDocumentApi.HandlerOptions build(Element spec) { + return new ContainerDocumentApi.HandlerOptions(getBindings(spec), threadpoolOptions(spec, "http-client-api")); } private static ContainerThreadpool.UserOptions threadpoolOptions(Element spec, String elementName) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/SearchHandler.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/SearchHandler.java index 54d943f498a..54cd061d2c5 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/SearchHandler.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/SearchHandler.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.container.xml; +import com.yahoo.container.bundle.BundleInstantiationSpecification; import com.yahoo.container.handler.threadpool.ContainerThreadpoolConfig; import com.yahoo.vespa.model.container.ApplicationContainerCluster; import com.yahoo.vespa.model.container.ContainerThreadpool; @@ -11,6 +12,8 @@ import com.yahoo.vespa.model.container.search.searchchain.SearchChains; import java.util.List; +import static com.yahoo.container.bundle.BundleInstantiationSpecification.fromSearchAndDocproc; + /** * Component definition for {@link com.yahoo.search.handler.SearchHandler} * @@ -19,38 +22,34 @@ import java.util.List; class SearchHandler extends ProcessingHandler<SearchChains> { static final String HANDLER_CLASS = com.yahoo.search.handler.SearchHandler.class.getName(); + static final String EXECUTION_FACTORY_CLASS = com.yahoo.search.searchchain.ExecutionFactory.class.getName(); + + static final BundleInstantiationSpecification HANDLER_SPEC = fromSearchAndDocproc(HANDLER_CLASS); static final BindingPattern DEFAULT_BINDING = SystemBindingPattern.fromHttpPath("/search/*"); SearchHandler(ApplicationContainerCluster cluster, List<BindingPattern> bindings, ContainerThreadpool.UserOptions threadpoolOptions) { - super(cluster.getSearchChains(), HANDLER_CLASS); + super(cluster.getSearchChains(), HANDLER_SPEC, new Threadpool(threadpoolOptions)); bindings.forEach(this::addServerBindings); - Threadpool threadpool = new Threadpool(cluster, threadpoolOptions); - inject(threadpool); - addComponent(threadpool); } + private static class Threadpool extends ContainerThreadpool { - private final ApplicationContainerCluster cluster; - Threadpool(ApplicationContainerCluster cluster, UserOptions options) { + Threadpool(UserOptions options) { super("search-handler", options); - this.cluster = cluster; } @Override - public void getConfig(ContainerThreadpoolConfig.Builder builder) { - super.getConfig(builder); - - builder.maxThreadExecutionTimeSeconds(190); - builder.keepAliveTime(5.0); - - // User options overrides below configuration - if (hasUserOptions()) return; - builder.maxThreads(-2).minThreads(-2).queueSize(-40); + public void setDefaultConfigValues(ContainerThreadpoolConfig.Builder builder) { + builder.maxThreadExecutionTimeSeconds(190) + .keepAliveTime(5.0) + .maxThreads(-2) + .minThreads(-2) + .queueSize(-40); } - } + } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/document/DocumentFactoryBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/document/DocumentFactoryBuilder.java index 5df862647a0..7f3d7981bb6 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/document/DocumentFactoryBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/document/DocumentFactoryBuilder.java @@ -31,7 +31,7 @@ public class DocumentFactoryBuilder { String concDocFactory=pkg+"."+CONCRETE_DOC_FACTORY_CLASS; String bundle = e.getAttribute("bundle"); Component<AbstractConfigProducer<?>, ComponentModel> component = new Component<>( - new ComponentModel(BundleInstantiationSpecification.getFromStrings(concDocFactory, concDocFactory, bundle))); + new ComponentModel(BundleInstantiationSpecification.fromStrings(concDocFactory, concDocFactory, bundle))); if (!cluster.getComponentsMap().containsKey(component.getComponentId())) cluster.addComponent(component); types.put(type, concDocFactory); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/Content.java b/config-model/src/main/java/com/yahoo/vespa/model/content/Content.java index d4595b3adfd..fc2f68ccd92 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/Content.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/Content.java @@ -54,7 +54,7 @@ public class Content extends ConfigModel { // Dependencies to other models private final AdminModel adminModel; - // to find or add the docproc container and supplement cluster controllers with clusters having less then 3 nodes + // to find or add the docproc container and supplement cluster controllers with clusters having less than 3 nodes private final Collection<ContainerModel> containers; @SuppressWarnings("UnusedDeclaration") // Created by reflection in ConfigModelRepo @@ -298,6 +298,7 @@ public class Content extends ConfigModel { content.ownedIndexingCluster = Optional.of(indexingCluster); indexingCluster.addDefaultHandlersWithVip(); + indexingCluster.addAllPlatformBundles(); addDocproc(indexingCluster); List<ApplicationContainer> nodes = new ArrayList<>(); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentNode.java b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentNode.java index d6ed6f4f7dd..c52bb6fa2de 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentNode.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentNode.java @@ -21,17 +21,21 @@ public abstract class ContentNode extends AbstractService private final int distributionKey; private final String rootDirectory; - private final boolean skipCommunicationManagerThread; - private final boolean skipMbusRequestThread; - private final boolean skipMbusReplyThread; + private final int mbus_network_threads; + private final int mbus_rpc_targets; + private final int mbus_events_before_wakeup; + private final int rpc_num_targets; + private final int rpc_events_before_wakeup; public ContentNode(ModelContext.FeatureFlags featureFlags, AbstractConfigProducer<?> parent, String clusterName, String rootDirectory, int distributionKey) { super(parent, "" + distributionKey); this.distributionKey = distributionKey; - this.skipCommunicationManagerThread = featureFlags.skipCommunicationManagerThread(); - this.skipMbusRequestThread = featureFlags.skipMbusRequestThread(); - this.skipMbusReplyThread = featureFlags.skipMbusReplyThread(); this.rootDirectory = rootDirectory; + mbus_network_threads = featureFlags.mbusNetworkThreads(); + mbus_rpc_targets = featureFlags.mbusCppRpcNumTargets(); + mbus_events_before_wakeup = featureFlags.mbusCppEventsBeforeWakeup(); + rpc_num_targets = featureFlags.rpcNumTargets(); + rpc_events_before_wakeup = featureFlags.rpcEventsBeforeWakeup(); initialize(); setProp("clustertype", "content"); @@ -75,9 +79,11 @@ public abstract class ContentNode extends AbstractService public void getConfig(StorCommunicationmanagerConfig.Builder builder) { builder.mbusport(getRelativePort(0)); builder.rpcport(getRelativePort(1)); - builder.skip_thread(skipCommunicationManagerThread); - builder.mbus.skip_request_thread(skipMbusRequestThread); - builder.mbus.skip_reply_thread(skipMbusReplyThread); + builder.mbus.num_network_threads(mbus_network_threads); + builder.mbus.num_rpc_targets(mbus_rpc_targets); + builder.mbus.events_before_wakeup(mbus_events_before_wakeup); + builder.rpc.num_targets_per_node(rpc_num_targets); + builder.rpc.events_before_wakeup(rpc_events_before_wakeup); } @Override diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/DispatchTuning.java b/config-model/src/main/java/com/yahoo/vespa/model/content/DispatchTuning.java index 4624ae9631d..6cad778dccf 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/DispatchTuning.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/DispatchTuning.java @@ -11,7 +11,7 @@ public class DispatchTuning { public static final DispatchTuning empty = new DispatchTuning.Builder().build(); - public enum DispatchPolicy { ROUNDROBIN, ADAPTIVE} + public enum DispatchPolicy { ROUNDROBIN, ADAPTIVE } private final Integer maxHitsPerPartition; private DispatchPolicy dispatchPolicy; @@ -69,7 +69,7 @@ public class DispatchTuning { private DispatchPolicy toDispatchPolicy(String policy) { switch (policy.toLowerCase()) { - case "adaptive": case "random": return DispatchPolicy.ADAPTIVE; // TODO: Deprecate 'random' on Java 8 + case "adaptive": case "random": return DispatchPolicy.ADAPTIVE; // TODO: Deprecate 'random' on Vespa 9 case "round-robin": return DispatchPolicy.ROUNDROBIN; default: throw new IllegalArgumentException("Unknown dispatch policy '" + policy + "'"); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/StorageGroup.java b/config-model/src/main/java/com/yahoo/vespa/model/content/StorageGroup.java index 8a8d2742df1..aef00be5ea9 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/StorageGroup.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/StorageGroup.java @@ -442,6 +442,11 @@ public class StorageGroup { nodeRequirement = Optional.of(NodesSpecification.from(nodesElement.get(), context)); else if (nodesElement.isEmpty() && subGroups.isEmpty() && context.getDeployState().isHosted()) // request one node nodeRequirement = Optional.of(NodesSpecification.nonDedicated(1, context)); + else if (nodesElement.isPresent() && nodesElement.get().stringAttribute("count") == null && context.getDeployState().isHosted()) + throw new IllegalArgumentException(""" + Clusters in hosted environments must have a <nodes count='N'> tag + matching all zones, and having no <node> subtags, + see https://cloud.vespa.ai/en/reference/services"""); else // Nodes or groups explicitly listed - resolve in GroupBuilder nodeRequirement = Optional.empty(); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/routing/DocumentProtocol.java b/config-model/src/main/java/com/yahoo/vespa/model/routing/DocumentProtocol.java index 754800a42e1..8a8d38e23e3 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/routing/DocumentProtocol.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/routing/DocumentProtocol.java @@ -182,44 +182,26 @@ public final class DocumentProtocol implements Protocol, return table; } - private static void addContainerClusterDocprocHops(Collection<ContainerCluster<?>> containerClusters, - RoutingTableSpec table) { - + private static void addContainerClusterDocprocHops(Collection<ContainerCluster<?>> containerClusters, RoutingTableSpec table) { for (ContainerCluster<?> cluster: containerClusters) { ContainerDocproc docproc = cluster.getDocproc(); if (docproc != null) { - String policy = policy(docproc); - for (DocprocChain chain : docproc.getChains().allChains().allComponents()) { - addChainHop(table, cluster.getConfigId(), policy, chain); + addChainHop(table, cluster.getConfigId(), chain); } } } } - private static void addChainHop(RoutingTableSpec table, String configId, String policy, DocprocChain chain) { - final StringBuilder selector = new StringBuilder(); - if (policy != null) { - selector.append(configId).append("/").append(policy).append("/").append(chain.getSessionName()); - } else { - selector.append("[LoadBalancer:cluster=").append(configId) - .append(";session=").append(chain.getSessionName()) - .append("]"); - } + private static void addChainHop(RoutingTableSpec table, String configId, DocprocChain chain) { + StringBuilder selector = new StringBuilder(); + selector.append("[LoadBalancer:cluster=").append(configId) + .append(";session=").append(chain.getSessionName()) + .append("]"); table.addHop(new HopSpec(chain.getServiceName(), selector.toString())); } - private static String policy(ContainerDocproc docproc) { - if (docproc.getNumNodesPerClient() > 0) { - return "[SubsetService:" + docproc.getNumNodesPerClient() + "]"; - } else if (docproc.isPreferLocalNode()) { - return "[LocalService]"; - } else { - return null; - } - } - /** * Create hops to all configured storage nodes for the Document protocol. The "Distributor" policy resolves its * recipients using slobrok lookups, so it requires no configured recipients. @@ -228,23 +210,22 @@ public final class DocumentProtocol implements Protocol, * @param table the routing table to add to */ private static void addContentRouting(List<ContentCluster> content, RoutingTableSpec table) { - - for (ContentCluster cluster : content) { - RouteSpec spec = new RouteSpec(cluster.getConfigId()); - - if (cluster.getSearch().hasIndexedCluster()) { - table.addRoute(spec.addHop("[MessageType:" + cluster.getConfigId() + "]")); - table.addRoute(new RouteSpec(getIndexedRouteName(cluster.getConfigId())) - .addHop(cluster.getSearch().getIndexed().getIndexingServiceName()) - .addHop("[Content:cluster=" + cluster.getName() + "]")); - table.addRoute(new RouteSpec(getDirectRouteName(cluster.getConfigId())) - .addHop("[Content:cluster=" + cluster.getName() + "]")); - } else { - table.addRoute(spec.addHop("[Content:cluster=" + cluster.getName() + "]")); - } - table.addRoute(new RouteSpec("storage/cluster." + cluster.getName()) - .addHop("route:" + cluster.getConfigId())); + for (ContentCluster cluster : content) { + RouteSpec spec = new RouteSpec(cluster.getConfigId()); + + if (cluster.getSearch().hasIndexedCluster()) { + table.addRoute(spec.addHop("[MessageType:" + cluster.getConfigId() + "]")); + table.addRoute(new RouteSpec(getIndexedRouteName(cluster.getConfigId())) + .addHop(cluster.getSearch().getIndexed().getIndexingServiceName()) + .addHop("[Content:cluster=" + cluster.getName() + "]")); + table.addRoute(new RouteSpec(getDirectRouteName(cluster.getConfigId())) + .addHop("[Content:cluster=" + cluster.getName() + "]")); + } else { + table.addRoute(spec.addHop("[Content:cluster=" + cluster.getName() + "]")); } + table.addRoute(new RouteSpec("storage/cluster." + cluster.getName()) + .addHop("route:" + cluster.getConfigId())); + } } /** @@ -255,9 +236,8 @@ public final class DocumentProtocol implements Protocol, * @param table the routing table to add to */ private static void addIndexingHop(List<ContentCluster> content, RoutingTableSpec table) { - if (content.isEmpty()) { - return; - } + if (content.isEmpty()) return; + HopSpec hop = new HopSpec("indexing", "[DocumentRouteSelector]"); for (ContentCluster cluster : content) { hop.addRecipient(cluster.getConfigId()); @@ -281,9 +261,8 @@ public final class DocumentProtocol implements Protocol, private static void addDefaultRoutes(List<ContentCluster> content, Collection<ContainerCluster<?>> containerClusters, RoutingTableSpec table) { - if (content.isEmpty() || !indexingHopExists(table)) { - return; - } + if (content.isEmpty() || !indexingHopExists(table)) return; + RouteSpec route = new RouteSpec("default"); String hop = getContainerClustersDocprocHop(containerClusters); if (hop != null) { @@ -326,9 +305,9 @@ public final class DocumentProtocol implements Protocol, } private static DocprocChain getDefaultChain(ContainerDocproc docproc) { - return docproc == null ? - null: - docproc.getChains().allChains().getComponent("default"); + return docproc == null + ? null + : docproc.getChains().allChains().getComponent("default"); } /** |