diff options
author | Jon Bratseth <bratseth@yahoo-inc.com> | 2017-12-13 15:26:41 +0100 |
---|---|---|
committer | Jon Bratseth <bratseth@yahoo-inc.com> | 2017-12-13 15:26:41 +0100 |
commit | d3c1c98b5e67cf8f9bb9f3efdef3161f63c19ffc (patch) | |
tree | 05aea3ca495d6acf87ee3817d6f2957a57c39339 /config-model | |
parent | 3783a9b21f8ab7ca3700903d9780a9f7374cf0c5 (diff) | |
parent | 540b84751e90d60c13c4da3e1e15d47b720c327e (diff) |
Merge with master
Diffstat (limited to 'config-model')
55 files changed, 553 insertions, 262 deletions
diff --git a/config-model/pom.xml b/config-model/pom.xml index 0fdc09e1a61..729d1a2ef9b 100644 --- a/config-model/pom.xml +++ b/config-model/pom.xml @@ -6,6 +6,7 @@ <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> <version>6-SNAPSHOT</version> + <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>config-model</artifactId> <packaging>container-plugin</packaging> diff --git a/config-model/src/main/java/com/yahoo/config/model/ConfigModelContext.java b/config-model/src/main/java/com/yahoo/config/model/ConfigModelContext.java index 8b3d983d6c1..78a8c161c3b 100644 --- a/config-model/src/main/java/com/yahoo/config/model/ConfigModelContext.java +++ b/config-model/src/main/java/com/yahoo/config/model/ConfigModelContext.java @@ -5,17 +5,14 @@ import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.producer.AbstractConfigProducer; -import org.w3c.dom.Element; -import java.util.Optional; import java.util.stream.Stream; /** * This class contains a context that is passed to a model builder, and can be used to retrieve the application package, * logger etc. * - * @author lulf - * @since 5.1 + * @author Ulf Lilleengen */ public final class ConfigModelContext { diff --git a/config-model/src/main/java/com/yahoo/config/model/admin/AdminModel.java b/config-model/src/main/java/com/yahoo/config/model/admin/AdminModel.java index 32cb1840e7c..5eb4afcc241 100644 --- a/config-model/src/main/java/com/yahoo/config/model/admin/AdminModel.java +++ b/config-model/src/main/java/com/yahoo/config/model/admin/AdminModel.java @@ -75,7 +75,11 @@ public class AdminModel extends ConfigModel { public void doBuild(AdminModel model, Element adminElement, ConfigModelContext modelContext) { AbstractConfigProducer parent = modelContext.getParentProducer(); DeployProperties properties = modelContext.getDeployState().getProperties(); - DomAdminV2Builder domBuilder = new DomAdminV2Builder(modelContext.getApplicationType(), modelContext.getDeployState().getFileRegistry(), properties.multitenant(), properties.configServerSpecs()); + DomAdminV2Builder domBuilder = new DomAdminV2Builder(modelContext.getApplicationType(), + modelContext.getDeployState().getFileRegistry(), + properties.multitenant(), + properties.configServerSpecs(), + modelContext.getDeployState().disableFiledistributor()); model.admin = domBuilder.build(parent, adminElement); // TODO: Is required since other models depend on admin. if (parent instanceof ApplicationConfigProducerRoot) { @@ -101,7 +105,11 @@ public class AdminModel extends ConfigModel { public void doBuild(AdminModel model, Element adminElement, ConfigModelContext modelContext) { AbstractConfigProducer parent = modelContext.getParentProducer(); DeployProperties properties = modelContext.getDeployState().getProperties(); - DomAdminV4Builder domBuilder = new DomAdminV4Builder(modelContext, properties.multitenant(), properties.configServerSpecs(), model.getContainerModels()); + DomAdminV4Builder domBuilder = new DomAdminV4Builder(modelContext, + properties.multitenant(), + properties.configServerSpecs(), + model.getContainerModels(), + modelContext.getDeployState().disableFiledistributor()); model.admin = domBuilder.build(parent, adminElement); // TODO: Is required since other models depend on admin. if (parent instanceof ApplicationConfigProducerRoot) { diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java b/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java index 25d45dee234..43d3fafdb78 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java @@ -68,8 +68,8 @@ public class DeployState implements ConfigDefinitionStore { private final ValidationOverrides validationOverrides; private final Version wantedNodeVespaVersion; private final Instant now; - private final HostProvisioner provisioner; + private final boolean disableFiledistributor; public static DeployState createTestState() { return new Builder().build(); @@ -82,8 +82,8 @@ public class DeployState implements ConfigDefinitionStore { private DeployState(ApplicationPackage applicationPackage, SearchDocumentModel searchDocumentModel, RankProfileRegistry rankProfileRegistry, FileRegistry fileRegistry, DeployLogger deployLogger, Optional<HostProvisioner> hostProvisioner, DeployProperties properties, Optional<ApplicationPackage> permanentApplicationPackage, Optional<ConfigDefinitionRepo> configDefinitionRepo, - java.util.Optional<Model> previousModel, Set<Rotation> rotations, Zone zone, QueryProfiles queryProfiles, - SemanticRules semanticRules, Instant now, Version wantedNodeVespaVersion) { + java.util.Optional<Model> previousModel, Set<Rotation> rotations, Zone zone, QueryProfiles queryProfiles, + SemanticRules semanticRules, Instant now, Version wantedNodeVespaVersion, boolean disableFiledistributor) { this.logger = deployLogger; this.fileRegistry = fileRegistry; this.rankProfileRegistry = rankProfileRegistry; @@ -102,6 +102,7 @@ public class DeployState implements ConfigDefinitionStore { this.validationOverrides = applicationPackage.getValidationOverrides().map(ValidationOverrides::fromXml).orElse(ValidationOverrides.empty); this.wantedNodeVespaVersion = wantedNodeVespaVersion; this.now = now; + this.disableFiledistributor = disableFiledistributor; } public static HostProvisioner getDefaultModelHostProvisioner(ApplicationPackage applicationPackage) { @@ -169,6 +170,7 @@ public class DeployState implements ConfigDefinitionStore { public ApplicationPackage getApplicationPackage() { return applicationPackage; } + public List<SearchDefinition> getSearchDefinitions() { return searchDefinitions; } @@ -214,6 +216,8 @@ public class DeployState implements ConfigDefinitionStore { public Instant now() { return now; } + public boolean disableFiledistributor() { return disableFiledistributor; } + public static class Builder { private ApplicationPackage applicationPackage = MockApplicationPackage.createEmpty(); @@ -228,6 +232,7 @@ public class DeployState implements ConfigDefinitionStore { private Zone zone = Zone.defaultZone(); private Instant now = Instant.now(); private Version wantedNodeVespaVersion = Vtag.currentVersion; + private boolean disableFiledistributor = false; public Builder applicationPackage(ApplicationPackage applicationPackage) { this.applicationPackage = applicationPackage; @@ -289,13 +294,19 @@ public class DeployState implements ConfigDefinitionStore { return this; } + public Builder disableFiledistributor(boolean disableFiledistributor) { + this.disableFiledistributor = disableFiledistributor; + return this; + } + public DeployState build() { RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); QueryProfiles queryProfiles = new QueryProfilesBuilder().build(applicationPackage); SemanticRules semanticRules = new SemanticRuleBuilder().build(applicationPackage); SearchDocumentModel searchDocumentModel = createSearchDocumentModel(rankProfileRegistry, logger, queryProfiles); return new DeployState(applicationPackage, searchDocumentModel, rankProfileRegistry, fileRegistry, logger, hostProvisioner, - properties, permanentApplicationPackage, configDefinitionRepo, previousModel, rotations, zone, queryProfiles, semanticRules, now, wantedNodeVespaVersion); + properties, permanentApplicationPackage, configDefinitionRepo, previousModel, rotations, + zone, queryProfiles, semanticRules, now, wantedNodeVespaVersion, disableFiledistributor); } private SearchDocumentModel createSearchDocumentModel(RankProfileRegistry rankProfileRegistry, DeployLogger logger, QueryProfiles queryProfiles) { diff --git a/config-model/src/main/java/com/yahoo/config/model/test/MockRoot.java b/config-model/src/main/java/com/yahoo/config/model/test/MockRoot.java index 277e02e43e2..2a3ff1ca905 100644 --- a/config-model/src/main/java/com/yahoo/config/model/test/MockRoot.java +++ b/config-model/src/main/java/com/yahoo/config/model/test/MockRoot.java @@ -11,6 +11,7 @@ import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.builder.xml.XmlHelper; import com.yahoo.config.model.producer.AbstractConfigProducer; import com.yahoo.config.model.producer.AbstractConfigProducerRoot; +import com.yahoo.config.provision.Zone; import com.yahoo.text.XML; import com.yahoo.vespa.model.ConfigProducer; import com.yahoo.vespa.model.HostSystem; @@ -147,7 +148,8 @@ public class MockRoot extends AbstractConfigProducerRoot { try { Document doc = XmlHelper.getDocumentBuilder().parse(new InputSource(new StringReader(servicesXml))); - setAdmin(new DomAdminV2Builder(ConfigModelContext.ApplicationType.DEFAULT, deployState.getFileRegistry(), false, new ArrayList<>()). + setAdmin(new DomAdminV2Builder(ConfigModelContext.ApplicationType.DEFAULT, deployState.getFileRegistry(), + false, new ArrayList<>(), deployState.disableFiledistributor()). build(this, XML.getChildren(doc.getDocumentElement(), "admin").get(0))); } catch (SAXException | IOException e) { throw new RuntimeException(e); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/ImmutableImportedSDField.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/ImmutableImportedSDField.java index d62d1f5200a..8b6df1a87db 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/document/ImmutableImportedSDField.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/ImmutableImportedSDField.java @@ -79,7 +79,12 @@ public class ImmutableImportedSDField implements ImmutableSDField { @Override public Index getIndex(String name) { - throw createUnsupportedException("index"); + if ( ! importedField.fieldName().equals(name)) { + throw new IllegalArgumentException("Getting an index (" + name + ") with different name than the imported field (" + + importedField.fieldName() + ") is not supported"); + } + String targetIndexName = importedField.targetField().getName(); + return importedField.targetField().getIndex(targetIndexName); } @Override diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImportedFieldsInSummayValidator.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImportedFieldsInSummayValidator.java new file mode 100644 index 00000000000..eaa85815736 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImportedFieldsInSummayValidator.java @@ -0,0 +1,63 @@ +package com.yahoo.searchdefinition.processing; + +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.document.DataType; +import com.yahoo.searchdefinition.RankProfileRegistry; +import com.yahoo.searchdefinition.Search; +import com.yahoo.searchdefinition.document.ImportedField; +import com.yahoo.vespa.documentmodel.DocumentSummary; +import com.yahoo.vespa.documentmodel.SummaryField; +import com.yahoo.vespa.model.container.search.QueryProfiles; + +import java.util.Map; + +/** + * Validates that imported fields in document summaries are of supported types. + * Currently, predicate fields are NOT supported. + * + * @author geirst + */ +public class ImportedFieldsInSummayValidator extends Processor { + + + public ImportedFieldsInSummayValidator(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { + super(search, deployLogger, rankProfileRegistry, queryProfiles); + } + + @Override + public void process() { + if (search.importedFields().isPresent()) { + validateDocumentSummaries(search.getSummaries()); + } + } + + private void validateDocumentSummaries(Map<String, DocumentSummary> summaries) { + for (DocumentSummary summary : summaries.values()) { + for (SummaryField field : summary.getSummaryFields()) { + ImportedField importedField = getImportedField(field); + if (importedField != null) { + validateImportedSummaryField(summary, field, importedField); + } + } + } + } + + private ImportedField getImportedField(SummaryField field) { + return search.importedFields().get().fields().get(field.getName()); + } + + private void validateImportedSummaryField(DocumentSummary summary, SummaryField field, ImportedField importedField) { + if (field.getDataType().equals(DataType.PREDICATE) && + importedField.targetField().getDataType().equals(DataType.PREDICATE)) { + fail(summary, field, "Is of type predicate. Not supported in document summaries"); + } + } + + private void fail(DocumentSummary summary, SummaryField importedField, String msg) { + throw new IllegalArgumentException("For search '" + search.getName() + "', document summary '" + summary.getName() + + "', imported summary field '" + importedField.getName() + "': " + msg); + } +} + + + diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processing.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processing.java index 26f98026d4f..90183848094 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processing.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processing.java @@ -73,6 +73,7 @@ public class Processing { TensorFieldProcessor::new, RankProfileTypeSettingsProcessor::new, ReferenceFieldsProcessor::new, + ImportedFieldsInSummayValidator::new, FastAccessValidator::new, ReservedMacroNames::new, diff --git a/config-model/src/main/java/com/yahoo/vespa/model/AbstractService.java b/config-model/src/main/java/com/yahoo/vespa/model/AbstractService.java index b4b89278c51..659d5b65a4c 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/AbstractService.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/AbstractService.java @@ -510,22 +510,6 @@ public abstract class AbstractService extends AbstractConfigProducer<AbstractCon } /** - * Sets up this service to be included when generating monitoring config. - * The ymon service name used will be {@link #getServiceType()} - */ - public void monitorService() { - monitorService(getServiceType()); - } - - /** - * Sets up this service to be included when generating ymon config. - * @param ymonServiceName the ymon service name to be used - */ - public void monitorService(String ymonServiceName) { - setProp("ymonService", ymonServiceName); - } - - /** * * The service HTTP port for health status * @return portnumber diff --git a/config-model/src/main/java/com/yahoo/vespa/model/ConfigProxy.java b/config-model/src/main/java/com/yahoo/vespa/model/ConfigProxy.java index d751e332ff3..c540a5f62d2 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/ConfigProxy.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/ConfigProxy.java @@ -16,6 +16,8 @@ package com.yahoo.vespa.model; */ public class ConfigProxy extends AbstractService { + public final static int BASEPORT = 19090; + /** * Creates a new ConfigProxy instance. * @@ -31,7 +33,7 @@ public class ConfigProxy extends AbstractService { /** * Returns the desired base port for this service. */ - public int getWantedPort() { return 19090; } + public int getWantedPort() { return BASEPORT; } /** * The desired base port is the only allowed base port. diff --git a/config-model/src/main/java/com/yahoo/vespa/model/VespaModelFactory.java b/config-model/src/main/java/com/yahoo/vespa/model/VespaModelFactory.java index 74512e70ebe..7a1fab8dbd0 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/VespaModelFactory.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/VespaModelFactory.java @@ -138,7 +138,8 @@ public class VespaModelFactory implements ModelFactory { .rotations(modelContext.properties().rotations()) .zone(zone) .now(clock.instant()) - .wantedNodeVespaVersion(modelContext.wantedNodeVespaVersion()); + .wantedNodeVespaVersion(modelContext.wantedNodeVespaVersion()) + .disableFiledistributor(modelContext.properties().disableFileDistributor()); modelContext.previousModel().ifPresent(builder::previousModel); return builder.build(); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java index 69e9a0e8585..d892d70e0de 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java @@ -14,6 +14,8 @@ import com.yahoo.vespa.model.admin.monitoring.MetricsConsumer; import com.yahoo.vespa.model.admin.monitoring.Monitoring; import com.yahoo.vespa.model.admin.monitoring.builder.Metrics; import com.yahoo.vespa.model.container.ContainerCluster; +import com.yahoo.vespa.model.filedistribution.DummyFileDistributionConfigProducer; +import com.yahoo.vespa.model.filedistribution.FileDistributionConfigProvider; import com.yahoo.vespa.model.filedistribution.FileDistributionConfigProducer; import com.yahoo.vespa.model.filedistribution.FileDistributor; import com.yahoo.vespa.model.filedistribution.FileDistributorService; @@ -59,12 +61,14 @@ public class Admin extends AbstractConfigProducer implements Serializable { Monitoring monitoring, Metrics metrics, Map<String, MetricsConsumer> legacyMetricsConsumers, - boolean multitenant) { + boolean multitenant, + FileDistributionConfigProducer fileDistributionConfigProducer) { super(parent, "admin"); this.monitoring = monitoring; this.metrics = metrics; this.legacyMetricsConsumers = legacyMetricsConsumers; this.multitenant = multitenant; + this.fileDistribution = fileDistributionConfigProducer; } public Configserver getConfigserver() { @@ -148,10 +152,6 @@ public class Admin extends AbstractConfigProducer implements Serializable { zooKeepersConfigProvider.getConfig(builder); } - public void setFileDistribution(FileDistributionConfigProducer fileDistribution) { - this.fileDistribution = fileDistribution; - } - public FileDistributionConfigProducer getFileDistributionConfigProducer() { return fileDistribution; } @@ -215,11 +215,25 @@ public class Admin extends AbstractConfigProducer implements Serializable { fileDistributor.fileSourceHost() + "'. Hostsystem=" + getHostSystem()); } - FileDistributorService fds = new FileDistributorService(fileDistribution, host.getHost().getHostName(), - fileDistribution.getFileDistributor(), fileDistribution.getOptions(), host == deployHost); - fds.setHostResource(host); - fds.initService(); - fileDistribution.addFileDistributionService(host.getHost(), fds); + FileDistributionConfigProvider configProvider = + new FileDistributionConfigProvider(fileDistributor, + fileDistribution.getOptions(), + host == deployHost, + host.getHost()); + if (fileDistribution.getOptions().disableFiledistributor()) { + DummyFileDistributionConfigProducer dummyFileDistributionConfigProducer = + new DummyFileDistributionConfigProducer(fileDistribution, + host.getHost().getHostName(), + configProvider); + fileDistribution.addFileDistributionConfigProducer(host.getHost(), dummyFileDistributionConfigProducer); + } else { + FileDistributorService fds = new FileDistributorService(fileDistribution, + host.getHost().getHostName(), + configProvider); + fds.setHostResource(host); + fds.initService(); + fileDistribution.addFileDistributionConfigProducer(host.getHost(), fds); + } } private boolean deployHostIsMissing(HostResource deployHost) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/Configserver.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/Configserver.java index 942d563d850..e8f1f59310b 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/Configserver.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/Configserver.java @@ -35,7 +35,6 @@ public class Configserver extends AbstractService { portsMeta.on(1).tag("http").tag("config").tag("state"); setProp("clustertype", "admin"); setProp("clustername", "admin"); - monitorService(); } /** diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/FileDistributionOptions.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/FileDistributionOptions.java index 0160978773d..8b39a1ecb94 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/FileDistributionOptions.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/FileDistributionOptions.java @@ -21,7 +21,7 @@ public class FileDistributionOptions implements FiledistributorConfig.Producer { private BinaryScaledAmount uploadBitRate = new BinaryScaledAmount(); private BinaryScaledAmount downloadBitRate = new BinaryScaledAmount(); - private boolean disabled = false; + private boolean disableFiledistributor = false; public void downloadBitRate(BinaryScaledAmount amount) { @@ -34,12 +34,12 @@ public class FileDistributionOptions implements FiledistributorConfig.Producer { uploadBitRate = amount; } - public void disabled(boolean value) { - disabled = value; + public void disableFiledistributor(boolean value) { + disableFiledistributor = value; } - public boolean disabled() { - return disabled; + public boolean disableFiledistributor() { + return disableFiledistributor; } private void ensureNonNegative(BinaryScaledAmount amount) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/Logserver.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/Logserver.java index 28cf18802cf..8b98dc9d06a 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/Logserver.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/Logserver.java @@ -23,7 +23,6 @@ public class Logserver extends AbstractService { portsMeta.on(3).tag("logtp").tag("telnet").tag("replicator"); setProp("clustertype", "admin"); setProp("clustername", "admin"); - monitorService(); } /** diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/Slobrok.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/Slobrok.java index cefc08981a4..093694f41b3 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/Slobrok.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/Slobrok.java @@ -23,7 +23,6 @@ public class Slobrok extends AbstractService { setProp("index", index); setProp("clustertype", "slobrok"); setProp("clustername", "admin"); - monitorService(); } @Override diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/SystemMetrics.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/SystemMetrics.java index cafdf83608f..3db09f6b566 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/SystemMetrics.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/SystemMetrics.java @@ -11,6 +11,7 @@ import java.util.Set; @SuppressWarnings("UnusedDeclaration") // Used by model amenders public class SystemMetrics { public static final String CPU_UTIL = "cpu.util"; + public static final String CPU_SYS_UTIL = "cpu.sys.util"; public static final String DISK_LIMIT = "disk.limit"; public static final String DISK_USED = "disk.used"; public static final String DISK_UTIL = "disk.util"; @@ -23,6 +24,7 @@ public class SystemMetrics { private static MetricSet createSystemMetricSet() { Set<Metric> dockerNodeMetrics = ImmutableSet.of(new Metric(CPU_UTIL), + new Metric(CPU_SYS_UTIL), new Metric(DISK_LIMIT), new Metric(DISK_USED), new Metric(DISK_UTIL), diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java index 2dcf874a66e..2243dc1e682 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java @@ -52,6 +52,7 @@ public class VespaMetricSet { private static Set<Metric> getOtherMetrics() { Set<Metric> metrics = new LinkedHashSet<>(); metrics.add(new Metric("slobrok.heartbeats.failed.count", "slobrok.heartbeats.failed")); + metrics.add(new Metric("logd.processed.lines.count", "logd.processed.lines")); return metrics; } @@ -220,7 +221,9 @@ public class VespaMetricSet { // resource usage metrics.add(new Metric("content.proton.resource_usage.disk.average")); + metrics.add(new Metric("content.proton.resource_usage.disk_utilization.average")); metrics.add(new Metric("content.proton.resource_usage.memory.average")); + metrics.add(new Metric("content.proton.resource_usage.memory_utilization.average")); metrics.add(new Metric("content.proton.resource_usage.memory_mappings.max")); metrics.add(new Metric("content.proton.resource_usage.open_file_descriptors.max")); metrics.add(new Metric("content.proton.documentdb.attribute.resource_usage.enum_store.average")); @@ -230,6 +233,7 @@ public class VespaMetricSet { // transaction log metrics.add(new Metric("content.proton.transactionlog.entries.average")); metrics.add(new Metric("content.proton.transactionlog.disk_usage.average")); + metrics.add(new Metric("content.proton.transactionlog.replay_time.last")); // document store metrics.add(new Metric("content.proton.documentdb.ready.document_store.disk_usage.average")); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminBuilderBase.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminBuilderBase.java index a3a1a0fbebc..24e56472688 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminBuilderBase.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminBuilderBase.java @@ -37,16 +37,19 @@ public abstract class DomAdminBuilderBase extends VespaDomBuilder.DomConfigProdu private final ApplicationType applicationType; private final List<ConfigServerSpec> configServerSpecs; private final FileRegistry fileRegistry; + private final boolean disableFiledistributor; protected final boolean multitenant; - public DomAdminBuilderBase(ApplicationType applicationType, FileRegistry fileRegistry, boolean multitenant, List<ConfigServerSpec> configServerSpecs) { + DomAdminBuilderBase(ApplicationType applicationType, FileRegistry fileRegistry, boolean multitenant, + List<ConfigServerSpec> configServerSpecs, boolean disableFiledistributor) { this.applicationType = applicationType; this.fileRegistry = fileRegistry; this.multitenant = multitenant; this.configServerSpecs = configServerSpecs; + this.disableFiledistributor = disableFiledistributor; } - protected List<Configserver> getConfigServersFromSpec(AbstractConfigProducer parent) { + List<Configserver> getConfigServersFromSpec(AbstractConfigProducer parent) { List<Configserver> configservers = new ArrayList<>(); for (ConfigServerSpec spec : configServerSpecs) { HostSystem hostSystem = parent.getHostSystem(); @@ -64,23 +67,26 @@ public abstract class DomAdminBuilderBase extends VespaDomBuilder.DomConfigProdu @Override protected Admin doBuild(AbstractConfigProducer parent, Element adminElement) { Monitoring monitoring = getMonitoring(getChildWithFallback(adminElement, "monitoring", "yamas")); - Metrics metrics = new MetricsBuilder(applicationType, predefinedMetricSets) .buildMetrics(XML.getChild(adminElement, "metrics")); Map<String, MetricsConsumer> legacyMetricsConsumers = DomMetricBuilderHelper .buildMetricsConsumers(XML.getChild(adminElement, "metric-consumers")); + FileDistributionConfigProducer fileDistributionConfigProducer = getFileDistributionConfigProducer(parent, adminElement); - Admin admin = new Admin(parent, monitoring, metrics, legacyMetricsConsumers, multitenant); - + Admin admin = new Admin(parent, monitoring, metrics, legacyMetricsConsumers, multitenant, fileDistributionConfigProducer); doBuildAdmin(admin, adminElement); - new ModelConfigProvider(admin); - FileDistributionOptions fileDistributionOptions = new DomFileDistributionOptionsBuilder().build(XML.getChild(adminElement, "filedistribution")); - admin.setFileDistribution(new FileDistributionConfigProducer.Builder(fileDistributionOptions).build(parent, fileRegistry)); return admin; } - + + private FileDistributionConfigProducer getFileDistributionConfigProducer(AbstractConfigProducer parent, Element adminElement) { + FileDistributionOptions fileDistributionOptions = FileDistributionOptions.defaultOptions(); + fileDistributionOptions.disableFiledistributor(disableFiledistributor); + fileDistributionOptions = new DomFileDistributionOptionsBuilder(fileDistributionOptions).build(XML.getChild(adminElement, "filedistribution")); + return new FileDistributionConfigProducer.Builder(fileDistributionOptions).build(parent, fileRegistry); + } + private Element getChildWithFallback(Element parent, String childName, String alternativeChildName) { Element child = XML.getChild(parent, childName); if (child != null) return child; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2Builder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2Builder.java index dd1d4e36255..d966f3b49f6 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2Builder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2Builder.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.builder.xml.dom; +import com.yahoo.config.application.api.FileRegistry; import com.yahoo.config.model.ConfigModelContext; import com.yahoo.config.model.api.ConfigServerSpec; import com.yahoo.config.model.producer.AbstractConfigProducer; @@ -18,7 +19,6 @@ import com.yahoo.vespa.model.builder.xml.dom.VespaDomBuilder.DomConfigProducerBu import com.yahoo.vespa.model.container.Container; import com.yahoo.vespa.model.container.ContainerCluster; import com.yahoo.vespa.model.container.xml.ContainerModelBuilder; -import com.yahoo.config.application.api.FileRegistry; import org.w3c.dom.Element; import java.util.List; @@ -37,8 +37,9 @@ public class DomAdminV2Builder extends DomAdminBuilderBase { public DomAdminV2Builder(ConfigModelContext.ApplicationType applicationType, FileRegistry fileRegistry, boolean multitenant, - List<ConfigServerSpec> configServerSpecs) { - super(applicationType, fileRegistry, multitenant, configServerSpecs); + List<ConfigServerSpec> configServerSpecs, + boolean disableFiledistributor) { + super(applicationType, fileRegistry, multitenant, configServerSpecs, disableFiledistributor); } @Override diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java index cb8ec205395..a7a785cea43 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java @@ -30,8 +30,10 @@ public class DomAdminV4Builder extends DomAdminBuilderBase { private final Collection<ContainerModel> containerModels; private final ConfigModelContext context; - public DomAdminV4Builder(ConfigModelContext context, boolean multitenant, List<ConfigServerSpec> configServerSpecs, Collection<ContainerModel> containerModels) { - super(context.getApplicationType(), context.getDeployState().getFileRegistry(), multitenant, configServerSpecs); + public DomAdminV4Builder(ConfigModelContext context, boolean multitenant, List<ConfigServerSpec> configServerSpecs, + Collection<ContainerModel> containerModels, boolean disableFiledistributor) { + super(context.getApplicationType(), context.getDeployState().getFileRegistry(), multitenant, + configServerSpecs, disableFiledistributor); this.containerModels = containerModels; this.context = context; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomFileDistributionOptionsBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomFileDistributionOptionsBuilder.java index 8a5d6846a64..9cbdd00899b 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomFileDistributionOptionsBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomFileDistributionOptionsBuilder.java @@ -15,6 +15,11 @@ import java.util.Optional; * @author hmusum */ public class DomFileDistributionOptionsBuilder { + private final FileDistributionOptions fileDistributionOptions; + + public DomFileDistributionOptionsBuilder(FileDistributionOptions fileDistributionOptions) { + this.fileDistributionOptions = fileDistributionOptions; + } private static void throwExceptionForElementInFileDistribution(String subElement, String reason) { throw new RuntimeException("In element '" + subElement + "' contained in 'filedistribution': " + reason); @@ -34,15 +39,15 @@ public class DomFileDistributionOptionsBuilder { } public FileDistributionOptions build(Element fileDistributionElement) { - FileDistributionOptions options = FileDistributionOptions.defaultOptions(); if (fileDistributionElement != null) { - getAmount("uploadbitrate", fileDistributionElement).ifPresent(options::uploadBitRate); - getAmount("downloadbitrate", fileDistributionElement).ifPresent(options::downloadBitRate); + getAmount("uploadbitrate", fileDistributionElement).ifPresent(fileDistributionOptions::uploadBitRate); + getAmount("downloadbitrate", fileDistributionElement).ifPresent(fileDistributionOptions::downloadBitRate); Element disable = XML.getChild(fileDistributionElement, "disabled"); + if (disable == null) disable = XML.getChild(fileDistributionElement, "disableFiledistributor"); if (disable != null) { - options.disabled(Boolean.valueOf(XML.getValue(disable))); + fileDistributionOptions.disableFiledistributor(Boolean.valueOf(XML.getValue(disable))); } } - return options; + return fileDistributionOptions; } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/clients/VespaSpoolerService.java b/config-model/src/main/java/com/yahoo/vespa/model/clients/VespaSpoolerService.java index 4969132ccfb..f4287cf982e 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/clients/VespaSpoolerService.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/clients/VespaSpoolerService.java @@ -19,7 +19,6 @@ public class VespaSpoolerService extends AbstractService implements SpoolerConfi public VespaSpoolerService(AbstractConfigProducer parent, int index, VespaSpooler spooler) { super(parent, "spooler." + index); this.spooler = spooler; - monitorService("spooler"); } public int getPortCount() { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java b/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java index 8991bfa6215..5925ec978bb 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java @@ -165,7 +165,6 @@ public class Container extends AbstractService implements } tagServers(); - monitorService(); } private void tagServers() { @@ -331,7 +330,7 @@ public class Container extends AbstractService implements FileDistributionConfigProducer fileDistribution = getRoot().getFileDistributionConfigProducer(); if (fileDistribution != null) { - builder.configid(fileDistribution.getFileDistributorService(getHost()).getConfigId()); + builder.configid(fileDistribution.getConfigProducer(getHost()).getConfigId()); } return builder; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/ConfigserverCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/ConfigserverCluster.java index 62828b314d0..bce78017bdd 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/ConfigserverCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/ConfigserverCluster.java @@ -144,6 +144,9 @@ public class ConfigserverCluster extends AbstractConfigProducer if (options.loadBalancerAddress().isPresent()) { builder.loadBalancerAddress(options.loadBalancerAddress().get()); } + if (options.disableFiledistributor().isPresent()) { + builder.disableFiledistributor(options.disableFiledistributor().get()); + } } private String[] getConfigModelPluginDirs() { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/option/CloudConfigOptions.java b/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/option/CloudConfigOptions.java index aeb86ae9d59..866bae6666a 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/option/CloudConfigOptions.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/option/CloudConfigOptions.java @@ -4,7 +4,7 @@ package com.yahoo.vespa.model.container.configserver.option; import java.util.Optional; /** - * @author tonytv + * @author Tony Vaagenes */ public interface CloudConfigOptions { @@ -44,4 +44,5 @@ public interface CloudConfigOptions { Optional<String> dockerRegistry(); Optional<String> dockerVespaBaseImage(); Optional<String> loadBalancerAddress(); + Optional<Boolean> disableFiledistributor(); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/ConnectorFactory.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/ConnectorFactory.java index 22c42056b3f..33f5edded3c 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/http/ConnectorFactory.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/ConnectorFactory.java @@ -5,6 +5,7 @@ import com.yahoo.component.ComponentId; import com.yahoo.container.bundle.BundleInstantiationSpecification; import com.yahoo.jdisc.http.ConnectorConfig; import com.yahoo.jdisc.http.ssl.DefaultSslKeyStoreConfigurator; +import com.yahoo.jdisc.http.ssl.DefaultSslTrustStoreConfigurator; import com.yahoo.osgi.provider.model.ComponentModel; import com.yahoo.text.XML; import com.yahoo.vespa.model.container.component.SimpleComponent; @@ -24,10 +25,14 @@ public class ConnectorFactory extends SimpleComponent implements ConnectorConfig private final Element legacyConfig; public ConnectorFactory(String name, int listenPort) { - this(name, listenPort, null, null); + this(name, listenPort, null, null, null); } - public ConnectorFactory(String name, int listenPort, Element legacyConfig, Element sslKeystoreConfigurator) { + public ConnectorFactory(String name, + int listenPort, + Element legacyConfig, + Element sslKeystoreConfigurator, + Element sslTruststoreConfigurator) { super(new ComponentModel( new BundleInstantiationSpecification(new ComponentId(name), fromString("com.yahoo.jdisc.http.server.jetty.ConnectorFactory"), @@ -35,9 +40,8 @@ public class ConnectorFactory extends SimpleComponent implements ConnectorConfig this.name = name; this.listenPort = listenPort; this.legacyConfig = legacyConfig; - SimpleComponent sslKeyStoreConfigurator = getSslKeyStoreConfigurator(name, sslKeystoreConfigurator); - addChild(sslKeyStoreConfigurator); - inject(sslKeyStoreConfigurator); + addSslKeyStoreConfigurator(name, sslKeystoreConfigurator); + addSslTrustStoreConfigurator(name, sslTruststoreConfigurator); } @Override @@ -149,16 +153,30 @@ public class ConnectorFactory extends SimpleComponent implements ConnectorConfig } } - private static SimpleComponent getSslKeyStoreConfigurator(String name, Element sslKeystoreConfigurator) { - String idSpec = "ssl-keystore-configurator@" + name; - if (sslKeystoreConfigurator != null) { - String className = sslKeystoreConfigurator.getAttribute("class"); - String bundleName = sslKeystoreConfigurator.getAttribute("bundle"); - return new SimpleComponent(new ComponentModel(idSpec, className, bundleName)); + private void addSslKeyStoreConfigurator(String name, Element sslKeystoreConfigurator) { + addSslConfigurator("ssl-keystore-configurator@" + name, + DefaultSslKeyStoreConfigurator.class, + sslKeystoreConfigurator); + } + + private void addSslTrustStoreConfigurator(String name, Element sslKeystoreConfigurator) { + addSslConfigurator("ssl-truststore-configurator@" + name, + DefaultSslTrustStoreConfigurator.class, + sslKeystoreConfigurator); + } + + private void addSslConfigurator(String idSpec, Class<?> defaultImplementation, Element configuratorElement) { + SimpleComponent configuratorComponent; + if (configuratorElement != null) { + String className = configuratorElement.getAttribute("class"); + String bundleName = configuratorElement.getAttribute("bundle"); + configuratorComponent = new SimpleComponent(new ComponentModel(idSpec, className, bundleName)); } else { - return new SimpleComponent( - new ComponentModel(idSpec, DefaultSslKeyStoreConfigurator.class.getName(), "jdisc_http_service")); + configuratorComponent = + new SimpleComponent(new ComponentModel(idSpec, defaultImplementation.getName(), "jdisc_http_service")); } + addChild(configuratorComponent); + inject(configuratorComponent); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/JettyConnectorBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/JettyConnectorBuilder.java index f2012a609a7..f88c091cd37 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/JettyConnectorBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/JettyConnectorBuilder.java @@ -12,8 +12,7 @@ import java.util.logging.Level; import java.util.logging.Logger; /** - * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> - * @since 5.21.0 + * @author Einar M R Rosenvinge */ public class JettyConnectorBuilder extends VespaDomBuilder.DomConfigProducerBuilder<ConnectorFactory> { private static final Logger log = Logger.getLogger(JettyConnectorBuilder.class.getName()); @@ -34,7 +33,8 @@ public class JettyConnectorBuilder extends VespaDomBuilder.DomConfigProducerBuil } } Element sslKeystoreConfigurator = XML.getChild(serverSpec, "ssl-keystore-configurator"); - return new ConnectorFactory(name, port, legacyServerConfig, sslKeystoreConfigurator); + Element sslTruststoreConfigurator = XML.getChild(serverSpec, "ssl-truststore-configurator"); + return new ConnectorFactory(name, port, legacyServerConfig, sslKeystoreConfigurator, sslTruststoreConfigurator); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentNode.java b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentNode.java index 7ef54360f00..4257e47218e 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentNode.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentNode.java @@ -50,8 +50,6 @@ public abstract class ContentNode extends AbstractService portsMeta.on(0).tag("messaging"); portsMeta.on(1).tag("rpc").tag("status"); portsMeta.on(2).tag("http").tag("status").tag("state"); - - monitorService(); } @Override diff --git a/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/DummyFileDistributionConfigProducer.java b/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/DummyFileDistributionConfigProducer.java new file mode 100644 index 00000000000..1b1e66594d1 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/DummyFileDistributionConfigProducer.java @@ -0,0 +1,42 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.filedistribution; + +import com.yahoo.cloud.config.filedistribution.FiledistributorConfig; +import com.yahoo.cloud.config.filedistribution.FiledistributorrpcConfig; +import com.yahoo.cloud.config.filedistribution.FilereferencesConfig; +import com.yahoo.config.model.producer.AbstractConfigProducer; + +/** + * @author hmusum + * <p> + * Dummy file distribution config producer, needed for serving file distribution config when there is no FiledistributorService. + */ +public class DummyFileDistributionConfigProducer extends AbstractConfigProducer implements + FiledistributorConfig.Producer, + FiledistributorrpcConfig.Producer, + FilereferencesConfig.Producer { + + private final FileDistributionConfigProvider configProvider; + + public DummyFileDistributionConfigProducer(AbstractConfigProducer parent, + String hostname, + FileDistributionConfigProvider configProvider) { + super(parent, hostname); + this.configProvider = configProvider; + } + + @Override + public void getConfig(FiledistributorConfig.Builder builder) { + configProvider.getConfig(builder); + } + + @Override + public void getConfig(FiledistributorrpcConfig.Builder builder) { + configProvider.getConfig(builder); + } + + @Override + public void getConfig(FilereferencesConfig.Builder builder) { + configProvider.getConfig(builder); + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributionConfigProducer.java b/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributionConfigProducer.java index 9e5e3074c5e..c863b180b21 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributionConfigProducer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributionConfigProducer.java @@ -10,11 +10,13 @@ import java.util.IdentityHashMap; import java.util.Map; /** - * @author tonytv + * @author hmusum + * <p> + * File distribution config producer, delegates getting config to {@link DummyFileDistributionConfigProducer} (one per host) */ public class FileDistributionConfigProducer extends AbstractConfigProducer { - private final Map<Host, FileDistributorService> fileDistributorServices = new IdentityHashMap<>(); + private final Map<Host, AbstractConfigProducer> fileDistributionConfigProducers = new IdentityHashMap<>(); private final FileDistributor fileDistributor; private final FileDistributionOptions options; @@ -24,14 +26,6 @@ public class FileDistributionConfigProducer extends AbstractConfigProducer { this.options = options; } - public FileDistributorService getFileDistributorService(Host host) { - FileDistributorService service = fileDistributorServices.get(host); - if (service == null) { - throw new IllegalStateException("No file distribution service for host " + host); - } - return service; - } - public FileDistributor getFileDistributor() { return fileDistributor; } @@ -40,8 +34,8 @@ public class FileDistributionConfigProducer extends AbstractConfigProducer { return options; } - public void addFileDistributionService(Host host, FileDistributorService fds) { - fileDistributorServices.put(host, fds); + public void addFileDistributionConfigProducer(Host host, AbstractConfigProducer fileDistributionConfigProducer) { + fileDistributionConfigProducers.put(host, fileDistributionConfigProducer); } public static class Builder { @@ -58,4 +52,8 @@ public class FileDistributionConfigProducer extends AbstractConfigProducer { } } + public AbstractConfigProducer getConfigProducer(Host host) { + return fileDistributionConfigProducers.get(host); + } + } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributionConfigProvider.java b/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributionConfigProvider.java new file mode 100644 index 00000000000..33cc581c3a8 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributionConfigProvider.java @@ -0,0 +1,59 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.filedistribution; + +import com.yahoo.cloud.config.filedistribution.FiledistributorConfig; +import com.yahoo.cloud.config.filedistribution.FiledistributorrpcConfig; +import com.yahoo.cloud.config.filedistribution.FilereferencesConfig; +import com.yahoo.config.FileReference; +import com.yahoo.config.model.api.FileDistribution; +import com.yahoo.vespa.model.ConfigProxy; +import com.yahoo.vespa.model.Host; +import com.yahoo.vespa.model.admin.FileDistributionOptions; + +import java.util.Collection; + +public class FileDistributionConfigProvider { + + private final FileDistributor fileDistributor; + private final FileDistributionOptions fileDistributionOptions; + private final boolean sendAllFiles; + private final Host host; + + public FileDistributionConfigProvider(FileDistributor fileDistributor, + FileDistributionOptions fileDistributionOptions, + boolean sendAllFiles, + Host host) { + this.fileDistributor = fileDistributor; + this.fileDistributionOptions = fileDistributionOptions; + this.sendAllFiles = sendAllFiles; + this.host = host; + } + + public void getConfig(FiledistributorConfig.Builder builder) { + fileDistributionOptions.getConfig(builder); + builder.torrentport(FileDistributorService.BASEPORT + 1); + builder.stateport(FileDistributorService.BASEPORT + 2); + builder.hostname(host.getHostName()); + builder.filedbpath(FileDistribution.getDefaultFileDBPath().toString()); + } + + public void getConfig(FiledistributorrpcConfig.Builder builder) { + // If disabled config proxy should act as file distributor, so use config proxy port + int port = (fileDistributionOptions.disableFiledistributor()) ? ConfigProxy.BASEPORT : FileDistributorService.BASEPORT; + builder.connectionspec("tcp/" + host.getHostName() + ":" + port); + } + + public void getConfig(FilereferencesConfig.Builder builder) { + for (FileReference reference : getFileReferences()) { + builder.filereferences(reference.value()); + } + } + + private Collection<FileReference> getFileReferences() { + if (sendAllFiles) { + return fileDistributor.allFilesToSend(); + } else { + return fileDistributor.filesToSendToHost(host); + } + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributor.java b/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributor.java index f6cc9203d00..8fcece3aa80 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributor.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributor.java @@ -9,13 +9,10 @@ import com.yahoo.vespa.model.Host; import java.util.*; import java.util.stream.Collectors; -import static java.util.Arrays.asList; - - /** * Responsible for directing distribution of files to hosts. * - * @author tonytv + * @author Tony Vaagenes */ public class FileDistributor { @@ -99,6 +96,7 @@ public class FileDistributor { for (Host host : getTargetHosts()) { if ( ! host.getHostName().equals(fileSourceHost)) { dbHandler.sendDeployedFiles(host.getHostName(), filesToSendToHost(host)); + dbHandler.startDownload(host.getHostName(), filesToSendToHost(host)); } } dbHandler.sendDeployedFiles(fileSourceHost, allFilesToSend()); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributorService.java b/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributorService.java index dd9e057e2fa..986334125d2 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributorService.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributorService.java @@ -4,60 +4,36 @@ package com.yahoo.vespa.model.filedistribution; import com.yahoo.cloud.config.filedistribution.FiledistributorConfig; import com.yahoo.cloud.config.filedistribution.FiledistributorrpcConfig; import com.yahoo.cloud.config.filedistribution.FilereferencesConfig; -import com.yahoo.config.FileReference; - -import com.yahoo.config.model.api.FileDistribution; import com.yahoo.config.model.producer.AbstractConfigProducer; import com.yahoo.vespa.model.AbstractService; -import com.yahoo.vespa.model.admin.FileDistributionOptions; - -import java.util.Collection; /** - * @author tonytv + * @author Tony Vaagenes + * + * Config is produced by {@link FileDistributionConfigProvider} */ public class FileDistributorService extends AbstractService implements FiledistributorConfig.Producer, FiledistributorrpcConfig.Producer, FilereferencesConfig.Producer { - private final static int BASEPORT = 19092; - private final FileDistributor fileDistributor; - private final FileDistributionOptions fileDistributionOptions; - private final boolean sendAllFiles; + final static int BASEPORT = 19092; - private Collection<FileReference> getFileReferences() { - if (sendAllFiles) { - return fileDistributor.allFilesToSend(); - } else { - return fileDistributor.filesToSendToHost(getHost()); - } - } + private final FileDistributionConfigProvider configProvider; - public FileDistributorService(AbstractConfigProducer parent, - String name, - FileDistributor fileDistributor, - FileDistributionOptions fileDistributionOptions, - boolean sendAllFiles) { - super(parent, name); + public FileDistributorService(AbstractConfigProducer parent, String hostname, FileDistributionConfigProvider configProvider) { + super(parent, hostname); + this.configProvider = configProvider; portsMeta.on(0).tag("rpc"); portsMeta.on(1).tag("torrent"); portsMeta.on(2).tag("http").tag("state"); setProp("clustertype", "filedistribution"); setProp("clustername", "admin"); - - this.fileDistributor = fileDistributor; - this.fileDistributionOptions = fileDistributionOptions; - this.sendAllFiles = sendAllFiles; - monitorService(); } @Override public String getStartupCommand() { - // If disabled config proxy should act as file distributor, so don't start this service - return (fileDistributionOptions.disabled()) - ? null - : "exec $ROOT/sbin/vespa-filedistributor" + " --configid " + getConfigId(); + return "exec $ROOT/sbin/vespa-filedistributor" + " --configid " + getConfigId(); } @Override @@ -70,6 +46,7 @@ public class FileDistributorService extends AbstractService implements return true; } + @Override public int getPortCount() { return 3; } @@ -81,24 +58,16 @@ public class FileDistributorService extends AbstractService implements @Override public void getConfig(FiledistributorConfig.Builder builder) { - fileDistributionOptions.getConfig(builder); - builder.torrentport(getRelativePort(1)); - builder.stateport(getRelativePort(2)); - builder.hostname(getHostName()); - builder.filedbpath(FileDistribution.getDefaultFileDBPath().toString()); + configProvider.getConfig(builder); } @Override public void getConfig(FiledistributorrpcConfig.Builder builder) { - // If disabled config proxy should act as file distributor, so use config proxy port - int port = (fileDistributionOptions.disabled()) ? 19090 : getRelativePort(0); - builder.connectionspec("tcp/" + getHostName() + ":" + port); + configProvider.getConfig(builder); } @Override public void getConfig(FilereferencesConfig.Builder builder) { - for (FileReference reference : getFileReferences()) { - builder.filereferences(reference.value()); - } + configProvider.getConfig(builder); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/Dispatch.java b/config-model/src/main/java/com/yahoo/vespa/model/search/Dispatch.java index c0867d252a3..789c5562d9c 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/Dispatch.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/Dispatch.java @@ -46,7 +46,6 @@ public class Dispatch extends AbstractService implements SearchInterface, setProp("clustertype", "search") .setProp("clustername", dispatchGroup.getClusterName()) .setProp("index", nodeSpec.groupIndex()); - monitorService(); } public static Dispatch createTld(DispatchGroup dispatchGroup, AbstractConfigProducer parent, int rowId) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java index c5f7272ee80..819d5904c25 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java @@ -19,6 +19,7 @@ import com.yahoo.vespa.model.admin.monitoring.Monitoring; import com.yahoo.vespa.model.application.validation.RestartConfigs; import com.yahoo.vespa.model.builder.xml.dom.VespaDomBuilder; import com.yahoo.vespa.model.content.ContentNode; +import com.yahoo.vespa.model.filedistribution.DummyFileDistributionConfigProducer; import com.yahoo.vespa.model.filedistribution.FileDistributionConfigProducer; import com.yahoo.vespa.model.filedistribution.FileDistributorService; import org.w3c.dom.Element; @@ -112,7 +113,6 @@ public class SearchNode extends AbstractService implements portsMeta.on(UNUSED_3).tag("unused"); portsMeta.on(HEALTH_PORT).tag("http").tag("json").tag("health").tag("state"); // Properties are set in DomSearchBuilder - monitorService(); this.tuning = tuning; } @@ -229,10 +229,12 @@ public class SearchNode extends AbstractService implements public void getConfig(FiledistributorrpcConfig.Builder builder) { FileDistributionConfigProducer fileDistribution = getRoot().getFileDistributionConfigProducer(); if (fileDistribution != null) { - FileDistributorService fds = fileDistribution.getFileDistributorService(getHost()); - if (fds != null) { - fds.getConfig(builder); - } + AbstractConfigProducer configProducer = fileDistribution.getConfigProducer(getHost()); + // TODO: Hack, will be fixed when FileDistributorService is gone + if (configProducer instanceof DummyFileDistributionConfigProducer) + ((DummyFileDistributionConfigProducer) configProducer).getConfig(builder); + else + ((FileDistributorService) configProducer).getConfig(builder); } } diff --git a/config-model/src/main/resources/schema/admin.rnc b/config-model/src/main/resources/schema/admin.rnc index b7ec252c4d7..486f1e367e9 100644 --- a/config-model/src/main/resources/schema/admin.rnc +++ b/config-model/src/main/resources/schema/admin.rnc @@ -81,7 +81,7 @@ LogServer = element logserver { FileDistribution = element filedistribution { element uploadbitrate { xsd:string { pattern = "\d+(\.\d*)?\s*[kmgKMG]?" } }? & element downloadbitrate { xsd:string { pattern = "\d+(\.\d*)?\s*[kmgKMG]?" } }? & - element disabled { xsd:boolean }? # Nov. 2017: Temporary, should not be documented + element disableFiledistributor { xsd:boolean }? # Nov. 2017: Temporary, should not be documented } Metrics = element metrics { diff --git a/config-model/src/main/resources/schema/containercluster.rnc b/config-model/src/main/resources/schema/containercluster.rnc index 6a90bef7bb2..95ac198adc4 100644 --- a/config-model/src/main/resources/schema/containercluster.rnc +++ b/config-model/src/main/resources/schema/containercluster.rnc @@ -62,6 +62,7 @@ HttpServer = element server { attribute port { xsd:nonNegativeInteger } & ComponentId & element ssl-keystore-configurator { BundleSpec }? & # FOR INTERNAL USE ONLY - SUBJECT TO CHANGE + element ssl-truststore-configurator { BundleSpec }? & # FOR INTERNAL USE ONLY - SUBJECT TO CHANGE GenericConfig* } diff --git a/config-model/src/test/cfg/application/invalid_legacy_user_config/configs/qr-searchers.cfg b/config-model/src/test/cfg/application/invalid_legacy_user_config/configs/qr-searchers.cfg deleted file mode 100644 index 07538a1dce2..00000000000 --- a/config-model/src/test/cfg/application/invalid_legacy_user_config/configs/qr-searchers.cfg +++ /dev/null @@ -1,5 +0,0 @@ -builtin[StemmingSearcher].enabled false -builtin[NoRankingSearcher].enabled false -tag.bold.open "^_" -tag.bold.close "^_" -tag.separator " ... " diff --git a/config-model/src/test/cfg/application/invalid_legacy_user_config/services.xml b/config-model/src/test/cfg/application/invalid_legacy_user_config/services.xml deleted file mode 100644 index 9a13d936e46..00000000000 --- a/config-model/src/test/cfg/application/invalid_legacy_user_config/services.xml +++ /dev/null @@ -1,10 +0,0 @@ -<?xml version="1.0" encoding="utf-8" ?> -<!-- Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> -<services version="1.0"> - - - <admin version="2.0"> - <adminserver hostalias="node1" /> - </admin> - -</services> diff --git a/config-model/src/test/derived/importedfields/attributes.cfg b/config-model/src/test/derived/importedfields/attributes.cfg index 62fe0052bc3..f51bbc012b9 100644 --- a/config-model/src/test/derived/importedfields/attributes.cfg +++ b/config-model/src/test/derived/importedfields/attributes.cfg @@ -138,3 +138,23 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported true +attribute[].name "my_predicate_field" +attribute[].datatype PREDICATE +attribute[].collectiontype SINGLE +attribute[].removeifzero false +attribute[].createifnonexistent false +attribute[].fastsearch false +attribute[].huge false +attribute[].sortascending true +attribute[].sortfunction UCA +attribute[].sortstrength PRIMARY +attribute[].sortlocale "" +attribute[].enablebitvectors false +attribute[].enableonlybitvector false +attribute[].fastaccess false +attribute[].arity 8 +attribute[].lowerbound 5 +attribute[].upperbound 300 +attribute[].densepostinglistthreshold 0.4 +attribute[].tensortype "" +attribute[].imported true diff --git a/config-model/src/test/derived/importedfields/child.sd b/config-model/src/test/derived/importedfields/child.sd index 703451c1293..07fa7d15719 100644 --- a/config-model/src/test/derived/importedfields/child.sd +++ b/config-model/src/test/derived/importedfields/child.sd @@ -9,6 +9,7 @@ search child { import field b_ref.string_field as my_string_field {} import field a_ref.int_array_field as my_int_array_field {} import field a_ref.int_wset_field as my_int_wset_field {} + import field a_ref.predicate_field as my_predicate_field {} fieldset myfieldset { fields: my_int_field, my_string_field diff --git a/config-model/src/test/derived/importedfields/imported-fields.cfg b/config-model/src/test/derived/importedfields/imported-fields.cfg index 3f2a083bdc2..59413ca5eb7 100644 --- a/config-model/src/test/derived/importedfields/imported-fields.cfg +++ b/config-model/src/test/derived/importedfields/imported-fields.cfg @@ -18,3 +18,8 @@ attribute[].referencefield "a_ref" attribute[].targetfield "int_wset_field" attribute[].datatype NONE attribute[].collectiontype SINGLE +attribute[].name "my_predicate_field" +attribute[].referencefield "a_ref" +attribute[].targetfield "predicate_field" +attribute[].datatype NONE +attribute[].collectiontype SINGLE diff --git a/config-model/src/test/derived/importedfields/index-info.cfg b/config-model/src/test/derived/importedfields/index-info.cfg index 7cb53c480ce..adba0036409 100644 --- a/config-model/src/test/derived/importedfields/index-info.cfg +++ b/config-model/src/test/derived/importedfields/index-info.cfg @@ -49,6 +49,12 @@ indexinfo[].command[].indexname "my_int_wset_field" indexinfo[].command[].command "multivalue" indexinfo[].command[].indexname "my_int_wset_field" indexinfo[].command[].command "attribute" +indexinfo[].command[].indexname "my_predicate_field" +indexinfo[].command[].command "predicate-bounds [5..300]" +indexinfo[].command[].indexname "my_predicate_field" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "my_predicate_field" +indexinfo[].command[].command "attribute" indexinfo[].command[].indexname "myfieldset" indexinfo[].command[].command "attribute" indexinfo[].command[].indexname "myfieldset" diff --git a/config-model/src/test/derived/importedfields/parent_a.sd b/config-model/src/test/derived/importedfields/parent_a.sd index eea12375daf..edc81df5609 100644 --- a/config-model/src/test/derived/importedfields/parent_a.sd +++ b/config-model/src/test/derived/importedfields/parent_a.sd @@ -10,5 +10,13 @@ search parent_a { field int_wset_field type weightedset<int> { indexing: attribute } + field predicate_field type predicate { + indexing: attribute + index { + arity: 8 + lower-bound: 5 + upper-bound: 300 + } + } } } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImportedFieldsInSummaryValidatorTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImportedFieldsInSummaryValidatorTestCase.java new file mode 100644 index 00000000000..4e1f8f1edd7 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImportedFieldsInSummaryValidatorTestCase.java @@ -0,0 +1,39 @@ +package com.yahoo.searchdefinition.processing; + +import com.yahoo.document.DataType; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; + +/** + * @author geirst + */ +public class ImportedFieldsInSummaryValidatorTestCase { + + @Rule + public final ExpectedException exceptionRule = ExpectedException.none(); + + @Test + public void validator_fails_if_imported_predicate_field_is_used_in_document_summary() { + exceptionRule.expect(IllegalArgumentException.class); + exceptionRule.expectMessage("For search 'child', document summary 'my_summary', " + + "imported summary field 'my_predicate_field': Is of type predicate. Not supported in document summaries"); + new SearchModel() + .addImportedField("my_predicate_field", "ref", "predicate_field") + .addSummaryField("my_predicate_field", DataType.PREDICATE) + .resolve(); + } + + private static class SearchModel extends ImportedFieldsResolverTestCase.SearchModel { + + public SearchModel() { + super(); + } + + public void resolve() { + super.resolve(); + new ImportedFieldsInSummayValidator(childSearch, null, null, null).process(); + } + } +} + diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImportedFieldsResolverTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImportedFieldsResolverTestCase.java index 5d270bc085a..9f7c7458738 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImportedFieldsResolverTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImportedFieldsResolverTestCase.java @@ -18,6 +18,8 @@ import com.yahoo.searchdefinition.document.SDField; import com.yahoo.searchdefinition.document.TemporaryImportedField; import com.yahoo.searchdefinition.document.TemporarySDField; import com.yahoo.tensor.TensorType; +import com.yahoo.vespa.documentmodel.DocumentSummary; +import com.yahoo.vespa.documentmodel.SummaryField; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; @@ -107,7 +109,7 @@ public class ImportedFieldsResolverTestCase { .resolve(); } - private static class SearchModel { + static class SearchModel { private final ApplicationPackage app = MockApplicationPackage.createEmpty(); public final Search grandParentSearch; @@ -124,6 +126,7 @@ public class ImportedFieldsResolverTestCase { parentSearch.getDocument().addField(createField("attribute_and_index", DataType.INT, "{ attribute | index }")); parentSearch.getDocument().addField(new TemporarySDField("not_attribute", DataType.INT)); parentSearch.getDocument().addField(createField("tensor_field", new TensorDataType(TensorType.fromSpec("tensor(x[])")), "{ attribute }")); + parentSearch.getDocument().addField(createField("predicate_field", DataType.PREDICATE, "{ attribute }")); addRefField(parentSearch, grandParentSearch, "ref"); addImportedField(parentSearch, "ancient_field", "ref", "ancient_field"); @@ -163,6 +166,16 @@ public class ImportedFieldsResolverTestCase { return this; } + public SearchModel addSummaryField(String fieldName, DataType dataType) { + DocumentSummary summary = childSearch.getSummary("my_summary"); + if (summary == null) { + summary = new DocumentSummary("my_summary"); + childSearch.addSummary(summary); + } + summary.add(new SummaryField(fieldName, dataType)); + return this; + } + public void resolve() { resolve(grandParentSearch); resolve(parentSearch); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/admin/AdminTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/admin/AdminTestCase.java index 946624a1cdb..fed9000f72b 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/admin/AdminTestCase.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/admin/AdminTestCase.java @@ -8,7 +8,6 @@ import com.yahoo.cloud.config.SentinelConfig; import com.yahoo.config.model.ApplicationConfigProducerRoot; import com.yahoo.config.model.deploy.DeployProperties; import com.yahoo.config.model.deploy.DeployState; -import com.yahoo.config.model.test.MockApplicationPackage; import com.yahoo.config.model.test.TestDriver; import com.yahoo.config.model.test.TestRoot; import com.yahoo.config.provision.ApplicationId; @@ -24,11 +23,9 @@ import com.yahoo.vespa.model.container.component.Component; import com.yahoo.vespa.model.container.component.StatisticsComponent; import com.yahoo.vespa.model.test.utils.VespaModelCreatorWithFilePkg; import com.yahoo.vespa.model.test.utils.VespaModelCreatorWithMockPkg; -import org.junit.Ignore; import org.junit.Test; import java.util.Set; -import java.util.stream.IntStream; import static org.hamcrest.CoreMatchers.is; import static org.junit.Assert.assertEquals; @@ -288,7 +285,7 @@ public class AdminTestCase { " <admin version='2.0'>" + " <adminserver hostalias='node0' />" + " <filedistribution>" + - " <disabled>true</disabled>" + + " <disableFiledistributor>true</disableFiledistributor>" + " </filedistribution>" + " </admin>" + "</services>"; @@ -307,4 +304,24 @@ public class AdminTestCase { assertThat(sentinelConfig.service(2).name(), is("logd")); // No filedistributor service } + + @Test + public void testDisableFileDistributorForAllApps() { + DeployState state = new DeployState.Builder() + .disableFiledistributor(true) + .properties( + new DeployProperties.Builder(). + zone(new Zone(Environment.dev, RegionName.from("baz"))). + applicationId(new ApplicationId.Builder(). + tenant("quux"). + applicationName("foo").instanceName("bim").build()).build()).build(); + TestRoot root = new TestDriver().buildModel(state); + String localhost = HostName.getLocalhost(); + SentinelConfig sentinelConfig = root.getConfig(SentinelConfig.class, "hosts/" + localhost); + assertThat(sentinelConfig.service().size(), is(3)); + assertThat(sentinelConfig.service(0).name(), is("logserver")); + assertThat(sentinelConfig.service(1).name(), is("slobrok")); + assertThat(sentinelConfig.service(2).name(), is("logd")); + } + } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2BuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2BuilderTest.java index 8bd62789d52..91f6dd65bda 100755 --- a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2BuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2BuilderTest.java @@ -207,7 +207,10 @@ public class DomAdminV2BuilderTest extends DomBuilderTest { } private Admin buildAdmin(Element xml, boolean multitenant, List<ConfigServerSpec> configServerSpecs) { - final DomAdminV2Builder domAdminBuilder = new DomAdminV2Builder(ConfigModelContext.ApplicationType.DEFAULT, root.getDeployState().getFileRegistry(), multitenant, configServerSpecs); + final DomAdminV2Builder domAdminBuilder = + new DomAdminV2Builder(ConfigModelContext.ApplicationType.DEFAULT, + root.getDeployState().getFileRegistry(), multitenant, + configServerSpecs, root.getDeployState().disableFiledistributor()); Admin admin = domAdminBuilder.build(root, xml); admin.addPerHostServices(root.getHostSystem().getHosts(), new DeployProperties.Builder().build()); return admin; diff --git a/config-model/src/test/java/com/yahoo/vespa/model/clients/test/Gateway20TestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/clients/test/Gateway20TestCase.java deleted file mode 100644 index e7df151a3c4..00000000000 --- a/config-model/src/test/java/com/yahoo/vespa/model/clients/test/Gateway20TestCase.java +++ /dev/null @@ -1,65 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.model.clients.test; - -import com.yahoo.container.ComponentsConfig; -import com.yahoo.container.QrConfig; -import com.yahoo.container.QrConfig.Builder; -import com.yahoo.net.HostName; -import com.yahoo.vespa.model.VespaModel; -import com.yahoo.vespa.model.test.utils.VespaModelCreatorWithFilePkg; -import org.junit.Test; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; - -import static org.junit.Assert.*; - -/** - * @author Gunnar Gauslaa Bergem - */ -public class Gateway20TestCase { - - private static String hostname = HostName.getLocalhost(); // Using the same way of getting hostname as filedistribution model - - @Test - public void testSimpleDocprocV3() throws Exception { - VespaModel model = new VespaModelCreatorWithFilePkg("src/test/cfg/clients/simpleconfig.v2.docprocv3").create(); - QrConfig qrConfig = new QrConfig((Builder) model.getConfig(new QrConfig.Builder(), "container/container.0")); - assertEquals(qrConfig.rpc().enabled(), true); - assertEquals("filedistribution/" + hostname, qrConfig.filedistributor().configid()); - assertEquals("container.container.0", qrConfig.discriminator()); - - ComponentsConfig componentsConfig = new ComponentsConfig((ComponentsConfig.Builder) model.getConfig(new ComponentsConfig.Builder(), "container/container.0")); - ArrayList<String> components = new ArrayList<>(); - for (ComponentsConfig.Components component : componentsConfig.components()) { - components.add(component.id()); - } - List<String> expectedComponents = Arrays.asList("com.yahoo.docproc.jdisc.DocumentProcessingHandler", - "com.yahoo.feedhandler.VespaFeedHandler", - "com.yahoo.feedhandler.VespaFeedHandlerCompatibility", - "com.yahoo.feedhandler.VespaFeedHandlerGet", - "com.yahoo.feedhandler.VespaFeedHandlerRemove", - "com.yahoo.feedhandler.VespaFeedHandlerRemoveLocation", - "com.yahoo.feedhandler.VespaFeedHandlerStatus", - "com.yahoo.feedhandler.VespaFeedHandlerVisit", - "com.yahoo.search.handler.SearchHandler", - "com.yahoo.container.jdisc.state.StateHandler"); - assertTrue(components.containsAll(expectedComponents)); - } - - @Test - public void testAdvanced() throws Exception { - VespaModel model = new VespaModelCreatorWithFilePkg("src/test/cfg/clients/advancedconfig.v2").create(); - - QrConfig qrConfig = new QrConfig((Builder) model.getConfig(new QrConfig.Builder(), "container/container.0")); - assertEquals(qrConfig.rpc().enabled(), true); - assertEquals(qrConfig.filedistributor().configid(), "filedistribution/" + hostname); - assertEquals("container.container.0", qrConfig.discriminator()); - - qrConfig = new QrConfig((Builder) model.getConfig(new QrConfig.Builder(), "container/container.0")); - assertEquals(qrConfig.rpc().enabled(), true); - assertEquals(qrConfig.filedistributor().configid(), "filedistribution/" + hostname); - } - -} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/configserver/TestOptions.java b/config-model/src/test/java/com/yahoo/vespa/model/container/configserver/TestOptions.java index e2c8f2e2c52..1784fe0e974 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/configserver/TestOptions.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/configserver/TestOptions.java @@ -6,8 +6,7 @@ import com.yahoo.vespa.model.container.configserver.option.CloudConfigOptions; import java.util.Optional; /** - * @author lulf - * @since 5. + * @author Ulf Lilleengen */ public class TestOptions implements CloudConfigOptions { private Optional<Integer> rpcPort = Optional.empty(); @@ -20,6 +19,7 @@ public class TestOptions implements CloudConfigOptions { private Optional<Boolean> useVespaVersionInRequest = Optional.empty(); private Optional<Boolean> hostedVespa = Optional.empty(); private Optional<Integer> numParallelTenantLoaders = Optional.empty(); + private Optional<Boolean> disableFiledistributor = Optional.empty(); @Override public Optional<Integer> rpcPort() { @@ -118,6 +118,9 @@ public class TestOptions implements CloudConfigOptions { @Override public Optional<String> loadBalancerAddress() { return Optional.empty(); } + @Override + public Optional<Boolean> disableFiledistributor() { return disableFiledistributor; } + public TestOptions numParallelTenantLoaders(int numLoaders) { this.numParallelTenantLoaders = Optional.of(numLoaders); return this; diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java index d09211aea45..5e093bdb32a 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java @@ -16,6 +16,7 @@ import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.Zone; import com.yahoo.container.ComponentsConfig; +import com.yahoo.container.QrConfig; import com.yahoo.container.config.StatisticsRequestHandler; import com.yahoo.container.core.ChainsConfig; import com.yahoo.container.core.VipStatusConfig; @@ -25,6 +26,7 @@ import com.yahoo.container.jdisc.JdiscBindingsConfig; import com.yahoo.container.servlet.ServletConfigConfig; import com.yahoo.container.usability.BindingsOverviewHandler; import com.yahoo.jdisc.http.ServletPathsConfig; +import com.yahoo.net.HostName; import com.yahoo.prelude.cluster.QrMonitorConfig; import com.yahoo.vespa.model.AbstractService; import com.yahoo.vespa.model.VespaModel; @@ -61,7 +63,6 @@ import static org.junit.Assert.fail; /** * @author gjoranv - * @since 5.1.9 */ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase { @@ -571,6 +572,41 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase { assertEquals(ContainerModelBuilder.HOSTED_VESPA_STATUS_FILE, vipStatusConfig.statusfile()); } + @Test + public void qrconfig_is_produced() throws IOException, SAXException { + String servicesXml = + "<services>" + + "<admin version='3.0'>" + + " <nodes count='1'/>" + + "</admin>" + + "<jdisc id ='default' version='1.0'>" + + " <nodes>" + + " <node hostalias='node1' />" + + " </nodes>" + + "</jdisc>" + + "</services>"; + + ApplicationPackage applicationPackage = new MockApplicationPackage.Builder() + .withServices(servicesXml) + .build(); + VespaModel model = new VespaModel(new NullConfigModelRegistry(), new DeployState.Builder() + .applicationPackage(applicationPackage) + .properties(new DeployProperties.Builder().build()) + .build()); + + String hostname = HostName.getLocalhost(); // Using the same way of getting hostname as filedistribution model + + QrConfig config = model.getConfig(QrConfig.class, "default/container.0"); + assertEquals("default.container.0", config.discriminator()); + assertEquals(19102, config.rpc().port()); + assertEquals("vespa/service/default/container.0", config.rpc().slobrokId()); + assertEquals(true, config.rpc().enabled()); + assertEquals("", config.rpc().host()); + assertEquals(false, config.restartOnDeploy()); + assertEquals(false, config.coveragereports()); + assertEquals("filedistribution/" + hostname, config.filedistributor().configid()); + } + private Element generateContainerElementWithRenderer(String rendererId) { return DomBuilderTest.parse( "<jdisc id='default' version='1.0'>", diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/JettyContainerModelBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/JettyContainerModelBuilderTest.java index 1e24b055095..54c4aabf44c 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/JettyContainerModelBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/JettyContainerModelBuilderTest.java @@ -7,6 +7,7 @@ import com.yahoo.container.bundle.BundleInstantiationSpecification; import com.yahoo.container.jdisc.FilterBindingsProvider; import com.yahoo.jdisc.http.ConnectorConfig; import com.yahoo.jdisc.http.ssl.DefaultSslKeyStoreConfigurator; +import com.yahoo.jdisc.http.ssl.DefaultSslTrustStoreConfigurator; import com.yahoo.vespa.model.container.ContainerCluster; import com.yahoo.vespa.model.container.component.SimpleComponent; import com.yahoo.vespa.model.container.http.ConnectorFactory; @@ -16,7 +17,9 @@ import org.w3c.dom.Element; import org.xml.sax.SAXException; import java.io.IOException; +import java.util.Arrays; import java.util.List; +import java.util.Set; import static com.yahoo.jdisc.http.ConnectorConfig.Ssl.KeyStoreType; import static org.hamcrest.CoreMatchers.equalTo; @@ -28,7 +31,6 @@ import static org.junit.Assert.assertThat; /** * @author einarmr - * @since 5.15 */ public class JettyContainerModelBuilderTest extends ContainerModelBuilderTestBase { @@ -190,12 +192,13 @@ public class JettyContainerModelBuilderTest extends ContainerModelBuilderTestBas } @Test - public void ssl_keystore_configurator_can_be_overriden() throws IOException, SAXException { + public void ssl_keystore_and_truststore_configurator_can_be_overriden() throws IOException, SAXException { Element clusterElem = DomBuilderTest.parse( "<jdisc id='default' version='1.0' jetty='true'>", " <http>", " <server port='9000' id='foo'>", " <ssl-keystore-configurator class='com.yahoo.MySslKeyStoreConfigurator' bundle='mybundle'/>", + " <ssl-truststore-configurator class='com.yahoo.MySslTrustStoreConfigurator' bundle='mybundle'/>", " </server>", " <server port='9001' id='bar'/>", " </http>", @@ -204,27 +207,47 @@ public class JettyContainerModelBuilderTest extends ContainerModelBuilderTestBas createModel(root, clusterElem); ContainerCluster cluster = (ContainerCluster) root.getChildren().get("default"); List<ConnectorFactory> connectorFactories = cluster.getChildrenByTypeRecursive(ConnectorFactory.class); - { ConnectorFactory firstConnector = connectorFactories.get(0); - assertThat(firstConnector.getInjectedComponentIds(), hasItem("ssl-keystore-configurator@foo")); - assertThat(firstConnector.getInjectedComponentIds().size(), equalTo(1)); - SimpleComponent sslKeystoreConfigurator = firstConnector.getChildrenByTypeRecursive(SimpleComponent.class).get(0); - BundleInstantiationSpecification spec = sslKeystoreConfigurator.model.bundleInstantiationSpec; - assertThat(spec.classId.toString(), is("com.yahoo.MySslKeyStoreConfigurator")); - assertThat(spec.bundle.toString(), is("mybundle")); + assertConnectorHasInjectedComponents(firstConnector, "ssl-keystore-configurator@foo", "ssl-truststore-configurator@foo"); + assertComponentHasClassNameAndBundle(getChildComponent(firstConnector, 0), + "com.yahoo.MySslKeyStoreConfigurator", + "mybundle"); + assertComponentHasClassNameAndBundle(getChildComponent(firstConnector, 1), + "com.yahoo.MySslTrustStoreConfigurator", + "mybundle"); } { - ConnectorFactory secondFactory = connectorFactories.get(1); - assertThat(secondFactory.getInjectedComponentIds(), hasItem("ssl-keystore-configurator@bar")); - assertThat(secondFactory.getInjectedComponentIds().size(), equalTo(1)); - SimpleComponent sslKeystoreConfigurator = secondFactory.getChildrenByTypeRecursive(SimpleComponent.class).get(0); - BundleInstantiationSpecification spec = sslKeystoreConfigurator.model.bundleInstantiationSpec; - assertThat(spec.classId.toString(), is(DefaultSslKeyStoreConfigurator.class.getName())); - assertThat(spec.bundle.toString(), is("jdisc_http_service")); + ConnectorFactory secondConnector = connectorFactories.get(1); + assertConnectorHasInjectedComponents(secondConnector, "ssl-keystore-configurator@bar", "ssl-truststore-configurator@bar"); + assertComponentHasClassNameAndBundle(getChildComponent(secondConnector, 0), + DefaultSslKeyStoreConfigurator.class.getName(), + "jdisc_http_service"); + assertComponentHasClassNameAndBundle(getChildComponent(secondConnector, 1), + DefaultSslTrustStoreConfigurator.class.getName(), + "jdisc_http_service"); } } + private static void assertConnectorHasInjectedComponents(ConnectorFactory connectorFactory, String... componentNames) { + Set<String> injectedComponentIds = connectorFactory.getInjectedComponentIds(); + assertThat(injectedComponentIds.size(), equalTo(componentNames.length)); + Arrays.stream(componentNames) + .forEach(name -> assertThat(injectedComponentIds, hasItem(name))); + } + + private static SimpleComponent getChildComponent(ConnectorFactory connectorFactory, int index) { + return connectorFactory.getChildrenByTypeRecursive(SimpleComponent.class).get(index); + } + + private static void assertComponentHasClassNameAndBundle(SimpleComponent simpleComponent, + String className, + String bundleName) { + BundleInstantiationSpecification spec = simpleComponent.model.bundleInstantiationSpec; + assertThat(spec.classId.toString(), is(className)); + assertThat(spec.bundle.toString(), is(bundleName)); + } + private void assertJettyServerInConfig() { ContainerCluster cluster = (ContainerCluster) root.getChildren().get("default"); List<JettyHttpServer> jettyServers = cluster.getChildrenByTypeRecursive(JettyHttpServer.class); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/utils/ContentClusterUtils.java b/config-model/src/test/java/com/yahoo/vespa/model/content/utils/ContentClusterUtils.java index 3a2633ed7b7..e557e3674b5 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/utils/ContentClusterUtils.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/utils/ContentClusterUtils.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.model.content.utils; import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.model.ConfigModelContext; import com.yahoo.config.model.api.HostProvisioner; +import com.yahoo.config.model.application.provider.MockFileRegistry; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.provision.InMemoryProvisioner; import com.yahoo.config.model.provision.SingleNodeProvisioner; @@ -11,9 +12,11 @@ import com.yahoo.config.model.test.MockApplicationPackage; import com.yahoo.config.model.test.MockRoot; import com.yahoo.text.XML; import com.yahoo.vespa.model.admin.Admin; +import com.yahoo.vespa.model.admin.FileDistributionOptions; import com.yahoo.vespa.model.admin.monitoring.DefaultMonitoring; import com.yahoo.vespa.model.admin.monitoring.builder.Metrics; import com.yahoo.vespa.model.content.cluster.ContentCluster; +import com.yahoo.vespa.model.filedistribution.FileDistributionConfigProducer; import org.w3c.dom.Document; import java.util.Collections; @@ -57,7 +60,8 @@ public class ContentClusterUtils { public static ContentCluster createCluster(String clusterXml, MockRoot root) throws Exception { Document doc = XML.getDocument(clusterXml); - Admin admin = new Admin(root, new DefaultMonitoring("vespa", 60), new Metrics(), Collections.emptyMap(), false); + Admin admin = new Admin(root, new DefaultMonitoring("vespa", 60), new Metrics(), Collections.emptyMap(), false, + new FileDistributionConfigProducer.Builder(FileDistributionOptions.defaultOptions()).build(root, new MockFileRegistry())); ConfigModelContext context = ConfigModelContext.create(null, root.getDeployState(), null, root, null); return new ContentCluster.Builder(admin).build(Collections.emptyList(), context, doc.getDocumentElement()); diff --git a/config-model/src/test/schema-test-files/services.xml b/config-model/src/test/schema-test-files/services.xml index a02346193cc..af316c2e3a7 100644 --- a/config-model/src/test/schema-test-files/services.xml +++ b/config-model/src/test/schema-test-files/services.xml @@ -108,6 +108,7 @@ <server port="4080" id="myServer"> <ssl-keystore-configurator class="com.yahoo.MySslKeyStoreConfigurator" bundle="mybundle" /> + <ssl-truststore-configurator class="com.yahoo.MySslTrustStoreConfigurator" bundle="mybundle" /> </server> <server port="4081" id="anotherServer"> <config name="container.jdisc.config.http-server"> |