summaryrefslogtreecommitdiffstats
path: root/config-model
diff options
context:
space:
mode:
authorJon Bratseth <bratseth@yahoo-inc.com>2017-12-14 18:47:08 +0100
committerJon Bratseth <bratseth@yahoo-inc.com>2017-12-14 18:47:08 +0100
commitff0b4082e7d328b5cf5dfe0e79d27471c3afdb50 (patch)
tree865e46af8d4c406418050383a4e25c761b2dce3d /config-model
parent597f59619a0e08f75a0e60ac09da7e5aea538404 (diff)
parentb27bada7c6b22d83557c6592d8c00fea2a6c35fe (diff)
Merge with master
Diffstat (limited to 'config-model')
-rw-r--r--config-model/pom.xml1
-rw-r--r--config-model/src/main/java/com/yahoo/config/model/ConfigModelContext.java5
-rw-r--r--config-model/src/main/java/com/yahoo/config/model/admin/AdminModel.java12
-rw-r--r--config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java19
-rw-r--r--config-model/src/main/java/com/yahoo/config/model/test/MockRoot.java4
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/document/ImmutableImportedSDField.java7
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/processing/ImportedFieldsInSummayValidator.java63
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/processing/Processing.java1
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/AbstractService.java16
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/ConfigProxy.java4
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/VespaModelFactory.java3
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java46
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/admin/Configserver.java1
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/admin/FileDistributionOptions.java10
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/admin/Logserver.java1
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/admin/Slobrok.java1
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/SystemMetrics.java2
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java4
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminBuilderBase.java24
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2Builder.java7
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java6
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomFileDistributionOptionsBuilder.java15
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/clients/VespaSpoolerService.java1
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/container/Container.java3
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/container/configserver/ConfigserverCluster.java3
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/container/configserver/option/CloudConfigOptions.java3
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/container/http/ConnectorFactory.java44
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/JettyConnectorBuilder.java6
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/content/ContentNode.java2
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/content/DistributorCluster.java20
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java18
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/filedistribution/DummyFileDistributionConfigProducer.java42
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributionConfigProducer.java22
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributionConfigProvider.java59
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributor.java6
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributorService.java57
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/search/Dispatch.java1
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java12
-rw-r--r--config-model/src/main/resources/schema/admin.rnc2
-rw-r--r--config-model/src/main/resources/schema/containercluster.rnc1
-rw-r--r--config-model/src/test/cfg/application/invalid_legacy_user_config/configs/qr-searchers.cfg5
-rw-r--r--config-model/src/test/cfg/application/invalid_legacy_user_config/services.xml10
-rw-r--r--config-model/src/test/derived/importedfields/attributes.cfg20
-rw-r--r--config-model/src/test/derived/importedfields/child.sd1
-rw-r--r--config-model/src/test/derived/importedfields/imported-fields.cfg5
-rw-r--r--config-model/src/test/derived/importedfields/index-info.cfg6
-rw-r--r--config-model/src/test/derived/importedfields/parent_a.sd8
-rw-r--r--config-model/src/test/java/com/yahoo/searchdefinition/processing/ImportedFieldsInSummaryValidatorTestCase.java39
-rw-r--r--config-model/src/test/java/com/yahoo/searchdefinition/processing/ImportedFieldsResolverTestCase.java15
-rw-r--r--config-model/src/test/java/com/yahoo/vespa/model/admin/AdminTestCase.java25
-rwxr-xr-xconfig-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2BuilderTest.java5
-rw-r--r--config-model/src/test/java/com/yahoo/vespa/model/clients/test/Gateway20TestCase.java65
-rw-r--r--config-model/src/test/java/com/yahoo/vespa/model/container/configserver/TestOptions.java7
-rw-r--r--config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java38
-rw-r--r--config-model/src/test/java/com/yahoo/vespa/model/container/xml/JettyContainerModelBuilderTest.java55
-rw-r--r--config-model/src/test/java/com/yahoo/vespa/model/content/ContentSearchClusterTest.java34
-rw-r--r--config-model/src/test/java/com/yahoo/vespa/model/content/DistributorTest.java79
-rw-r--r--config-model/src/test/java/com/yahoo/vespa/model/content/utils/ContentClusterBuilder.java27
-rw-r--r--config-model/src/test/java/com/yahoo/vespa/model/content/utils/ContentClusterUtils.java6
-rw-r--r--config-model/src/test/java/com/yahoo/vespa/model/content/utils/DocType.java54
-rw-r--r--config-model/src/test/schema-test-files/services.xml1
61 files changed, 678 insertions, 381 deletions
diff --git a/config-model/pom.xml b/config-model/pom.xml
index 0fdc09e1a61..729d1a2ef9b 100644
--- a/config-model/pom.xml
+++ b/config-model/pom.xml
@@ -6,6 +6,7 @@
<groupId>com.yahoo.vespa</groupId>
<artifactId>parent</artifactId>
<version>6-SNAPSHOT</version>
+ <relativePath>../parent/pom.xml</relativePath>
</parent>
<artifactId>config-model</artifactId>
<packaging>container-plugin</packaging>
diff --git a/config-model/src/main/java/com/yahoo/config/model/ConfigModelContext.java b/config-model/src/main/java/com/yahoo/config/model/ConfigModelContext.java
index 8b3d983d6c1..78a8c161c3b 100644
--- a/config-model/src/main/java/com/yahoo/config/model/ConfigModelContext.java
+++ b/config-model/src/main/java/com/yahoo/config/model/ConfigModelContext.java
@@ -5,17 +5,14 @@ import com.yahoo.config.application.api.ApplicationPackage;
import com.yahoo.config.application.api.DeployLogger;
import com.yahoo.config.model.deploy.DeployState;
import com.yahoo.config.model.producer.AbstractConfigProducer;
-import org.w3c.dom.Element;
-import java.util.Optional;
import java.util.stream.Stream;
/**
* This class contains a context that is passed to a model builder, and can be used to retrieve the application package,
* logger etc.
*
- * @author lulf
- * @since 5.1
+ * @author Ulf Lilleengen
*/
public final class ConfigModelContext {
diff --git a/config-model/src/main/java/com/yahoo/config/model/admin/AdminModel.java b/config-model/src/main/java/com/yahoo/config/model/admin/AdminModel.java
index 32cb1840e7c..5eb4afcc241 100644
--- a/config-model/src/main/java/com/yahoo/config/model/admin/AdminModel.java
+++ b/config-model/src/main/java/com/yahoo/config/model/admin/AdminModel.java
@@ -75,7 +75,11 @@ public class AdminModel extends ConfigModel {
public void doBuild(AdminModel model, Element adminElement, ConfigModelContext modelContext) {
AbstractConfigProducer parent = modelContext.getParentProducer();
DeployProperties properties = modelContext.getDeployState().getProperties();
- DomAdminV2Builder domBuilder = new DomAdminV2Builder(modelContext.getApplicationType(), modelContext.getDeployState().getFileRegistry(), properties.multitenant(), properties.configServerSpecs());
+ DomAdminV2Builder domBuilder = new DomAdminV2Builder(modelContext.getApplicationType(),
+ modelContext.getDeployState().getFileRegistry(),
+ properties.multitenant(),
+ properties.configServerSpecs(),
+ modelContext.getDeployState().disableFiledistributor());
model.admin = domBuilder.build(parent, adminElement);
// TODO: Is required since other models depend on admin.
if (parent instanceof ApplicationConfigProducerRoot) {
@@ -101,7 +105,11 @@ public class AdminModel extends ConfigModel {
public void doBuild(AdminModel model, Element adminElement, ConfigModelContext modelContext) {
AbstractConfigProducer parent = modelContext.getParentProducer();
DeployProperties properties = modelContext.getDeployState().getProperties();
- DomAdminV4Builder domBuilder = new DomAdminV4Builder(modelContext, properties.multitenant(), properties.configServerSpecs(), model.getContainerModels());
+ DomAdminV4Builder domBuilder = new DomAdminV4Builder(modelContext,
+ properties.multitenant(),
+ properties.configServerSpecs(),
+ model.getContainerModels(),
+ modelContext.getDeployState().disableFiledistributor());
model.admin = domBuilder.build(parent, adminElement);
// TODO: Is required since other models depend on admin.
if (parent instanceof ApplicationConfigProducerRoot) {
diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java b/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java
index 25d45dee234..43d3fafdb78 100644
--- a/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java
+++ b/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java
@@ -68,8 +68,8 @@ public class DeployState implements ConfigDefinitionStore {
private final ValidationOverrides validationOverrides;
private final Version wantedNodeVespaVersion;
private final Instant now;
-
private final HostProvisioner provisioner;
+ private final boolean disableFiledistributor;
public static DeployState createTestState() {
return new Builder().build();
@@ -82,8 +82,8 @@ public class DeployState implements ConfigDefinitionStore {
private DeployState(ApplicationPackage applicationPackage, SearchDocumentModel searchDocumentModel, RankProfileRegistry rankProfileRegistry,
FileRegistry fileRegistry, DeployLogger deployLogger, Optional<HostProvisioner> hostProvisioner, DeployProperties properties,
Optional<ApplicationPackage> permanentApplicationPackage, Optional<ConfigDefinitionRepo> configDefinitionRepo,
- java.util.Optional<Model> previousModel, Set<Rotation> rotations, Zone zone, QueryProfiles queryProfiles,
- SemanticRules semanticRules, Instant now, Version wantedNodeVespaVersion) {
+ java.util.Optional<Model> previousModel, Set<Rotation> rotations, Zone zone, QueryProfiles queryProfiles,
+ SemanticRules semanticRules, Instant now, Version wantedNodeVespaVersion, boolean disableFiledistributor) {
this.logger = deployLogger;
this.fileRegistry = fileRegistry;
this.rankProfileRegistry = rankProfileRegistry;
@@ -102,6 +102,7 @@ public class DeployState implements ConfigDefinitionStore {
this.validationOverrides = applicationPackage.getValidationOverrides().map(ValidationOverrides::fromXml).orElse(ValidationOverrides.empty);
this.wantedNodeVespaVersion = wantedNodeVespaVersion;
this.now = now;
+ this.disableFiledistributor = disableFiledistributor;
}
public static HostProvisioner getDefaultModelHostProvisioner(ApplicationPackage applicationPackage) {
@@ -169,6 +170,7 @@ public class DeployState implements ConfigDefinitionStore {
public ApplicationPackage getApplicationPackage() {
return applicationPackage;
}
+
public List<SearchDefinition> getSearchDefinitions() {
return searchDefinitions;
}
@@ -214,6 +216,8 @@ public class DeployState implements ConfigDefinitionStore {
public Instant now() { return now; }
+ public boolean disableFiledistributor() { return disableFiledistributor; }
+
public static class Builder {
private ApplicationPackage applicationPackage = MockApplicationPackage.createEmpty();
@@ -228,6 +232,7 @@ public class DeployState implements ConfigDefinitionStore {
private Zone zone = Zone.defaultZone();
private Instant now = Instant.now();
private Version wantedNodeVespaVersion = Vtag.currentVersion;
+ private boolean disableFiledistributor = false;
public Builder applicationPackage(ApplicationPackage applicationPackage) {
this.applicationPackage = applicationPackage;
@@ -289,13 +294,19 @@ public class DeployState implements ConfigDefinitionStore {
return this;
}
+ public Builder disableFiledistributor(boolean disableFiledistributor) {
+ this.disableFiledistributor = disableFiledistributor;
+ return this;
+ }
+
public DeployState build() {
RankProfileRegistry rankProfileRegistry = new RankProfileRegistry();
QueryProfiles queryProfiles = new QueryProfilesBuilder().build(applicationPackage);
SemanticRules semanticRules = new SemanticRuleBuilder().build(applicationPackage);
SearchDocumentModel searchDocumentModel = createSearchDocumentModel(rankProfileRegistry, logger, queryProfiles);
return new DeployState(applicationPackage, searchDocumentModel, rankProfileRegistry, fileRegistry, logger, hostProvisioner,
- properties, permanentApplicationPackage, configDefinitionRepo, previousModel, rotations, zone, queryProfiles, semanticRules, now, wantedNodeVespaVersion);
+ properties, permanentApplicationPackage, configDefinitionRepo, previousModel, rotations,
+ zone, queryProfiles, semanticRules, now, wantedNodeVespaVersion, disableFiledistributor);
}
private SearchDocumentModel createSearchDocumentModel(RankProfileRegistry rankProfileRegistry, DeployLogger logger, QueryProfiles queryProfiles) {
diff --git a/config-model/src/main/java/com/yahoo/config/model/test/MockRoot.java b/config-model/src/main/java/com/yahoo/config/model/test/MockRoot.java
index 277e02e43e2..2a3ff1ca905 100644
--- a/config-model/src/main/java/com/yahoo/config/model/test/MockRoot.java
+++ b/config-model/src/main/java/com/yahoo/config/model/test/MockRoot.java
@@ -11,6 +11,7 @@ import com.yahoo.config.model.deploy.DeployState;
import com.yahoo.config.model.builder.xml.XmlHelper;
import com.yahoo.config.model.producer.AbstractConfigProducer;
import com.yahoo.config.model.producer.AbstractConfigProducerRoot;
+import com.yahoo.config.provision.Zone;
import com.yahoo.text.XML;
import com.yahoo.vespa.model.ConfigProducer;
import com.yahoo.vespa.model.HostSystem;
@@ -147,7 +148,8 @@ public class MockRoot extends AbstractConfigProducerRoot {
try {
Document doc = XmlHelper.getDocumentBuilder().parse(new InputSource(new StringReader(servicesXml)));
- setAdmin(new DomAdminV2Builder(ConfigModelContext.ApplicationType.DEFAULT, deployState.getFileRegistry(), false, new ArrayList<>()).
+ setAdmin(new DomAdminV2Builder(ConfigModelContext.ApplicationType.DEFAULT, deployState.getFileRegistry(),
+ false, new ArrayList<>(), deployState.disableFiledistributor()).
build(this, XML.getChildren(doc.getDocumentElement(), "admin").get(0)));
} catch (SAXException | IOException e) {
throw new RuntimeException(e);
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/ImmutableImportedSDField.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/ImmutableImportedSDField.java
index a70e77a17a2..c8918f39834 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/document/ImmutableImportedSDField.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/ImmutableImportedSDField.java
@@ -79,7 +79,12 @@ public class ImmutableImportedSDField implements ImmutableSDField {
@Override
public Index getIndex(String name) {
- throw createUnsupportedException();
+ if (!importedField.fieldName().equals(name)) {
+ throw new IllegalArgumentException("Getting an index (" + name + ") with different name than the imported field ("
+ + importedField.fieldName() + ") is not supported");
+ }
+ String targetIndexName = importedField.targetField().getName();
+ return importedField.targetField().getIndex(targetIndexName);
}
@Override
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImportedFieldsInSummayValidator.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImportedFieldsInSummayValidator.java
new file mode 100644
index 00000000000..eaa85815736
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImportedFieldsInSummayValidator.java
@@ -0,0 +1,63 @@
+package com.yahoo.searchdefinition.processing;
+
+import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.document.DataType;
+import com.yahoo.searchdefinition.RankProfileRegistry;
+import com.yahoo.searchdefinition.Search;
+import com.yahoo.searchdefinition.document.ImportedField;
+import com.yahoo.vespa.documentmodel.DocumentSummary;
+import com.yahoo.vespa.documentmodel.SummaryField;
+import com.yahoo.vespa.model.container.search.QueryProfiles;
+
+import java.util.Map;
+
+/**
+ * Validates that imported fields in document summaries are of supported types.
+ * Currently, predicate fields are NOT supported.
+ *
+ * @author geirst
+ */
+public class ImportedFieldsInSummayValidator extends Processor {
+
+
+ public ImportedFieldsInSummayValidator(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) {
+ super(search, deployLogger, rankProfileRegistry, queryProfiles);
+ }
+
+ @Override
+ public void process() {
+ if (search.importedFields().isPresent()) {
+ validateDocumentSummaries(search.getSummaries());
+ }
+ }
+
+ private void validateDocumentSummaries(Map<String, DocumentSummary> summaries) {
+ for (DocumentSummary summary : summaries.values()) {
+ for (SummaryField field : summary.getSummaryFields()) {
+ ImportedField importedField = getImportedField(field);
+ if (importedField != null) {
+ validateImportedSummaryField(summary, field, importedField);
+ }
+ }
+ }
+ }
+
+ private ImportedField getImportedField(SummaryField field) {
+ return search.importedFields().get().fields().get(field.getName());
+ }
+
+ private void validateImportedSummaryField(DocumentSummary summary, SummaryField field, ImportedField importedField) {
+ if (field.getDataType().equals(DataType.PREDICATE) &&
+ importedField.targetField().getDataType().equals(DataType.PREDICATE)) {
+ fail(summary, field, "Is of type predicate. Not supported in document summaries");
+ }
+ }
+
+ private void fail(DocumentSummary summary, SummaryField importedField, String msg) {
+ throw new IllegalArgumentException("For search '" + search.getName() + "', document summary '" + summary.getName() +
+ "', imported summary field '" + importedField.getName() + "': " + msg);
+ }
+}
+
+
+
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processing.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processing.java
index 26f98026d4f..90183848094 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processing.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processing.java
@@ -73,6 +73,7 @@ public class Processing {
TensorFieldProcessor::new,
RankProfileTypeSettingsProcessor::new,
ReferenceFieldsProcessor::new,
+ ImportedFieldsInSummayValidator::new,
FastAccessValidator::new,
ReservedMacroNames::new,
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/AbstractService.java b/config-model/src/main/java/com/yahoo/vespa/model/AbstractService.java
index dfa2d2cefb0..3b75be5167d 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/AbstractService.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/AbstractService.java
@@ -510,22 +510,6 @@ public abstract class AbstractService extends AbstractConfigProducer<AbstractCon
}
/**
- * Sets up this service to be included when generating monitoring config.
- * The ymon service name used will be {@link #getServiceType()}
- */
- public void monitorService() {
- monitorService(getServiceType());
- }
-
- /**
- * Sets up this service to be included when generating ymon config.
- * @param ymonServiceName the ymon service name to be used
- */
- public void monitorService(String ymonServiceName) {
- setProp("ymonService", ymonServiceName);
- }
-
- /**
*
* The service HTTP port for health status
* @return portnumber
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/ConfigProxy.java b/config-model/src/main/java/com/yahoo/vespa/model/ConfigProxy.java
index d751e332ff3..c540a5f62d2 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/ConfigProxy.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/ConfigProxy.java
@@ -16,6 +16,8 @@ package com.yahoo.vespa.model;
*/
public class ConfigProxy extends AbstractService {
+ public final static int BASEPORT = 19090;
+
/**
* Creates a new ConfigProxy instance.
*
@@ -31,7 +33,7 @@ public class ConfigProxy extends AbstractService {
/**
* Returns the desired base port for this service.
*/
- public int getWantedPort() { return 19090; }
+ public int getWantedPort() { return BASEPORT; }
/**
* The desired base port is the only allowed base port.
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/VespaModelFactory.java b/config-model/src/main/java/com/yahoo/vespa/model/VespaModelFactory.java
index 74512e70ebe..7a1fab8dbd0 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/VespaModelFactory.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/VespaModelFactory.java
@@ -138,7 +138,8 @@ public class VespaModelFactory implements ModelFactory {
.rotations(modelContext.properties().rotations())
.zone(zone)
.now(clock.instant())
- .wantedNodeVespaVersion(modelContext.wantedNodeVespaVersion());
+ .wantedNodeVespaVersion(modelContext.wantedNodeVespaVersion())
+ .disableFiledistributor(modelContext.properties().disableFileDistributor());
modelContext.previousModel().ifPresent(builder::previousModel);
return builder.build();
}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java
index 60d00ce5487..59b7388f5bb 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java
@@ -1,20 +1,26 @@
// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.model.admin;
-import com.yahoo.cloud.config.log.LogdConfig;
+import com.yahoo.cloud.config.SlobroksConfig;
import com.yahoo.cloud.config.ZookeepersConfig;
+import com.yahoo.cloud.config.log.LogdConfig;
import com.yahoo.config.model.api.ConfigServerSpec;
import com.yahoo.config.model.deploy.DeployProperties;
import com.yahoo.config.model.producer.AbstractConfigProducer;
-import com.yahoo.cloud.config.SlobroksConfig;
import com.yahoo.config.provision.ApplicationId;
import com.yahoo.config.provision.Zone;
-import com.yahoo.vespa.model.*;
+import com.yahoo.vespa.model.AbstractService;
+import com.yahoo.vespa.model.ConfigProxy;
+import com.yahoo.vespa.model.ConfigSentinel;
+import com.yahoo.vespa.model.HostResource;
+import com.yahoo.vespa.model.Logd;
import com.yahoo.vespa.model.admin.monitoring.MetricsConsumer;
import com.yahoo.vespa.model.admin.monitoring.Monitoring;
import com.yahoo.vespa.model.admin.monitoring.builder.Metrics;
import com.yahoo.vespa.model.container.ContainerCluster;
+import com.yahoo.vespa.model.filedistribution.DummyFileDistributionConfigProducer;
import com.yahoo.vespa.model.filedistribution.FileDistributionConfigProducer;
+import com.yahoo.vespa.model.filedistribution.FileDistributionConfigProvider;
import com.yahoo.vespa.model.filedistribution.FileDistributor;
import com.yahoo.vespa.model.filedistribution.FileDistributorService;
@@ -24,8 +30,6 @@ import java.util.Collections;
import java.util.List;
import java.util.Map;
-import com.yahoo.vespa.model.HostResource;
-
/**
* This is the admin pseudo-plugin of the Vespa model, responsible for
* creating all admin services.
@@ -59,12 +63,14 @@ public class Admin extends AbstractConfigProducer implements Serializable {
Monitoring monitoring,
Metrics metrics,
Map<String, MetricsConsumer> legacyMetricsConsumers,
- boolean multitenant) {
+ boolean multitenant,
+ FileDistributionConfigProducer fileDistributionConfigProducer) {
super(parent, "admin");
this.monitoring = monitoring;
this.metrics = metrics;
this.legacyMetricsConsumers = legacyMetricsConsumers;
this.multitenant = multitenant;
+ this.fileDistribution = fileDistributionConfigProducer;
}
public Configserver getConfigserver() {
@@ -148,10 +154,6 @@ public class Admin extends AbstractConfigProducer implements Serializable {
zooKeepersConfigProvider.getConfig(builder);
}
- public void setFileDistribution(FileDistributionConfigProducer fileDistribution) {
- this.fileDistribution = fileDistribution;
- }
-
public FileDistributionConfigProducer getFileDistributionConfigProducer() {
return fileDistribution;
}
@@ -215,11 +217,25 @@ public class Admin extends AbstractConfigProducer implements Serializable {
fileDistributor.fileSourceHost() + "'. Hostsystem=" + getHostSystem());
}
- FileDistributorService fds = new FileDistributorService(fileDistribution, host.getHost().getHostname(),
- fileDistribution.getFileDistributor(), fileDistribution.getOptions(), host == deployHost);
- fds.setHostResource(host);
- fds.initService();
- fileDistribution.addFileDistributionService(host.getHost(), fds);
+ FileDistributionConfigProvider configProvider =
+ new FileDistributionConfigProvider(fileDistributor,
+ fileDistribution.getOptions(),
+ host == deployHost,
+ host.getHost());
+ if (fileDistribution.getOptions().disableFiledistributor()) {
+ DummyFileDistributionConfigProducer dummyFileDistributionConfigProducer =
+ new DummyFileDistributionConfigProducer(fileDistribution,
+ host.getHost().getHostname(),
+ configProvider);
+ fileDistribution.addFileDistributionConfigProducer(host.getHost(), dummyFileDistributionConfigProducer);
+ } else {
+ FileDistributorService fds = new FileDistributorService(fileDistribution,
+ host.getHost().getHostname(),
+ configProvider);
+ fds.setHostResource(host);
+ fds.initService();
+ fileDistribution.addFileDistributionConfigProducer(host.getHost(), fds);
+ }
}
private boolean deployHostIsMissing(HostResource deployHost) {
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/Configserver.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/Configserver.java
index 942d563d850..e8f1f59310b 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/admin/Configserver.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/Configserver.java
@@ -35,7 +35,6 @@ public class Configserver extends AbstractService {
portsMeta.on(1).tag("http").tag("config").tag("state");
setProp("clustertype", "admin");
setProp("clustername", "admin");
- monitorService();
}
/**
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/FileDistributionOptions.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/FileDistributionOptions.java
index 0160978773d..8b39a1ecb94 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/admin/FileDistributionOptions.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/FileDistributionOptions.java
@@ -21,7 +21,7 @@ public class FileDistributionOptions implements FiledistributorConfig.Producer {
private BinaryScaledAmount uploadBitRate = new BinaryScaledAmount();
private BinaryScaledAmount downloadBitRate = new BinaryScaledAmount();
- private boolean disabled = false;
+ private boolean disableFiledistributor = false;
public void downloadBitRate(BinaryScaledAmount amount) {
@@ -34,12 +34,12 @@ public class FileDistributionOptions implements FiledistributorConfig.Producer {
uploadBitRate = amount;
}
- public void disabled(boolean value) {
- disabled = value;
+ public void disableFiledistributor(boolean value) {
+ disableFiledistributor = value;
}
- public boolean disabled() {
- return disabled;
+ public boolean disableFiledistributor() {
+ return disableFiledistributor;
}
private void ensureNonNegative(BinaryScaledAmount amount) {
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/Logserver.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/Logserver.java
index 28cf18802cf..8b98dc9d06a 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/admin/Logserver.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/Logserver.java
@@ -23,7 +23,6 @@ public class Logserver extends AbstractService {
portsMeta.on(3).tag("logtp").tag("telnet").tag("replicator");
setProp("clustertype", "admin");
setProp("clustername", "admin");
- monitorService();
}
/**
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/Slobrok.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/Slobrok.java
index cefc08981a4..093694f41b3 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/admin/Slobrok.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/Slobrok.java
@@ -23,7 +23,6 @@ public class Slobrok extends AbstractService {
setProp("index", index);
setProp("clustertype", "slobrok");
setProp("clustername", "admin");
- monitorService();
}
@Override
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/SystemMetrics.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/SystemMetrics.java
index cafdf83608f..3db09f6b566 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/SystemMetrics.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/SystemMetrics.java
@@ -11,6 +11,7 @@ import java.util.Set;
@SuppressWarnings("UnusedDeclaration") // Used by model amenders
public class SystemMetrics {
public static final String CPU_UTIL = "cpu.util";
+ public static final String CPU_SYS_UTIL = "cpu.sys.util";
public static final String DISK_LIMIT = "disk.limit";
public static final String DISK_USED = "disk.used";
public static final String DISK_UTIL = "disk.util";
@@ -23,6 +24,7 @@ public class SystemMetrics {
private static MetricSet createSystemMetricSet() {
Set<Metric> dockerNodeMetrics =
ImmutableSet.of(new Metric(CPU_UTIL),
+ new Metric(CPU_SYS_UTIL),
new Metric(DISK_LIMIT),
new Metric(DISK_USED),
new Metric(DISK_UTIL),
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java
index 2dcf874a66e..2243dc1e682 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java
@@ -52,6 +52,7 @@ public class VespaMetricSet {
private static Set<Metric> getOtherMetrics() {
Set<Metric> metrics = new LinkedHashSet<>();
metrics.add(new Metric("slobrok.heartbeats.failed.count", "slobrok.heartbeats.failed"));
+ metrics.add(new Metric("logd.processed.lines.count", "logd.processed.lines"));
return metrics;
}
@@ -220,7 +221,9 @@ public class VespaMetricSet {
// resource usage
metrics.add(new Metric("content.proton.resource_usage.disk.average"));
+ metrics.add(new Metric("content.proton.resource_usage.disk_utilization.average"));
metrics.add(new Metric("content.proton.resource_usage.memory.average"));
+ metrics.add(new Metric("content.proton.resource_usage.memory_utilization.average"));
metrics.add(new Metric("content.proton.resource_usage.memory_mappings.max"));
metrics.add(new Metric("content.proton.resource_usage.open_file_descriptors.max"));
metrics.add(new Metric("content.proton.documentdb.attribute.resource_usage.enum_store.average"));
@@ -230,6 +233,7 @@ public class VespaMetricSet {
// transaction log
metrics.add(new Metric("content.proton.transactionlog.entries.average"));
metrics.add(new Metric("content.proton.transactionlog.disk_usage.average"));
+ metrics.add(new Metric("content.proton.transactionlog.replay_time.last"));
// document store
metrics.add(new Metric("content.proton.documentdb.ready.document_store.disk_usage.average"));
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminBuilderBase.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminBuilderBase.java
index a3a1a0fbebc..24e56472688 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminBuilderBase.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminBuilderBase.java
@@ -37,16 +37,19 @@ public abstract class DomAdminBuilderBase extends VespaDomBuilder.DomConfigProdu
private final ApplicationType applicationType;
private final List<ConfigServerSpec> configServerSpecs;
private final FileRegistry fileRegistry;
+ private final boolean disableFiledistributor;
protected final boolean multitenant;
- public DomAdminBuilderBase(ApplicationType applicationType, FileRegistry fileRegistry, boolean multitenant, List<ConfigServerSpec> configServerSpecs) {
+ DomAdminBuilderBase(ApplicationType applicationType, FileRegistry fileRegistry, boolean multitenant,
+ List<ConfigServerSpec> configServerSpecs, boolean disableFiledistributor) {
this.applicationType = applicationType;
this.fileRegistry = fileRegistry;
this.multitenant = multitenant;
this.configServerSpecs = configServerSpecs;
+ this.disableFiledistributor = disableFiledistributor;
}
- protected List<Configserver> getConfigServersFromSpec(AbstractConfigProducer parent) {
+ List<Configserver> getConfigServersFromSpec(AbstractConfigProducer parent) {
List<Configserver> configservers = new ArrayList<>();
for (ConfigServerSpec spec : configServerSpecs) {
HostSystem hostSystem = parent.getHostSystem();
@@ -64,23 +67,26 @@ public abstract class DomAdminBuilderBase extends VespaDomBuilder.DomConfigProdu
@Override
protected Admin doBuild(AbstractConfigProducer parent, Element adminElement) {
Monitoring monitoring = getMonitoring(getChildWithFallback(adminElement, "monitoring", "yamas"));
-
Metrics metrics = new MetricsBuilder(applicationType, predefinedMetricSets)
.buildMetrics(XML.getChild(adminElement, "metrics"));
Map<String, MetricsConsumer> legacyMetricsConsumers = DomMetricBuilderHelper
.buildMetricsConsumers(XML.getChild(adminElement, "metric-consumers"));
+ FileDistributionConfigProducer fileDistributionConfigProducer = getFileDistributionConfigProducer(parent, adminElement);
- Admin admin = new Admin(parent, monitoring, metrics, legacyMetricsConsumers, multitenant);
-
+ Admin admin = new Admin(parent, monitoring, metrics, legacyMetricsConsumers, multitenant, fileDistributionConfigProducer);
doBuildAdmin(admin, adminElement);
-
new ModelConfigProvider(admin);
- FileDistributionOptions fileDistributionOptions = new DomFileDistributionOptionsBuilder().build(XML.getChild(adminElement, "filedistribution"));
- admin.setFileDistribution(new FileDistributionConfigProducer.Builder(fileDistributionOptions).build(parent, fileRegistry));
return admin;
}
-
+
+ private FileDistributionConfigProducer getFileDistributionConfigProducer(AbstractConfigProducer parent, Element adminElement) {
+ FileDistributionOptions fileDistributionOptions = FileDistributionOptions.defaultOptions();
+ fileDistributionOptions.disableFiledistributor(disableFiledistributor);
+ fileDistributionOptions = new DomFileDistributionOptionsBuilder(fileDistributionOptions).build(XML.getChild(adminElement, "filedistribution"));
+ return new FileDistributionConfigProducer.Builder(fileDistributionOptions).build(parent, fileRegistry);
+ }
+
private Element getChildWithFallback(Element parent, String childName, String alternativeChildName) {
Element child = XML.getChild(parent, childName);
if (child != null) return child;
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2Builder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2Builder.java
index dd1d4e36255..d966f3b49f6 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2Builder.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2Builder.java
@@ -1,6 +1,7 @@
// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.model.builder.xml.dom;
+import com.yahoo.config.application.api.FileRegistry;
import com.yahoo.config.model.ConfigModelContext;
import com.yahoo.config.model.api.ConfigServerSpec;
import com.yahoo.config.model.producer.AbstractConfigProducer;
@@ -18,7 +19,6 @@ import com.yahoo.vespa.model.builder.xml.dom.VespaDomBuilder.DomConfigProducerBu
import com.yahoo.vespa.model.container.Container;
import com.yahoo.vespa.model.container.ContainerCluster;
import com.yahoo.vespa.model.container.xml.ContainerModelBuilder;
-import com.yahoo.config.application.api.FileRegistry;
import org.w3c.dom.Element;
import java.util.List;
@@ -37,8 +37,9 @@ public class DomAdminV2Builder extends DomAdminBuilderBase {
public DomAdminV2Builder(ConfigModelContext.ApplicationType applicationType,
FileRegistry fileRegistry,
boolean multitenant,
- List<ConfigServerSpec> configServerSpecs) {
- super(applicationType, fileRegistry, multitenant, configServerSpecs);
+ List<ConfigServerSpec> configServerSpecs,
+ boolean disableFiledistributor) {
+ super(applicationType, fileRegistry, multitenant, configServerSpecs, disableFiledistributor);
}
@Override
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java
index f33c86134cb..1f7c1ba676c 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java
@@ -31,8 +31,10 @@ public class DomAdminV4Builder extends DomAdminBuilderBase {
private final Collection<ContainerModel> containerModels;
private final ConfigModelContext context;
- public DomAdminV4Builder(ConfigModelContext context, boolean multitenant, List<ConfigServerSpec> configServerSpecs, Collection<ContainerModel> containerModels) {
- super(context.getApplicationType(), context.getDeployState().getFileRegistry(), multitenant, configServerSpecs);
+ public DomAdminV4Builder(ConfigModelContext context, boolean multitenant, List<ConfigServerSpec> configServerSpecs,
+ Collection<ContainerModel> containerModels, boolean disableFiledistributor) {
+ super(context.getApplicationType(), context.getDeployState().getFileRegistry(), multitenant,
+ configServerSpecs, disableFiledistributor);
this.containerModels = containerModels;
this.context = context;
}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomFileDistributionOptionsBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomFileDistributionOptionsBuilder.java
index 8a5d6846a64..9cbdd00899b 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomFileDistributionOptionsBuilder.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomFileDistributionOptionsBuilder.java
@@ -15,6 +15,11 @@ import java.util.Optional;
* @author hmusum
*/
public class DomFileDistributionOptionsBuilder {
+ private final FileDistributionOptions fileDistributionOptions;
+
+ public DomFileDistributionOptionsBuilder(FileDistributionOptions fileDistributionOptions) {
+ this.fileDistributionOptions = fileDistributionOptions;
+ }
private static void throwExceptionForElementInFileDistribution(String subElement, String reason) {
throw new RuntimeException("In element '" + subElement + "' contained in 'filedistribution': " + reason);
@@ -34,15 +39,15 @@ public class DomFileDistributionOptionsBuilder {
}
public FileDistributionOptions build(Element fileDistributionElement) {
- FileDistributionOptions options = FileDistributionOptions.defaultOptions();
if (fileDistributionElement != null) {
- getAmount("uploadbitrate", fileDistributionElement).ifPresent(options::uploadBitRate);
- getAmount("downloadbitrate", fileDistributionElement).ifPresent(options::downloadBitRate);
+ getAmount("uploadbitrate", fileDistributionElement).ifPresent(fileDistributionOptions::uploadBitRate);
+ getAmount("downloadbitrate", fileDistributionElement).ifPresent(fileDistributionOptions::downloadBitRate);
Element disable = XML.getChild(fileDistributionElement, "disabled");
+ if (disable == null) disable = XML.getChild(fileDistributionElement, "disableFiledistributor");
if (disable != null) {
- options.disabled(Boolean.valueOf(XML.getValue(disable)));
+ fileDistributionOptions.disableFiledistributor(Boolean.valueOf(XML.getValue(disable)));
}
}
- return options;
+ return fileDistributionOptions;
}
}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/clients/VespaSpoolerService.java b/config-model/src/main/java/com/yahoo/vespa/model/clients/VespaSpoolerService.java
index 4969132ccfb..f4287cf982e 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/clients/VespaSpoolerService.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/clients/VespaSpoolerService.java
@@ -19,7 +19,6 @@ public class VespaSpoolerService extends AbstractService implements SpoolerConfi
public VespaSpoolerService(AbstractConfigProducer parent, int index, VespaSpooler spooler) {
super(parent, "spooler." + index);
this.spooler = spooler;
- monitorService("spooler");
}
public int getPortCount() {
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java b/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java
index 8991bfa6215..5925ec978bb 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java
@@ -165,7 +165,6 @@ public class Container extends AbstractService implements
}
tagServers();
- monitorService();
}
private void tagServers() {
@@ -331,7 +330,7 @@ public class Container extends AbstractService implements
FileDistributionConfigProducer fileDistribution = getRoot().getFileDistributionConfigProducer();
if (fileDistribution != null) {
- builder.configid(fileDistribution.getFileDistributorService(getHost()).getConfigId());
+ builder.configid(fileDistribution.getConfigProducer(getHost()).getConfigId());
}
return builder;
}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/ConfigserverCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/ConfigserverCluster.java
index 62828b314d0..bce78017bdd 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/ConfigserverCluster.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/ConfigserverCluster.java
@@ -144,6 +144,9 @@ public class ConfigserverCluster extends AbstractConfigProducer
if (options.loadBalancerAddress().isPresent()) {
builder.loadBalancerAddress(options.loadBalancerAddress().get());
}
+ if (options.disableFiledistributor().isPresent()) {
+ builder.disableFiledistributor(options.disableFiledistributor().get());
+ }
}
private String[] getConfigModelPluginDirs() {
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/option/CloudConfigOptions.java b/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/option/CloudConfigOptions.java
index aeb86ae9d59..866bae6666a 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/option/CloudConfigOptions.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/option/CloudConfigOptions.java
@@ -4,7 +4,7 @@ package com.yahoo.vespa.model.container.configserver.option;
import java.util.Optional;
/**
- * @author tonytv
+ * @author Tony Vaagenes
*/
public interface CloudConfigOptions {
@@ -44,4 +44,5 @@ public interface CloudConfigOptions {
Optional<String> dockerRegistry();
Optional<String> dockerVespaBaseImage();
Optional<String> loadBalancerAddress();
+ Optional<Boolean> disableFiledistributor();
}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/ConnectorFactory.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/ConnectorFactory.java
index 22c42056b3f..33f5edded3c 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/container/http/ConnectorFactory.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/ConnectorFactory.java
@@ -5,6 +5,7 @@ import com.yahoo.component.ComponentId;
import com.yahoo.container.bundle.BundleInstantiationSpecification;
import com.yahoo.jdisc.http.ConnectorConfig;
import com.yahoo.jdisc.http.ssl.DefaultSslKeyStoreConfigurator;
+import com.yahoo.jdisc.http.ssl.DefaultSslTrustStoreConfigurator;
import com.yahoo.osgi.provider.model.ComponentModel;
import com.yahoo.text.XML;
import com.yahoo.vespa.model.container.component.SimpleComponent;
@@ -24,10 +25,14 @@ public class ConnectorFactory extends SimpleComponent implements ConnectorConfig
private final Element legacyConfig;
public ConnectorFactory(String name, int listenPort) {
- this(name, listenPort, null, null);
+ this(name, listenPort, null, null, null);
}
- public ConnectorFactory(String name, int listenPort, Element legacyConfig, Element sslKeystoreConfigurator) {
+ public ConnectorFactory(String name,
+ int listenPort,
+ Element legacyConfig,
+ Element sslKeystoreConfigurator,
+ Element sslTruststoreConfigurator) {
super(new ComponentModel(
new BundleInstantiationSpecification(new ComponentId(name),
fromString("com.yahoo.jdisc.http.server.jetty.ConnectorFactory"),
@@ -35,9 +40,8 @@ public class ConnectorFactory extends SimpleComponent implements ConnectorConfig
this.name = name;
this.listenPort = listenPort;
this.legacyConfig = legacyConfig;
- SimpleComponent sslKeyStoreConfigurator = getSslKeyStoreConfigurator(name, sslKeystoreConfigurator);
- addChild(sslKeyStoreConfigurator);
- inject(sslKeyStoreConfigurator);
+ addSslKeyStoreConfigurator(name, sslKeystoreConfigurator);
+ addSslTrustStoreConfigurator(name, sslTruststoreConfigurator);
}
@Override
@@ -149,16 +153,30 @@ public class ConnectorFactory extends SimpleComponent implements ConnectorConfig
}
}
- private static SimpleComponent getSslKeyStoreConfigurator(String name, Element sslKeystoreConfigurator) {
- String idSpec = "ssl-keystore-configurator@" + name;
- if (sslKeystoreConfigurator != null) {
- String className = sslKeystoreConfigurator.getAttribute("class");
- String bundleName = sslKeystoreConfigurator.getAttribute("bundle");
- return new SimpleComponent(new ComponentModel(idSpec, className, bundleName));
+ private void addSslKeyStoreConfigurator(String name, Element sslKeystoreConfigurator) {
+ addSslConfigurator("ssl-keystore-configurator@" + name,
+ DefaultSslKeyStoreConfigurator.class,
+ sslKeystoreConfigurator);
+ }
+
+ private void addSslTrustStoreConfigurator(String name, Element sslKeystoreConfigurator) {
+ addSslConfigurator("ssl-truststore-configurator@" + name,
+ DefaultSslTrustStoreConfigurator.class,
+ sslKeystoreConfigurator);
+ }
+
+ private void addSslConfigurator(String idSpec, Class<?> defaultImplementation, Element configuratorElement) {
+ SimpleComponent configuratorComponent;
+ if (configuratorElement != null) {
+ String className = configuratorElement.getAttribute("class");
+ String bundleName = configuratorElement.getAttribute("bundle");
+ configuratorComponent = new SimpleComponent(new ComponentModel(idSpec, className, bundleName));
} else {
- return new SimpleComponent(
- new ComponentModel(idSpec, DefaultSslKeyStoreConfigurator.class.getName(), "jdisc_http_service"));
+ configuratorComponent =
+ new SimpleComponent(new ComponentModel(idSpec, defaultImplementation.getName(), "jdisc_http_service"));
}
+ addChild(configuratorComponent);
+ inject(configuratorComponent);
}
}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/JettyConnectorBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/JettyConnectorBuilder.java
index f2012a609a7..f88c091cd37 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/JettyConnectorBuilder.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/JettyConnectorBuilder.java
@@ -12,8 +12,7 @@ import java.util.logging.Level;
import java.util.logging.Logger;
/**
- * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
- * @since 5.21.0
+ * @author Einar M R Rosenvinge
*/
public class JettyConnectorBuilder extends VespaDomBuilder.DomConfigProducerBuilder<ConnectorFactory> {
private static final Logger log = Logger.getLogger(JettyConnectorBuilder.class.getName());
@@ -34,7 +33,8 @@ public class JettyConnectorBuilder extends VespaDomBuilder.DomConfigProducerBuil
}
}
Element sslKeystoreConfigurator = XML.getChild(serverSpec, "ssl-keystore-configurator");
- return new ConnectorFactory(name, port, legacyServerConfig, sslKeystoreConfigurator);
+ Element sslTruststoreConfigurator = XML.getChild(serverSpec, "ssl-truststore-configurator");
+ return new ConnectorFactory(name, port, legacyServerConfig, sslKeystoreConfigurator, sslTruststoreConfigurator);
}
}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentNode.java b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentNode.java
index 7ef54360f00..4257e47218e 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentNode.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentNode.java
@@ -50,8 +50,6 @@ public abstract class ContentNode extends AbstractService
portsMeta.on(0).tag("messaging");
portsMeta.on(1).tag("rpc").tag("status");
portsMeta.on(2).tag("http").tag("status").tag("state");
-
- monitorService();
}
@Override
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/DistributorCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/DistributorCluster.java
index 81aca977400..0405f96cd89 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/content/DistributorCluster.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/content/DistributorCluster.java
@@ -1,8 +1,6 @@
// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.model.content;
-import com.yahoo.documentmodel.NewDocumentType;
-import com.yahoo.vespa.config.content.core.BucketspacesConfig;
import com.yahoo.vespa.config.content.core.StorDistributormanagerConfig;
import com.yahoo.vespa.config.content.core.StorServerConfig;
import com.yahoo.document.select.DocumentSelector;
@@ -16,15 +14,13 @@ import org.w3c.dom.Element;
import java.util.logging.Logger;
-
/**
* Generates distributor-specific configuration.
*/
public class DistributorCluster extends AbstractConfigProducer<Distributor> implements
StorDistributormanagerConfig.Producer,
StorServerConfig.Producer,
- MetricsmanagerConfig.Producer,
- BucketspacesConfig.Producer {
+ MetricsmanagerConfig.Producer {
public static final Logger log = Logger.getLogger(DistributorCluster.class.getPackage().toString());
@@ -151,20 +147,6 @@ public class DistributorCluster extends AbstractConfigProducer<Distributor> impl
builder.is_distributor(true);
}
- private static final String DEFAULT_BUCKET_SPACE = "default";
- private static final String GLOBAL_BUCKET_SPACE = "global";
-
- @Override
- public void getConfig(BucketspacesConfig.Builder builder) {
- for (NewDocumentType docType : parent.getDocumentDefinitions().values()) {
- BucketspacesConfig.Documenttype.Builder docTypeBuilder = new BucketspacesConfig.Documenttype.Builder();
- docTypeBuilder.name(docType.getName());
- String bucketSpace = (parent.isGloballyDistributed(docType) ? GLOBAL_BUCKET_SPACE : DEFAULT_BUCKET_SPACE);
- docTypeBuilder.bucketspace(bucketSpace);
- builder.documenttype(docTypeBuilder);
- }
- }
-
public String getClusterName() {
return parent.getName();
}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java
index fc267133c9f..db849d0ae50 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java
@@ -11,6 +11,7 @@ import com.yahoo.config.provision.Zone;
import com.yahoo.vespa.config.content.MessagetyperouteselectorpolicyConfig;
import com.yahoo.vespa.config.content.FleetcontrollerConfig;
import com.yahoo.vespa.config.content.StorDistributionConfig;
+import com.yahoo.vespa.config.content.core.BucketspacesConfig;
import com.yahoo.vespa.config.content.core.StorDistributormanagerConfig;
import com.yahoo.documentmodel.NewDocumentType;
import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol;
@@ -57,7 +58,8 @@ public class ContentCluster extends AbstractConfigProducer implements StorDistri
StorDistributormanagerConfig.Producer,
FleetcontrollerConfig.Producer,
MetricsmanagerConfig.Producer,
- MessagetyperouteselectorpolicyConfig.Producer {
+ MessagetyperouteselectorpolicyConfig.Producer,
+ BucketspacesConfig.Producer {
// TODO: Make private
private String documentSelection;
@@ -698,4 +700,18 @@ public class ContentCluster extends AbstractConfigProducer implements StorDistri
}
}
+
+ private static final String DEFAULT_BUCKET_SPACE = "default";
+ private static final String GLOBAL_BUCKET_SPACE = "global";
+
+ @Override
+ public void getConfig(BucketspacesConfig.Builder builder) {
+ for (NewDocumentType docType : getDocumentDefinitions().values()) {
+ BucketspacesConfig.Documenttype.Builder docTypeBuilder = new BucketspacesConfig.Documenttype.Builder();
+ docTypeBuilder.name(docType.getName());
+ String bucketSpace = (isGloballyDistributed(docType) ? GLOBAL_BUCKET_SPACE : DEFAULT_BUCKET_SPACE);
+ docTypeBuilder.bucketspace(bucketSpace);
+ builder.documenttype(docTypeBuilder);
+ }
+ }
}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/DummyFileDistributionConfigProducer.java b/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/DummyFileDistributionConfigProducer.java
new file mode 100644
index 00000000000..1b1e66594d1
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/DummyFileDistributionConfigProducer.java
@@ -0,0 +1,42 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.filedistribution;
+
+import com.yahoo.cloud.config.filedistribution.FiledistributorConfig;
+import com.yahoo.cloud.config.filedistribution.FiledistributorrpcConfig;
+import com.yahoo.cloud.config.filedistribution.FilereferencesConfig;
+import com.yahoo.config.model.producer.AbstractConfigProducer;
+
+/**
+ * @author hmusum
+ * <p>
+ * Dummy file distribution config producer, needed for serving file distribution config when there is no FiledistributorService.
+ */
+public class DummyFileDistributionConfigProducer extends AbstractConfigProducer implements
+ FiledistributorConfig.Producer,
+ FiledistributorrpcConfig.Producer,
+ FilereferencesConfig.Producer {
+
+ private final FileDistributionConfigProvider configProvider;
+
+ public DummyFileDistributionConfigProducer(AbstractConfigProducer parent,
+ String hostname,
+ FileDistributionConfigProvider configProvider) {
+ super(parent, hostname);
+ this.configProvider = configProvider;
+ }
+
+ @Override
+ public void getConfig(FiledistributorConfig.Builder builder) {
+ configProvider.getConfig(builder);
+ }
+
+ @Override
+ public void getConfig(FiledistributorrpcConfig.Builder builder) {
+ configProvider.getConfig(builder);
+ }
+
+ @Override
+ public void getConfig(FilereferencesConfig.Builder builder) {
+ configProvider.getConfig(builder);
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributionConfigProducer.java b/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributionConfigProducer.java
index 9e5e3074c5e..c863b180b21 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributionConfigProducer.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributionConfigProducer.java
@@ -10,11 +10,13 @@ import java.util.IdentityHashMap;
import java.util.Map;
/**
- * @author tonytv
+ * @author hmusum
+ * <p>
+ * File distribution config producer, delegates getting config to {@link DummyFileDistributionConfigProducer} (one per host)
*/
public class FileDistributionConfigProducer extends AbstractConfigProducer {
- private final Map<Host, FileDistributorService> fileDistributorServices = new IdentityHashMap<>();
+ private final Map<Host, AbstractConfigProducer> fileDistributionConfigProducers = new IdentityHashMap<>();
private final FileDistributor fileDistributor;
private final FileDistributionOptions options;
@@ -24,14 +26,6 @@ public class FileDistributionConfigProducer extends AbstractConfigProducer {
this.options = options;
}
- public FileDistributorService getFileDistributorService(Host host) {
- FileDistributorService service = fileDistributorServices.get(host);
- if (service == null) {
- throw new IllegalStateException("No file distribution service for host " + host);
- }
- return service;
- }
-
public FileDistributor getFileDistributor() {
return fileDistributor;
}
@@ -40,8 +34,8 @@ public class FileDistributionConfigProducer extends AbstractConfigProducer {
return options;
}
- public void addFileDistributionService(Host host, FileDistributorService fds) {
- fileDistributorServices.put(host, fds);
+ public void addFileDistributionConfigProducer(Host host, AbstractConfigProducer fileDistributionConfigProducer) {
+ fileDistributionConfigProducers.put(host, fileDistributionConfigProducer);
}
public static class Builder {
@@ -58,4 +52,8 @@ public class FileDistributionConfigProducer extends AbstractConfigProducer {
}
}
+ public AbstractConfigProducer getConfigProducer(Host host) {
+ return fileDistributionConfigProducers.get(host);
+ }
+
}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributionConfigProvider.java b/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributionConfigProvider.java
new file mode 100644
index 00000000000..a78eb1f74f9
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributionConfigProvider.java
@@ -0,0 +1,59 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.filedistribution;
+
+import com.yahoo.cloud.config.filedistribution.FiledistributorConfig;
+import com.yahoo.cloud.config.filedistribution.FiledistributorrpcConfig;
+import com.yahoo.cloud.config.filedistribution.FilereferencesConfig;
+import com.yahoo.config.FileReference;
+import com.yahoo.config.model.api.FileDistribution;
+import com.yahoo.vespa.model.ConfigProxy;
+import com.yahoo.vespa.model.Host;
+import com.yahoo.vespa.model.admin.FileDistributionOptions;
+
+import java.util.Collection;
+
+public class FileDistributionConfigProvider {
+
+ private final FileDistributor fileDistributor;
+ private final FileDistributionOptions fileDistributionOptions;
+ private final boolean sendAllFiles;
+ private final Host host;
+
+ public FileDistributionConfigProvider(FileDistributor fileDistributor,
+ FileDistributionOptions fileDistributionOptions,
+ boolean sendAllFiles,
+ Host host) {
+ this.fileDistributor = fileDistributor;
+ this.fileDistributionOptions = fileDistributionOptions;
+ this.sendAllFiles = sendAllFiles;
+ this.host = host;
+ }
+
+ public void getConfig(FiledistributorConfig.Builder builder) {
+ fileDistributionOptions.getConfig(builder);
+ builder.torrentport(FileDistributorService.BASEPORT + 1);
+ builder.stateport(FileDistributorService.BASEPORT + 2);
+ builder.hostname(host.getHostname());
+ builder.filedbpath(FileDistribution.getDefaultFileDBPath().toString());
+ }
+
+ public void getConfig(FiledistributorrpcConfig.Builder builder) {
+ // If disabled config proxy should act as file distributor, so use config proxy port
+ int port = (fileDistributionOptions.disableFiledistributor()) ? ConfigProxy.BASEPORT : FileDistributorService.BASEPORT;
+ builder.connectionspec("tcp/" + host.getHostname() + ":" + port);
+ }
+
+ public void getConfig(FilereferencesConfig.Builder builder) {
+ for (FileReference reference : getFileReferences()) {
+ builder.filereferences(reference.value());
+ }
+ }
+
+ private Collection<FileReference> getFileReferences() {
+ if (sendAllFiles) {
+ return fileDistributor.allFilesToSend();
+ } else {
+ return fileDistributor.filesToSendToHost(host);
+ }
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributor.java b/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributor.java
index 443c88355ed..f3aad88a446 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributor.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributor.java
@@ -9,11 +9,10 @@ import com.yahoo.vespa.model.Host;
import java.util.*;
import java.util.stream.Collectors;
-
/**
* Responsible for directing distribution of files to hosts.
*
- * @author tonytv
+ * @author Tony Vaagenes
*/
public class FileDistributor {
@@ -97,6 +96,7 @@ public class FileDistributor {
for (Host host : getTargetHosts()) {
if ( ! host.getHostname().equals(fileSourceHost)) {
dbHandler.sendDeployedFiles(host.getHostname(), filesToSendToHost(host));
+ dbHandler.startDownload(host.getHostname(), filesToSendToHost(host));
}
}
dbHandler.sendDeployedFiles(fileSourceHost, allFilesToSend());
@@ -108,5 +108,5 @@ public class FileDistributor {
public void reloadDeployFileDistributor(FileDistribution dbHandler) {
dbHandler.reloadDeployFileDistributor();
}
-
+
}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributorService.java b/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributorService.java
index dd9e057e2fa..986334125d2 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributorService.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributorService.java
@@ -4,60 +4,36 @@ package com.yahoo.vespa.model.filedistribution;
import com.yahoo.cloud.config.filedistribution.FiledistributorConfig;
import com.yahoo.cloud.config.filedistribution.FiledistributorrpcConfig;
import com.yahoo.cloud.config.filedistribution.FilereferencesConfig;
-import com.yahoo.config.FileReference;
-
-import com.yahoo.config.model.api.FileDistribution;
import com.yahoo.config.model.producer.AbstractConfigProducer;
import com.yahoo.vespa.model.AbstractService;
-import com.yahoo.vespa.model.admin.FileDistributionOptions;
-
-import java.util.Collection;
/**
- * @author tonytv
+ * @author Tony Vaagenes
+ *
+ * Config is produced by {@link FileDistributionConfigProvider}
*/
public class FileDistributorService extends AbstractService implements
FiledistributorConfig.Producer,
FiledistributorrpcConfig.Producer,
FilereferencesConfig.Producer {
- private final static int BASEPORT = 19092;
- private final FileDistributor fileDistributor;
- private final FileDistributionOptions fileDistributionOptions;
- private final boolean sendAllFiles;
+ final static int BASEPORT = 19092;
- private Collection<FileReference> getFileReferences() {
- if (sendAllFiles) {
- return fileDistributor.allFilesToSend();
- } else {
- return fileDistributor.filesToSendToHost(getHost());
- }
- }
+ private final FileDistributionConfigProvider configProvider;
- public FileDistributorService(AbstractConfigProducer parent,
- String name,
- FileDistributor fileDistributor,
- FileDistributionOptions fileDistributionOptions,
- boolean sendAllFiles) {
- super(parent, name);
+ public FileDistributorService(AbstractConfigProducer parent, String hostname, FileDistributionConfigProvider configProvider) {
+ super(parent, hostname);
+ this.configProvider = configProvider;
portsMeta.on(0).tag("rpc");
portsMeta.on(1).tag("torrent");
portsMeta.on(2).tag("http").tag("state");
setProp("clustertype", "filedistribution");
setProp("clustername", "admin");
-
- this.fileDistributor = fileDistributor;
- this.fileDistributionOptions = fileDistributionOptions;
- this.sendAllFiles = sendAllFiles;
- monitorService();
}
@Override
public String getStartupCommand() {
- // If disabled config proxy should act as file distributor, so don't start this service
- return (fileDistributionOptions.disabled())
- ? null
- : "exec $ROOT/sbin/vespa-filedistributor" + " --configid " + getConfigId();
+ return "exec $ROOT/sbin/vespa-filedistributor" + " --configid " + getConfigId();
}
@Override
@@ -70,6 +46,7 @@ public class FileDistributorService extends AbstractService implements
return true;
}
+ @Override
public int getPortCount() {
return 3;
}
@@ -81,24 +58,16 @@ public class FileDistributorService extends AbstractService implements
@Override
public void getConfig(FiledistributorConfig.Builder builder) {
- fileDistributionOptions.getConfig(builder);
- builder.torrentport(getRelativePort(1));
- builder.stateport(getRelativePort(2));
- builder.hostname(getHostName());
- builder.filedbpath(FileDistribution.getDefaultFileDBPath().toString());
+ configProvider.getConfig(builder);
}
@Override
public void getConfig(FiledistributorrpcConfig.Builder builder) {
- // If disabled config proxy should act as file distributor, so use config proxy port
- int port = (fileDistributionOptions.disabled()) ? 19090 : getRelativePort(0);
- builder.connectionspec("tcp/" + getHostName() + ":" + port);
+ configProvider.getConfig(builder);
}
@Override
public void getConfig(FilereferencesConfig.Builder builder) {
- for (FileReference reference : getFileReferences()) {
- builder.filereferences(reference.value());
- }
+ configProvider.getConfig(builder);
}
}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/Dispatch.java b/config-model/src/main/java/com/yahoo/vespa/model/search/Dispatch.java
index c7610e784ec..87146447dd1 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/search/Dispatch.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/search/Dispatch.java
@@ -46,7 +46,6 @@ public class Dispatch extends AbstractService implements SearchInterface,
setProp("clustertype", "search")
.setProp("clustername", dispatchGroup.getClusterName())
.setProp("index", nodeSpec.groupIndex());
- monitorService();
}
public static Dispatch createTld(DispatchGroup dispatchGroup, AbstractConfigProducer parent, int rowId) {
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java
index 05b0b382814..8420a1f4b91 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java
@@ -19,6 +19,7 @@ import com.yahoo.vespa.model.admin.monitoring.Monitoring;
import com.yahoo.vespa.model.application.validation.RestartConfigs;
import com.yahoo.vespa.model.builder.xml.dom.VespaDomBuilder;
import com.yahoo.vespa.model.content.ContentNode;
+import com.yahoo.vespa.model.filedistribution.DummyFileDistributionConfigProducer;
import com.yahoo.vespa.model.filedistribution.FileDistributionConfigProducer;
import com.yahoo.vespa.model.filedistribution.FileDistributorService;
import org.w3c.dom.Element;
@@ -112,7 +113,6 @@ public class SearchNode extends AbstractService implements
portsMeta.on(UNUSED_3).tag("unused");
portsMeta.on(HEALTH_PORT).tag("http").tag("json").tag("health").tag("state");
// Properties are set in DomSearchBuilder
- monitorService();
this.tuning = tuning;
}
@@ -229,10 +229,12 @@ public class SearchNode extends AbstractService implements
public void getConfig(FiledistributorrpcConfig.Builder builder) {
FileDistributionConfigProducer fileDistribution = getRoot().getFileDistributionConfigProducer();
if (fileDistribution != null) {
- FileDistributorService fds = fileDistribution.getFileDistributorService(getHost());
- if (fds != null) {
- fds.getConfig(builder);
- }
+ AbstractConfigProducer configProducer = fileDistribution.getConfigProducer(getHost());
+ // TODO: Hack, will be fixed when FileDistributorService is gone
+ if (configProducer instanceof DummyFileDistributionConfigProducer)
+ ((DummyFileDistributionConfigProducer) configProducer).getConfig(builder);
+ else
+ ((FileDistributorService) configProducer).getConfig(builder);
}
}
diff --git a/config-model/src/main/resources/schema/admin.rnc b/config-model/src/main/resources/schema/admin.rnc
index b7ec252c4d7..486f1e367e9 100644
--- a/config-model/src/main/resources/schema/admin.rnc
+++ b/config-model/src/main/resources/schema/admin.rnc
@@ -81,7 +81,7 @@ LogServer = element logserver {
FileDistribution = element filedistribution {
element uploadbitrate { xsd:string { pattern = "\d+(\.\d*)?\s*[kmgKMG]?" } }? &
element downloadbitrate { xsd:string { pattern = "\d+(\.\d*)?\s*[kmgKMG]?" } }? &
- element disabled { xsd:boolean }? # Nov. 2017: Temporary, should not be documented
+ element disableFiledistributor { xsd:boolean }? # Nov. 2017: Temporary, should not be documented
}
Metrics = element metrics {
diff --git a/config-model/src/main/resources/schema/containercluster.rnc b/config-model/src/main/resources/schema/containercluster.rnc
index 6a90bef7bb2..95ac198adc4 100644
--- a/config-model/src/main/resources/schema/containercluster.rnc
+++ b/config-model/src/main/resources/schema/containercluster.rnc
@@ -62,6 +62,7 @@ HttpServer = element server {
attribute port { xsd:nonNegativeInteger } &
ComponentId &
element ssl-keystore-configurator { BundleSpec }? & # FOR INTERNAL USE ONLY - SUBJECT TO CHANGE
+ element ssl-truststore-configurator { BundleSpec }? & # FOR INTERNAL USE ONLY - SUBJECT TO CHANGE
GenericConfig*
}
diff --git a/config-model/src/test/cfg/application/invalid_legacy_user_config/configs/qr-searchers.cfg b/config-model/src/test/cfg/application/invalid_legacy_user_config/configs/qr-searchers.cfg
deleted file mode 100644
index 07538a1dce2..00000000000
--- a/config-model/src/test/cfg/application/invalid_legacy_user_config/configs/qr-searchers.cfg
+++ /dev/null
@@ -1,5 +0,0 @@
-builtin[StemmingSearcher].enabled false
-builtin[NoRankingSearcher].enabled false
-tag.bold.open "^_"
-tag.bold.close "^_"
-tag.separator " ... "
diff --git a/config-model/src/test/cfg/application/invalid_legacy_user_config/services.xml b/config-model/src/test/cfg/application/invalid_legacy_user_config/services.xml
deleted file mode 100644
index 9a13d936e46..00000000000
--- a/config-model/src/test/cfg/application/invalid_legacy_user_config/services.xml
+++ /dev/null
@@ -1,10 +0,0 @@
-<?xml version="1.0" encoding="utf-8" ?>
-<!-- Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
-<services version="1.0">
-
-
- <admin version="2.0">
- <adminserver hostalias="node1" />
- </admin>
-
-</services>
diff --git a/config-model/src/test/derived/importedfields/attributes.cfg b/config-model/src/test/derived/importedfields/attributes.cfg
index 62fe0052bc3..f51bbc012b9 100644
--- a/config-model/src/test/derived/importedfields/attributes.cfg
+++ b/config-model/src/test/derived/importedfields/attributes.cfg
@@ -138,3 +138,23 @@ attribute[].upperbound 9223372036854775807
attribute[].densepostinglistthreshold 0.4
attribute[].tensortype ""
attribute[].imported true
+attribute[].name "my_predicate_field"
+attribute[].datatype PREDICATE
+attribute[].collectiontype SINGLE
+attribute[].removeifzero false
+attribute[].createifnonexistent false
+attribute[].fastsearch false
+attribute[].huge false
+attribute[].sortascending true
+attribute[].sortfunction UCA
+attribute[].sortstrength PRIMARY
+attribute[].sortlocale ""
+attribute[].enablebitvectors false
+attribute[].enableonlybitvector false
+attribute[].fastaccess false
+attribute[].arity 8
+attribute[].lowerbound 5
+attribute[].upperbound 300
+attribute[].densepostinglistthreshold 0.4
+attribute[].tensortype ""
+attribute[].imported true
diff --git a/config-model/src/test/derived/importedfields/child.sd b/config-model/src/test/derived/importedfields/child.sd
index 703451c1293..07fa7d15719 100644
--- a/config-model/src/test/derived/importedfields/child.sd
+++ b/config-model/src/test/derived/importedfields/child.sd
@@ -9,6 +9,7 @@ search child {
import field b_ref.string_field as my_string_field {}
import field a_ref.int_array_field as my_int_array_field {}
import field a_ref.int_wset_field as my_int_wset_field {}
+ import field a_ref.predicate_field as my_predicate_field {}
fieldset myfieldset {
fields: my_int_field, my_string_field
diff --git a/config-model/src/test/derived/importedfields/imported-fields.cfg b/config-model/src/test/derived/importedfields/imported-fields.cfg
index 3f2a083bdc2..59413ca5eb7 100644
--- a/config-model/src/test/derived/importedfields/imported-fields.cfg
+++ b/config-model/src/test/derived/importedfields/imported-fields.cfg
@@ -18,3 +18,8 @@ attribute[].referencefield "a_ref"
attribute[].targetfield "int_wset_field"
attribute[].datatype NONE
attribute[].collectiontype SINGLE
+attribute[].name "my_predicate_field"
+attribute[].referencefield "a_ref"
+attribute[].targetfield "predicate_field"
+attribute[].datatype NONE
+attribute[].collectiontype SINGLE
diff --git a/config-model/src/test/derived/importedfields/index-info.cfg b/config-model/src/test/derived/importedfields/index-info.cfg
index 7cb53c480ce..adba0036409 100644
--- a/config-model/src/test/derived/importedfields/index-info.cfg
+++ b/config-model/src/test/derived/importedfields/index-info.cfg
@@ -49,6 +49,12 @@ indexinfo[].command[].indexname "my_int_wset_field"
indexinfo[].command[].command "multivalue"
indexinfo[].command[].indexname "my_int_wset_field"
indexinfo[].command[].command "attribute"
+indexinfo[].command[].indexname "my_predicate_field"
+indexinfo[].command[].command "predicate-bounds [5..300]"
+indexinfo[].command[].indexname "my_predicate_field"
+indexinfo[].command[].command "index"
+indexinfo[].command[].indexname "my_predicate_field"
+indexinfo[].command[].command "attribute"
indexinfo[].command[].indexname "myfieldset"
indexinfo[].command[].command "attribute"
indexinfo[].command[].indexname "myfieldset"
diff --git a/config-model/src/test/derived/importedfields/parent_a.sd b/config-model/src/test/derived/importedfields/parent_a.sd
index eea12375daf..edc81df5609 100644
--- a/config-model/src/test/derived/importedfields/parent_a.sd
+++ b/config-model/src/test/derived/importedfields/parent_a.sd
@@ -10,5 +10,13 @@ search parent_a {
field int_wset_field type weightedset<int> {
indexing: attribute
}
+ field predicate_field type predicate {
+ indexing: attribute
+ index {
+ arity: 8
+ lower-bound: 5
+ upper-bound: 300
+ }
+ }
}
}
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImportedFieldsInSummaryValidatorTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImportedFieldsInSummaryValidatorTestCase.java
new file mode 100644
index 00000000000..4e1f8f1edd7
--- /dev/null
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImportedFieldsInSummaryValidatorTestCase.java
@@ -0,0 +1,39 @@
+package com.yahoo.searchdefinition.processing;
+
+import com.yahoo.document.DataType;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+
+/**
+ * @author geirst
+ */
+public class ImportedFieldsInSummaryValidatorTestCase {
+
+ @Rule
+ public final ExpectedException exceptionRule = ExpectedException.none();
+
+ @Test
+ public void validator_fails_if_imported_predicate_field_is_used_in_document_summary() {
+ exceptionRule.expect(IllegalArgumentException.class);
+ exceptionRule.expectMessage("For search 'child', document summary 'my_summary', " +
+ "imported summary field 'my_predicate_field': Is of type predicate. Not supported in document summaries");
+ new SearchModel()
+ .addImportedField("my_predicate_field", "ref", "predicate_field")
+ .addSummaryField("my_predicate_field", DataType.PREDICATE)
+ .resolve();
+ }
+
+ private static class SearchModel extends ImportedFieldsResolverTestCase.SearchModel {
+
+ public SearchModel() {
+ super();
+ }
+
+ public void resolve() {
+ super.resolve();
+ new ImportedFieldsInSummayValidator(childSearch, null, null, null).process();
+ }
+ }
+}
+
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImportedFieldsResolverTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImportedFieldsResolverTestCase.java
index 5d270bc085a..9f7c7458738 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImportedFieldsResolverTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImportedFieldsResolverTestCase.java
@@ -18,6 +18,8 @@ import com.yahoo.searchdefinition.document.SDField;
import com.yahoo.searchdefinition.document.TemporaryImportedField;
import com.yahoo.searchdefinition.document.TemporarySDField;
import com.yahoo.tensor.TensorType;
+import com.yahoo.vespa.documentmodel.DocumentSummary;
+import com.yahoo.vespa.documentmodel.SummaryField;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;
@@ -107,7 +109,7 @@ public class ImportedFieldsResolverTestCase {
.resolve();
}
- private static class SearchModel {
+ static class SearchModel {
private final ApplicationPackage app = MockApplicationPackage.createEmpty();
public final Search grandParentSearch;
@@ -124,6 +126,7 @@ public class ImportedFieldsResolverTestCase {
parentSearch.getDocument().addField(createField("attribute_and_index", DataType.INT, "{ attribute | index }"));
parentSearch.getDocument().addField(new TemporarySDField("not_attribute", DataType.INT));
parentSearch.getDocument().addField(createField("tensor_field", new TensorDataType(TensorType.fromSpec("tensor(x[])")), "{ attribute }"));
+ parentSearch.getDocument().addField(createField("predicate_field", DataType.PREDICATE, "{ attribute }"));
addRefField(parentSearch, grandParentSearch, "ref");
addImportedField(parentSearch, "ancient_field", "ref", "ancient_field");
@@ -163,6 +166,16 @@ public class ImportedFieldsResolverTestCase {
return this;
}
+ public SearchModel addSummaryField(String fieldName, DataType dataType) {
+ DocumentSummary summary = childSearch.getSummary("my_summary");
+ if (summary == null) {
+ summary = new DocumentSummary("my_summary");
+ childSearch.addSummary(summary);
+ }
+ summary.add(new SummaryField(fieldName, dataType));
+ return this;
+ }
+
public void resolve() {
resolve(grandParentSearch);
resolve(parentSearch);
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/admin/AdminTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/admin/AdminTestCase.java
index 946624a1cdb..fed9000f72b 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/admin/AdminTestCase.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/admin/AdminTestCase.java
@@ -8,7 +8,6 @@ import com.yahoo.cloud.config.SentinelConfig;
import com.yahoo.config.model.ApplicationConfigProducerRoot;
import com.yahoo.config.model.deploy.DeployProperties;
import com.yahoo.config.model.deploy.DeployState;
-import com.yahoo.config.model.test.MockApplicationPackage;
import com.yahoo.config.model.test.TestDriver;
import com.yahoo.config.model.test.TestRoot;
import com.yahoo.config.provision.ApplicationId;
@@ -24,11 +23,9 @@ import com.yahoo.vespa.model.container.component.Component;
import com.yahoo.vespa.model.container.component.StatisticsComponent;
import com.yahoo.vespa.model.test.utils.VespaModelCreatorWithFilePkg;
import com.yahoo.vespa.model.test.utils.VespaModelCreatorWithMockPkg;
-import org.junit.Ignore;
import org.junit.Test;
import java.util.Set;
-import java.util.stream.IntStream;
import static org.hamcrest.CoreMatchers.is;
import static org.junit.Assert.assertEquals;
@@ -288,7 +285,7 @@ public class AdminTestCase {
" <admin version='2.0'>" +
" <adminserver hostalias='node0' />" +
" <filedistribution>" +
- " <disabled>true</disabled>" +
+ " <disableFiledistributor>true</disableFiledistributor>" +
" </filedistribution>" +
" </admin>" +
"</services>";
@@ -307,4 +304,24 @@ public class AdminTestCase {
assertThat(sentinelConfig.service(2).name(), is("logd"));
// No filedistributor service
}
+
+ @Test
+ public void testDisableFileDistributorForAllApps() {
+ DeployState state = new DeployState.Builder()
+ .disableFiledistributor(true)
+ .properties(
+ new DeployProperties.Builder().
+ zone(new Zone(Environment.dev, RegionName.from("baz"))).
+ applicationId(new ApplicationId.Builder().
+ tenant("quux").
+ applicationName("foo").instanceName("bim").build()).build()).build();
+ TestRoot root = new TestDriver().buildModel(state);
+ String localhost = HostName.getLocalhost();
+ SentinelConfig sentinelConfig = root.getConfig(SentinelConfig.class, "hosts/" + localhost);
+ assertThat(sentinelConfig.service().size(), is(3));
+ assertThat(sentinelConfig.service(0).name(), is("logserver"));
+ assertThat(sentinelConfig.service(1).name(), is("slobrok"));
+ assertThat(sentinelConfig.service(2).name(), is("logd"));
+ }
+
}
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2BuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2BuilderTest.java
index 6c1bf6c9748..446ceb686e8 100755
--- a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2BuilderTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2BuilderTest.java
@@ -207,7 +207,10 @@ public class DomAdminV2BuilderTest extends DomBuilderTest {
}
private Admin buildAdmin(Element xml, boolean multitenant, List<ConfigServerSpec> configServerSpecs) {
- final DomAdminV2Builder domAdminBuilder = new DomAdminV2Builder(ConfigModelContext.ApplicationType.DEFAULT, root.getDeployState().getFileRegistry(), multitenant, configServerSpecs);
+ final DomAdminV2Builder domAdminBuilder =
+ new DomAdminV2Builder(ConfigModelContext.ApplicationType.DEFAULT,
+ root.getDeployState().getFileRegistry(), multitenant,
+ configServerSpecs, root.getDeployState().disableFiledistributor());
Admin admin = domAdminBuilder.build(root, xml);
admin.addPerHostServices(root.getHostSystem().getHosts(), new DeployProperties.Builder().build());
return admin;
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/clients/test/Gateway20TestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/clients/test/Gateway20TestCase.java
deleted file mode 100644
index e7df151a3c4..00000000000
--- a/config-model/src/test/java/com/yahoo/vespa/model/clients/test/Gateway20TestCase.java
+++ /dev/null
@@ -1,65 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.model.clients.test;
-
-import com.yahoo.container.ComponentsConfig;
-import com.yahoo.container.QrConfig;
-import com.yahoo.container.QrConfig.Builder;
-import com.yahoo.net.HostName;
-import com.yahoo.vespa.model.VespaModel;
-import com.yahoo.vespa.model.test.utils.VespaModelCreatorWithFilePkg;
-import org.junit.Test;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-
-import static org.junit.Assert.*;
-
-/**
- * @author Gunnar Gauslaa Bergem
- */
-public class Gateway20TestCase {
-
- private static String hostname = HostName.getLocalhost(); // Using the same way of getting hostname as filedistribution model
-
- @Test
- public void testSimpleDocprocV3() throws Exception {
- VespaModel model = new VespaModelCreatorWithFilePkg("src/test/cfg/clients/simpleconfig.v2.docprocv3").create();
- QrConfig qrConfig = new QrConfig((Builder) model.getConfig(new QrConfig.Builder(), "container/container.0"));
- assertEquals(qrConfig.rpc().enabled(), true);
- assertEquals("filedistribution/" + hostname, qrConfig.filedistributor().configid());
- assertEquals("container.container.0", qrConfig.discriminator());
-
- ComponentsConfig componentsConfig = new ComponentsConfig((ComponentsConfig.Builder) model.getConfig(new ComponentsConfig.Builder(), "container/container.0"));
- ArrayList<String> components = new ArrayList<>();
- for (ComponentsConfig.Components component : componentsConfig.components()) {
- components.add(component.id());
- }
- List<String> expectedComponents = Arrays.asList("com.yahoo.docproc.jdisc.DocumentProcessingHandler",
- "com.yahoo.feedhandler.VespaFeedHandler",
- "com.yahoo.feedhandler.VespaFeedHandlerCompatibility",
- "com.yahoo.feedhandler.VespaFeedHandlerGet",
- "com.yahoo.feedhandler.VespaFeedHandlerRemove",
- "com.yahoo.feedhandler.VespaFeedHandlerRemoveLocation",
- "com.yahoo.feedhandler.VespaFeedHandlerStatus",
- "com.yahoo.feedhandler.VespaFeedHandlerVisit",
- "com.yahoo.search.handler.SearchHandler",
- "com.yahoo.container.jdisc.state.StateHandler");
- assertTrue(components.containsAll(expectedComponents));
- }
-
- @Test
- public void testAdvanced() throws Exception {
- VespaModel model = new VespaModelCreatorWithFilePkg("src/test/cfg/clients/advancedconfig.v2").create();
-
- QrConfig qrConfig = new QrConfig((Builder) model.getConfig(new QrConfig.Builder(), "container/container.0"));
- assertEquals(qrConfig.rpc().enabled(), true);
- assertEquals(qrConfig.filedistributor().configid(), "filedistribution/" + hostname);
- assertEquals("container.container.0", qrConfig.discriminator());
-
- qrConfig = new QrConfig((Builder) model.getConfig(new QrConfig.Builder(), "container/container.0"));
- assertEquals(qrConfig.rpc().enabled(), true);
- assertEquals(qrConfig.filedistributor().configid(), "filedistribution/" + hostname);
- }
-
-}
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/configserver/TestOptions.java b/config-model/src/test/java/com/yahoo/vespa/model/container/configserver/TestOptions.java
index e2c8f2e2c52..1784fe0e974 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/container/configserver/TestOptions.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/container/configserver/TestOptions.java
@@ -6,8 +6,7 @@ import com.yahoo.vespa.model.container.configserver.option.CloudConfigOptions;
import java.util.Optional;
/**
- * @author lulf
- * @since 5.
+ * @author Ulf Lilleengen
*/
public class TestOptions implements CloudConfigOptions {
private Optional<Integer> rpcPort = Optional.empty();
@@ -20,6 +19,7 @@ public class TestOptions implements CloudConfigOptions {
private Optional<Boolean> useVespaVersionInRequest = Optional.empty();
private Optional<Boolean> hostedVespa = Optional.empty();
private Optional<Integer> numParallelTenantLoaders = Optional.empty();
+ private Optional<Boolean> disableFiledistributor = Optional.empty();
@Override
public Optional<Integer> rpcPort() {
@@ -118,6 +118,9 @@ public class TestOptions implements CloudConfigOptions {
@Override
public Optional<String> loadBalancerAddress() { return Optional.empty(); }
+ @Override
+ public Optional<Boolean> disableFiledistributor() { return disableFiledistributor; }
+
public TestOptions numParallelTenantLoaders(int numLoaders) {
this.numParallelTenantLoaders = Optional.of(numLoaders);
return this;
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java
index d09211aea45..5e093bdb32a 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java
@@ -16,6 +16,7 @@ import com.yahoo.config.provision.Environment;
import com.yahoo.config.provision.RegionName;
import com.yahoo.config.provision.Zone;
import com.yahoo.container.ComponentsConfig;
+import com.yahoo.container.QrConfig;
import com.yahoo.container.config.StatisticsRequestHandler;
import com.yahoo.container.core.ChainsConfig;
import com.yahoo.container.core.VipStatusConfig;
@@ -25,6 +26,7 @@ import com.yahoo.container.jdisc.JdiscBindingsConfig;
import com.yahoo.container.servlet.ServletConfigConfig;
import com.yahoo.container.usability.BindingsOverviewHandler;
import com.yahoo.jdisc.http.ServletPathsConfig;
+import com.yahoo.net.HostName;
import com.yahoo.prelude.cluster.QrMonitorConfig;
import com.yahoo.vespa.model.AbstractService;
import com.yahoo.vespa.model.VespaModel;
@@ -61,7 +63,6 @@ import static org.junit.Assert.fail;
/**
* @author gjoranv
- * @since 5.1.9
*/
public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase {
@@ -571,6 +572,41 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase {
assertEquals(ContainerModelBuilder.HOSTED_VESPA_STATUS_FILE, vipStatusConfig.statusfile());
}
+ @Test
+ public void qrconfig_is_produced() throws IOException, SAXException {
+ String servicesXml =
+ "<services>" +
+ "<admin version='3.0'>" +
+ " <nodes count='1'/>" +
+ "</admin>" +
+ "<jdisc id ='default' version='1.0'>" +
+ " <nodes>" +
+ " <node hostalias='node1' />" +
+ " </nodes>" +
+ "</jdisc>" +
+ "</services>";
+
+ ApplicationPackage applicationPackage = new MockApplicationPackage.Builder()
+ .withServices(servicesXml)
+ .build();
+ VespaModel model = new VespaModel(new NullConfigModelRegistry(), new DeployState.Builder()
+ .applicationPackage(applicationPackage)
+ .properties(new DeployProperties.Builder().build())
+ .build());
+
+ String hostname = HostName.getLocalhost(); // Using the same way of getting hostname as filedistribution model
+
+ QrConfig config = model.getConfig(QrConfig.class, "default/container.0");
+ assertEquals("default.container.0", config.discriminator());
+ assertEquals(19102, config.rpc().port());
+ assertEquals("vespa/service/default/container.0", config.rpc().slobrokId());
+ assertEquals(true, config.rpc().enabled());
+ assertEquals("", config.rpc().host());
+ assertEquals(false, config.restartOnDeploy());
+ assertEquals(false, config.coveragereports());
+ assertEquals("filedistribution/" + hostname, config.filedistributor().configid());
+ }
+
private Element generateContainerElementWithRenderer(String rendererId) {
return DomBuilderTest.parse(
"<jdisc id='default' version='1.0'>",
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/JettyContainerModelBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/JettyContainerModelBuilderTest.java
index 1e24b055095..54c4aabf44c 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/JettyContainerModelBuilderTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/JettyContainerModelBuilderTest.java
@@ -7,6 +7,7 @@ import com.yahoo.container.bundle.BundleInstantiationSpecification;
import com.yahoo.container.jdisc.FilterBindingsProvider;
import com.yahoo.jdisc.http.ConnectorConfig;
import com.yahoo.jdisc.http.ssl.DefaultSslKeyStoreConfigurator;
+import com.yahoo.jdisc.http.ssl.DefaultSslTrustStoreConfigurator;
import com.yahoo.vespa.model.container.ContainerCluster;
import com.yahoo.vespa.model.container.component.SimpleComponent;
import com.yahoo.vespa.model.container.http.ConnectorFactory;
@@ -16,7 +17,9 @@ import org.w3c.dom.Element;
import org.xml.sax.SAXException;
import java.io.IOException;
+import java.util.Arrays;
import java.util.List;
+import java.util.Set;
import static com.yahoo.jdisc.http.ConnectorConfig.Ssl.KeyStoreType;
import static org.hamcrest.CoreMatchers.equalTo;
@@ -28,7 +31,6 @@ import static org.junit.Assert.assertThat;
/**
* @author einarmr
- * @since 5.15
*/
public class JettyContainerModelBuilderTest extends ContainerModelBuilderTestBase {
@@ -190,12 +192,13 @@ public class JettyContainerModelBuilderTest extends ContainerModelBuilderTestBas
}
@Test
- public void ssl_keystore_configurator_can_be_overriden() throws IOException, SAXException {
+ public void ssl_keystore_and_truststore_configurator_can_be_overriden() throws IOException, SAXException {
Element clusterElem = DomBuilderTest.parse(
"<jdisc id='default' version='1.0' jetty='true'>",
" <http>",
" <server port='9000' id='foo'>",
" <ssl-keystore-configurator class='com.yahoo.MySslKeyStoreConfigurator' bundle='mybundle'/>",
+ " <ssl-truststore-configurator class='com.yahoo.MySslTrustStoreConfigurator' bundle='mybundle'/>",
" </server>",
" <server port='9001' id='bar'/>",
" </http>",
@@ -204,27 +207,47 @@ public class JettyContainerModelBuilderTest extends ContainerModelBuilderTestBas
createModel(root, clusterElem);
ContainerCluster cluster = (ContainerCluster) root.getChildren().get("default");
List<ConnectorFactory> connectorFactories = cluster.getChildrenByTypeRecursive(ConnectorFactory.class);
-
{
ConnectorFactory firstConnector = connectorFactories.get(0);
- assertThat(firstConnector.getInjectedComponentIds(), hasItem("ssl-keystore-configurator@foo"));
- assertThat(firstConnector.getInjectedComponentIds().size(), equalTo(1));
- SimpleComponent sslKeystoreConfigurator = firstConnector.getChildrenByTypeRecursive(SimpleComponent.class).get(0);
- BundleInstantiationSpecification spec = sslKeystoreConfigurator.model.bundleInstantiationSpec;
- assertThat(spec.classId.toString(), is("com.yahoo.MySslKeyStoreConfigurator"));
- assertThat(spec.bundle.toString(), is("mybundle"));
+ assertConnectorHasInjectedComponents(firstConnector, "ssl-keystore-configurator@foo", "ssl-truststore-configurator@foo");
+ assertComponentHasClassNameAndBundle(getChildComponent(firstConnector, 0),
+ "com.yahoo.MySslKeyStoreConfigurator",
+ "mybundle");
+ assertComponentHasClassNameAndBundle(getChildComponent(firstConnector, 1),
+ "com.yahoo.MySslTrustStoreConfigurator",
+ "mybundle");
}
{
- ConnectorFactory secondFactory = connectorFactories.get(1);
- assertThat(secondFactory.getInjectedComponentIds(), hasItem("ssl-keystore-configurator@bar"));
- assertThat(secondFactory.getInjectedComponentIds().size(), equalTo(1));
- SimpleComponent sslKeystoreConfigurator = secondFactory.getChildrenByTypeRecursive(SimpleComponent.class).get(0);
- BundleInstantiationSpecification spec = sslKeystoreConfigurator.model.bundleInstantiationSpec;
- assertThat(spec.classId.toString(), is(DefaultSslKeyStoreConfigurator.class.getName()));
- assertThat(spec.bundle.toString(), is("jdisc_http_service"));
+ ConnectorFactory secondConnector = connectorFactories.get(1);
+ assertConnectorHasInjectedComponents(secondConnector, "ssl-keystore-configurator@bar", "ssl-truststore-configurator@bar");
+ assertComponentHasClassNameAndBundle(getChildComponent(secondConnector, 0),
+ DefaultSslKeyStoreConfigurator.class.getName(),
+ "jdisc_http_service");
+ assertComponentHasClassNameAndBundle(getChildComponent(secondConnector, 1),
+ DefaultSslTrustStoreConfigurator.class.getName(),
+ "jdisc_http_service");
}
}
+ private static void assertConnectorHasInjectedComponents(ConnectorFactory connectorFactory, String... componentNames) {
+ Set<String> injectedComponentIds = connectorFactory.getInjectedComponentIds();
+ assertThat(injectedComponentIds.size(), equalTo(componentNames.length));
+ Arrays.stream(componentNames)
+ .forEach(name -> assertThat(injectedComponentIds, hasItem(name)));
+ }
+
+ private static SimpleComponent getChildComponent(ConnectorFactory connectorFactory, int index) {
+ return connectorFactory.getChildrenByTypeRecursive(SimpleComponent.class).get(index);
+ }
+
+ private static void assertComponentHasClassNameAndBundle(SimpleComponent simpleComponent,
+ String className,
+ String bundleName) {
+ BundleInstantiationSpecification spec = simpleComponent.model.bundleInstantiationSpec;
+ assertThat(spec.classId.toString(), is(className));
+ assertThat(spec.bundle.toString(), is(bundleName));
+ }
+
private void assertJettyServerInConfig() {
ContainerCluster cluster = (ContainerCluster) root.getChildren().get("default");
List<JettyHttpServer> jettyServers = cluster.getChildrenByTypeRecursive(JettyHttpServer.class);
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentSearchClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentSearchClusterTest.java
index 5f18b28d6ce..41bba055a50 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentSearchClusterTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentSearchClusterTest.java
@@ -1,9 +1,11 @@
// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.model.content;
+import com.yahoo.vespa.config.content.core.BucketspacesConfig;
import com.yahoo.vespa.config.search.core.ProtonConfig;
import com.yahoo.vespa.model.content.cluster.ContentCluster;
import com.yahoo.vespa.model.content.utils.ContentClusterBuilder;
+import com.yahoo.vespa.model.content.utils.DocType;
import com.yahoo.vespa.model.content.utils.SearchDefinitionBuilder;
import org.junit.Test;
@@ -15,6 +17,7 @@ import static com.yahoo.config.model.test.TestUtil.joinLines;
import static com.yahoo.vespa.model.content.utils.ContentClusterUtils.createCluster;
import static com.yahoo.vespa.model.content.utils.SearchDefinitionBuilder.createSearchDefinitions;
import static junit.framework.TestCase.assertEquals;
+import static junit.framework.TestCase.assertFalse;
/**
* Unit tests for content search cluster.
@@ -36,8 +39,8 @@ public class ContentSearchClusterTest {
private static ContentCluster createClusterWithGlobalType() throws Exception {
return createCluster(new ContentClusterBuilder().docTypes(Arrays.asList(
- new ContentClusterBuilder.DocType("global", true),
- new ContentClusterBuilder.DocType("regular"))).getXml(),
+ DocType.indexGlobal("global"),
+ DocType.index("regular"))).getXml(),
createSearchDefinitions("global", "regular"));
}
@@ -108,10 +111,31 @@ public class ContentSearchClusterTest {
.content("field ref_to_c type reference<c> { indexing: attribute }").build());
searchDefinitions.add(new SearchDefinitionBuilder().name("c").build());
return createCluster(new ContentClusterBuilder().docTypes(Arrays.asList(
- new ContentClusterBuilder.DocType("a"),
- new ContentClusterBuilder.DocType("b", true),
- new ContentClusterBuilder.DocType("c", true))).getXml(),
+ DocType.index("a"),
+ DocType.indexGlobal("b"),
+ DocType.indexGlobal("c"))).getXml(),
searchDefinitions);
}
+ private static BucketspacesConfig getBucketspacesConfig(ContentCluster cluster) {
+ BucketspacesConfig.Builder builder = new BucketspacesConfig.Builder();
+ cluster.getConfig(builder);
+ return new BucketspacesConfig(builder);
+ }
+
+ private static void assertDocumentType(String expName, String expBucketSpace, BucketspacesConfig.Documenttype docType) {
+ assertEquals(expName, docType.name());
+ assertEquals(expBucketSpace, docType.bucketspace());
+ }
+
+ @Test
+ public void require_that_bucket_spaces_config_is_produced_for_content_cluster() throws Exception {
+ BucketspacesConfig config = getBucketspacesConfig(createClusterWithGlobalType());
+ assertEquals(2, config.documenttype().size());
+ assertDocumentType("global", "global", config.documenttype(0));
+ assertDocumentType("regular", "default", config.documenttype(1));
+ // Safeguard against flipping the switch
+ assertFalse(config.enable_multiple_bucket_spaces());
+ }
+
}
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/DistributorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/DistributorTest.java
index d4e804d3f95..48b7ccdad6b 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/content/DistributorTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/content/DistributorTest.java
@@ -1,23 +1,22 @@
// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.model.content;
-import com.yahoo.vespa.config.content.core.BucketspacesConfig;
import com.yahoo.vespa.config.content.core.StorCommunicationmanagerConfig;
import com.yahoo.vespa.config.content.core.StorDistributormanagerConfig;
import com.yahoo.vespa.config.content.core.StorServerConfig;
import com.yahoo.config.model.test.MockRoot;
import com.yahoo.vespa.model.content.cluster.ContentCluster;
import com.yahoo.vespa.model.content.utils.ContentClusterUtils;
+import com.yahoo.vespa.model.content.utils.DocType;
import com.yahoo.vespa.model.test.utils.ApplicationPackageUtils;
import org.junit.Test;
-import java.util.Arrays;
import java.util.List;
-import java.util.stream.Collectors;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertThat;
import static org.hamcrest.Matchers.*;
+
/**
* Test for content DistributorCluster.
*/
@@ -303,100 +302,48 @@ public class DistributorTest {
return new StorDistributormanagerConfig(builder);
}
- private static class DocDef {
- public final String type;
- public final String mode;
- public final boolean global;
-
- private DocDef(String type, String mode, boolean global) {
- this.type = type;
- this.mode = mode;
- this.global = global;
- }
-
- public static DocDef storeOnly(String type) {
- return new DocDef(type, "store-only", false);
- }
-
- public static DocDef index(String type) {
- return new DocDef(type, "index", false);
- }
-
- public static DocDef indexGlobal(String type) {
- return new DocDef(type, "index", true);
- }
-
- public static DocDef streaming(String type) {
- return new DocDef(type, "streaming", false);
- }
- }
-
- private String generateXmlForDocDefs(DocDef... defs) {
+ private String generateXmlForDocTypes(DocType... docTypes) {
return "<content id='storage'>\n" +
- " <documents>\n" +
- Arrays.stream(defs)
- .map(def -> String.format(" <document type='%s' mode='%s' global='%s'/>",
- def.type, def.mode, (def.global ? "true" : "false")))
- .collect(Collectors.joining("\n")) +
- "\n </documents>\n" +
- "</content>";
+ DocType.listToXml(docTypes) +
+ "\n</content>";
}
@Test
public void bucket_activation_disabled_if_no_documents_in_indexed_mode() {
StorDistributormanagerConfig config = clusterXmlToConfig(
- generateXmlForDocDefs(DocDef.storeOnly("music")));
+ generateXmlForDocTypes(DocType.storeOnly("music")));
assertThat(config.disable_bucket_activation(), is(true));
}
@Test
public void bucket_activation_enabled_with_single_indexed_document() {
StorDistributormanagerConfig config = clusterXmlToConfig(
- generateXmlForDocDefs(DocDef.index("music")));
+ generateXmlForDocTypes(DocType.index("music")));
assertThat(config.disable_bucket_activation(), is(false));
}
@Test
public void bucket_activation_enabled_with_multiple_indexed_documents() {
StorDistributormanagerConfig config = clusterXmlToConfig(
- generateXmlForDocDefs(DocDef.index("music"),
- DocDef.index("movies")));
+ generateXmlForDocTypes(DocType.index("music"),
+ DocType.index("movies")));
assertThat(config.disable_bucket_activation(), is(false));
}
@Test
public void bucket_activation_enabled_if_at_least_one_document_indexed() {
StorDistributormanagerConfig config = clusterXmlToConfig(
- generateXmlForDocDefs(DocDef.storeOnly("music"),
- DocDef.streaming("bunnies"),
- DocDef.index("movies")));
+ generateXmlForDocTypes(DocType.storeOnly("music"),
+ DocType.streaming("bunnies"),
+ DocType.index("movies")));
assertThat(config.disable_bucket_activation(), is(false));
}
@Test
public void bucket_activation_disabled_for_single_streaming_type() {
StorDistributormanagerConfig config = clusterXmlToConfig(
- generateXmlForDocDefs(DocDef.streaming("music")));
+ generateXmlForDocTypes(DocType.streaming("music")));
assertThat(config.disable_bucket_activation(), is(true));
}
- private BucketspacesConfig clusterXmlToBucketspacesConfig(String xml) {
- BucketspacesConfig.Builder builder = new BucketspacesConfig.Builder();
- parse(xml).getConfig(builder);
- return new BucketspacesConfig(builder);
- }
-
- private void assertDocumentType(String expName, String expBucketSpace, BucketspacesConfig.Documenttype docType) {
- assertEquals(expName, docType.name());
- assertEquals(expBucketSpace, docType.bucketspace());
- }
-
- @Test
- public void bucket_spaces_config_is_produced_for_distributor_cluster() {
- BucketspacesConfig config = clusterXmlToBucketspacesConfig(
- generateXmlForDocDefs(DocDef.index("music"), DocDef.indexGlobal("movies")));
- assertEquals(2, config.documenttype().size());
- assertDocumentType("movies", "global", config.documenttype(0));
- assertDocumentType("music", "default", config.documenttype(1));
- }
}
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/utils/ContentClusterBuilder.java b/config-model/src/test/java/com/yahoo/vespa/model/content/utils/ContentClusterBuilder.java
index 592e90efd22..95c57bb544c 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/content/utils/ContentClusterBuilder.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/content/utils/ContentClusterBuilder.java
@@ -18,29 +18,10 @@ import static com.yahoo.config.model.test.TestUtil.joinLines;
*/
public class ContentClusterBuilder {
- public static class DocType {
- private final String name;
- private final boolean global;
-
- public DocType(String name, boolean global) {
- this.name = name;
- this.global = global;
- }
-
- public DocType(String name) {
- this(name, false);
- }
-
- public String toXml() {
- return (global ? "<document mode='index' type='" + name + "' global='true'/>" :
- "<document mode='index' type='" + name + "'/>");
- }
- }
-
private String name = "mycluster";
private int redundancy = 1;
private int searchableCopies = 1;
- private List<DocType> docTypes = Arrays.asList(new DocType("test"));
+ private List<DocType> docTypes = Arrays.asList(DocType.index("test"));
private String groupXml = getSimpleGroupXml();
private Optional<String> dispatchXml = Optional.empty();
private Optional<Double> protonDiskLimit = Optional.empty();
@@ -66,7 +47,7 @@ public class ContentClusterBuilder {
public ContentClusterBuilder docTypes(String ... docTypes) {
this.docTypes = Arrays.asList(docTypes).stream().
- map(type -> new DocType(type)).
+ map(type -> DocType.index(type)).
collect(Collectors.toList());
return this;
}
@@ -103,9 +84,7 @@ public class ContentClusterBuilder {
public String getXml() {
String xml = joinLines("<content version='1.0' id='" + name + "'>",
" <redundancy>" + redundancy + "</redundancy>",
- " <documents>",
- docTypes.stream().map(DocType::toXml).collect(Collectors.joining("\n")),
- " </documents>",
+ DocType.listToXml(docTypes),
" <engine>",
" <proton>",
" <searchable-copies>" + searchableCopies + "</searchable-copies>",
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/utils/ContentClusterUtils.java b/config-model/src/test/java/com/yahoo/vespa/model/content/utils/ContentClusterUtils.java
index 3a2633ed7b7..e557e3674b5 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/content/utils/ContentClusterUtils.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/content/utils/ContentClusterUtils.java
@@ -4,6 +4,7 @@ package com.yahoo.vespa.model.content.utils;
import com.yahoo.config.application.api.ApplicationPackage;
import com.yahoo.config.model.ConfigModelContext;
import com.yahoo.config.model.api.HostProvisioner;
+import com.yahoo.config.model.application.provider.MockFileRegistry;
import com.yahoo.config.model.deploy.DeployState;
import com.yahoo.config.model.provision.InMemoryProvisioner;
import com.yahoo.config.model.provision.SingleNodeProvisioner;
@@ -11,9 +12,11 @@ import com.yahoo.config.model.test.MockApplicationPackage;
import com.yahoo.config.model.test.MockRoot;
import com.yahoo.text.XML;
import com.yahoo.vespa.model.admin.Admin;
+import com.yahoo.vespa.model.admin.FileDistributionOptions;
import com.yahoo.vespa.model.admin.monitoring.DefaultMonitoring;
import com.yahoo.vespa.model.admin.monitoring.builder.Metrics;
import com.yahoo.vespa.model.content.cluster.ContentCluster;
+import com.yahoo.vespa.model.filedistribution.FileDistributionConfigProducer;
import org.w3c.dom.Document;
import java.util.Collections;
@@ -57,7 +60,8 @@ public class ContentClusterUtils {
public static ContentCluster createCluster(String clusterXml, MockRoot root) throws Exception {
Document doc = XML.getDocument(clusterXml);
- Admin admin = new Admin(root, new DefaultMonitoring("vespa", 60), new Metrics(), Collections.emptyMap(), false);
+ Admin admin = new Admin(root, new DefaultMonitoring("vespa", 60), new Metrics(), Collections.emptyMap(), false,
+ new FileDistributionConfigProducer.Builder(FileDistributionOptions.defaultOptions()).build(root, new MockFileRegistry()));
ConfigModelContext context = ConfigModelContext.create(null, root.getDeployState(), null, root, null);
return new ContentCluster.Builder(admin).build(Collections.emptyList(), context, doc.getDocumentElement());
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/utils/DocType.java b/config-model/src/test/java/com/yahoo/vespa/model/content/utils/DocType.java
new file mode 100644
index 00000000000..3a5f679509b
--- /dev/null
+++ b/config-model/src/test/java/com/yahoo/vespa/model/content/utils/DocType.java
@@ -0,0 +1,54 @@
+package com.yahoo.vespa.model.content.utils;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.stream.Collectors;
+
+/**
+ * Definition of a document type used for testing.
+ *
+ * @author geirst
+ */
+public class DocType {
+ private final String type;
+ private final String mode;
+ private final boolean global;
+
+ private DocType(String type, String mode, boolean global) {
+ this.type = type;
+ this.mode = mode;
+ this.global = global;
+ }
+
+ public String toXml() {
+ return (global ? "<document mode='" + mode + "' type='" + type + "' global='true'/>" :
+ "<document mode='" + mode + "' type='" + type + "'/>");
+ }
+
+ public static DocType storeOnly(String type) {
+ return new DocType(type, "store-only", false);
+ }
+
+ public static DocType index(String type) {
+ return new DocType(type, "index", false);
+ }
+
+ public static DocType indexGlobal(String type) {
+ return new DocType(type, "index", true);
+ }
+
+ public static DocType streaming(String type) {
+ return new DocType(type, "streaming", false);
+ }
+
+ public static String listToXml(DocType... docTypes) {
+ return listToXml(Arrays.asList(docTypes));
+ }
+
+ public static String listToXml(List<DocType> docTypes) {
+ return "<documents>\n" +
+ docTypes.stream().map(DocType::toXml).collect(Collectors.joining("\n")) + "\n" +
+ "</documents>";
+ }
+
+}
diff --git a/config-model/src/test/schema-test-files/services.xml b/config-model/src/test/schema-test-files/services.xml
index a02346193cc..af316c2e3a7 100644
--- a/config-model/src/test/schema-test-files/services.xml
+++ b/config-model/src/test/schema-test-files/services.xml
@@ -108,6 +108,7 @@
<server port="4080" id="myServer">
<ssl-keystore-configurator class="com.yahoo.MySslKeyStoreConfigurator" bundle="mybundle" />
+ <ssl-truststore-configurator class="com.yahoo.MySslTrustStoreConfigurator" bundle="mybundle" />
</server>
<server port="4081" id="anotherServer">
<config name="container.jdisc.config.http-server">