diff options
Diffstat (limited to 'config-model')
30 files changed, 973 insertions, 335 deletions
diff --git a/config-model/pom.xml b/config-model/pom.xml index c159d00eb24..53ec473745b 100644 --- a/config-model/pom.xml +++ b/config-model/pom.xml @@ -272,11 +272,6 @@ </dependency> <dependency> <groupId>com.yahoo.vespa</groupId> - <artifactId>filedistributionmanager</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> <artifactId>searchsummary</artifactId> <version>${project.version}</version> </dependency> diff --git a/config-model/src/main/java/com/yahoo/config/model/admin/AdminModel.java b/config-model/src/main/java/com/yahoo/config/model/admin/AdminModel.java index 5912b476783..d2ffe10677b 100644 --- a/config-model/src/main/java/com/yahoo/config/model/admin/AdminModel.java +++ b/config-model/src/main/java/com/yahoo/config/model/admin/AdminModel.java @@ -78,8 +78,7 @@ public class AdminModel extends ConfigModel { DomAdminV2Builder domBuilder = new DomAdminV2Builder(modelContext.getApplicationType(), modelContext.getDeployState().getFileRegistry(), properties.multitenant(), - properties.configServerSpecs(), - modelContext.getDeployState().disableFiledistributor()); + properties.configServerSpecs()); model.admin = domBuilder.build(parent, adminElement); // TODO: Is required since other models depend on admin. if (parent instanceof ApplicationConfigProducerRoot) { @@ -108,8 +107,7 @@ public class AdminModel extends ConfigModel { DomAdminV4Builder domBuilder = new DomAdminV4Builder(modelContext, properties.multitenant(), properties.configServerSpecs(), - model.getContainerModels(), - modelContext.getDeployState().disableFiledistributor()); + model.getContainerModels()); model.admin = domBuilder.build(parent, adminElement); // TODO: Is required since other models depend on admin. if (parent instanceof ApplicationConfigProducerRoot) { diff --git a/config-model/src/main/java/com/yahoo/config/model/test/MockRoot.java b/config-model/src/main/java/com/yahoo/config/model/test/MockRoot.java index da034619d94..a3429c0471d 100644 --- a/config-model/src/main/java/com/yahoo/config/model/test/MockRoot.java +++ b/config-model/src/main/java/com/yahoo/config/model/test/MockRoot.java @@ -149,7 +149,7 @@ public class MockRoot extends AbstractConfigProducerRoot { try { Document doc = XmlHelper.getDocumentBuilder().parse(new InputSource(new StringReader(servicesXml))); setAdmin(new DomAdminV2Builder(ConfigModelContext.ApplicationType.DEFAULT, deployState.getFileRegistry(), - false, new ArrayList<>(), deployState.disableFiledistributor()). + false, new ArrayList<>()). build(this, XML.getChildren(doc.getDocumentElement(), "admin").get(0))); } catch (SAXException | IOException e) { throw new RuntimeException(e); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/ConstantTensorTransformer.java b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/ConstantTensorTransformer.java index 72c79608d14..843248961c2 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/ConstantTensorTransformer.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/ConstantTensorTransformer.java @@ -52,7 +52,7 @@ public class ConstantTensorTransformer extends ExpressionTransformer<RankProfile private ExpressionNode transformConstantReference(ReferenceNode node, RankProfileTransformContext context) { Value value = context.constants().get(node.getName()); - if (value == null || ! (value instanceof TensorValue)) { + if (value == null || value.type().rank() == 0) { return node; } TensorValue tensorValue = (TensorValue)value; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/AbstractService.java b/config-model/src/main/java/com/yahoo/vespa/model/AbstractService.java index 65f7bbedc68..540393cec7b 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/AbstractService.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/AbstractService.java @@ -6,7 +6,6 @@ import com.yahoo.config.model.api.PortInfo; import com.yahoo.config.model.api.ServiceInfo; import com.yahoo.config.model.producer.AbstractConfigProducer; import com.yahoo.vespa.defaults.Defaults; -import com.yahoo.vespa.filedistribution.PathDoesNotExistException; import java.util.Collection; import java.util.LinkedHashMap; @@ -502,18 +501,10 @@ public abstract class AbstractService extends AbstractConfigProducer<AbstractCon * @return the file reference hash */ public FileReference sendFile(String relativePath) { - try { - return getRoot().getFileDistributor().sendFileToHost(relativePath, getHost()); - } catch (PathDoesNotExistException e) { - throw new RuntimeException("File does not exist: '" + relativePath + "'."); - } + return getRoot().getFileDistributor().sendFileToHost(relativePath, getHost()); } public FileReference sendUri(String uri) { - try { - return getRoot().getFileDistributor().sendUriToHost(uri, getHost()); - } catch (PathDoesNotExistException e) { - throw new RuntimeException("Uri does not exist: '" + uri + "'."); - } + return getRoot().getFileDistributor().sendUriToHost(uri, getHost()); } /** diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java index eaa4f18e35a..68030cd47da 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java @@ -22,7 +22,6 @@ import com.yahoo.vespa.model.filedistribution.DummyFileDistributionConfigProduce import com.yahoo.vespa.model.filedistribution.FileDistributionConfigProducer; import com.yahoo.vespa.model.filedistribution.FileDistributionConfigProvider; import com.yahoo.vespa.model.filedistribution.FileDistributor; -import com.yahoo.vespa.model.filedistribution.FileDistributorService; import java.io.Serializable; import java.util.ArrayList; @@ -229,23 +228,13 @@ public class Admin extends AbstractConfigProducer implements Serializable { FileDistributionConfigProvider configProvider = new FileDistributionConfigProvider(fileDistributor, - fileDistribution.getOptions(), host == deployHost, host.getHost()); - if (fileDistribution.getOptions().disableFiledistributor()) { - DummyFileDistributionConfigProducer dummyFileDistributionConfigProducer = - new DummyFileDistributionConfigProducer(fileDistribution, - host.getHost().getHostname(), - configProvider); - fileDistribution.addFileDistributionConfigProducer(host.getHost(), dummyFileDistributionConfigProducer); - } else { - FileDistributorService fds = new FileDistributorService(fileDistribution, - host.getHost().getHostname(), - configProvider); - fds.setHostResource(host); - fds.initService(); - fileDistribution.addFileDistributionConfigProducer(host.getHost(), fds); - } + DummyFileDistributionConfigProducer dummyFileDistributionConfigProducer = + new DummyFileDistributionConfigProducer(fileDistribution, + host.getHost().getHostname(), + configProvider); + fileDistribution.addFileDistributionConfigProducer(host.getHost(), dummyFileDistributionConfigProducer); } private boolean deployHostIsMissing(HostResource deployHost) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/FileDistributionOptions.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/FileDistributionOptions.java index 8b39a1ecb94..22855709208 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/FileDistributionOptions.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/FileDistributionOptions.java @@ -12,10 +12,6 @@ import com.yahoo.cloud.config.filedistribution.FiledistributorConfig; */ public class FileDistributionOptions implements FiledistributorConfig.Producer { - public static FileDistributionOptions defaultOptions() { - return new FileDistributionOptions(); - } - private FileDistributionOptions() { } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java index 981ea1a9829..12ef85faaef 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java @@ -113,6 +113,8 @@ public class VespaMetricSet { metrics.add(new Metric("jdisc.deactivated_containers.total.last")); metrics.add(new Metric("jdisc.deactivated_containers.with_retained_refs.last")); + metrics.add(new Metric("athenz-tenant-cert.expiry.seconds.last", "athenz-tenant-cert.expiry.seconds")); + return metrics; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminBuilderBase.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminBuilderBase.java index 83e905929d2..64cbcaafd9f 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminBuilderBase.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminBuilderBase.java @@ -37,16 +37,14 @@ public abstract class DomAdminBuilderBase extends VespaDomBuilder.DomConfigProdu private final ApplicationType applicationType; private final List<ConfigServerSpec> configServerSpecs; private final FileRegistry fileRegistry; - private final boolean disableFiledistributor; protected final boolean multitenant; DomAdminBuilderBase(ApplicationType applicationType, FileRegistry fileRegistry, boolean multitenant, - List<ConfigServerSpec> configServerSpecs, boolean disableFiledistributor) { + List<ConfigServerSpec> configServerSpecs) { this.applicationType = applicationType; this.fileRegistry = fileRegistry; this.multitenant = multitenant; this.configServerSpecs = configServerSpecs; - this.disableFiledistributor = disableFiledistributor; } List<Configserver> getConfigServersFromSpec(AbstractConfigProducer parent) { @@ -71,7 +69,7 @@ public abstract class DomAdminBuilderBase extends VespaDomBuilder.DomConfigProdu .buildMetrics(XML.getChild(adminElement, "metrics")); Map<String, MetricsConsumer> legacyMetricsConsumers = DomMetricBuilderHelper .buildMetricsConsumers(XML.getChild(adminElement, "metric-consumers")); - FileDistributionConfigProducer fileDistributionConfigProducer = getFileDistributionConfigProducer(parent, adminElement); + FileDistributionConfigProducer fileDistributionConfigProducer = getFileDistributionConfigProducer(parent); Admin admin = new Admin(parent, monitoring, metrics, legacyMetricsConsumers, multitenant, fileDistributionConfigProducer); doBuildAdmin(admin, adminElement); @@ -80,11 +78,8 @@ public abstract class DomAdminBuilderBase extends VespaDomBuilder.DomConfigProdu return admin; } - private FileDistributionConfigProducer getFileDistributionConfigProducer(AbstractConfigProducer parent, Element adminElement) { - FileDistributionOptions fileDistributionOptions = FileDistributionOptions.defaultOptions(); - fileDistributionOptions.disableFiledistributor(disableFiledistributor); - fileDistributionOptions = new DomFileDistributionOptionsBuilder(fileDistributionOptions).build(XML.getChild(adminElement, "filedistribution")); - return new FileDistributionConfigProducer.Builder(fileDistributionOptions).build(parent, fileRegistry, configServerSpecs); + private FileDistributionConfigProducer getFileDistributionConfigProducer(AbstractConfigProducer parent) { + return new FileDistributionConfigProducer.Builder().build(parent, fileRegistry, configServerSpecs); } private Element getChildWithFallback(Element parent, String childName, String alternativeChildName) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2Builder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2Builder.java index d966f3b49f6..da8c73c0958 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2Builder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2Builder.java @@ -37,9 +37,8 @@ public class DomAdminV2Builder extends DomAdminBuilderBase { public DomAdminV2Builder(ConfigModelContext.ApplicationType applicationType, FileRegistry fileRegistry, boolean multitenant, - List<ConfigServerSpec> configServerSpecs, - boolean disableFiledistributor) { - super(applicationType, fileRegistry, multitenant, configServerSpecs, disableFiledistributor); + List<ConfigServerSpec> configServerSpecs) { + super(applicationType, fileRegistry, multitenant, configServerSpecs); } @Override diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java index 1f7c1ba676c..cc9957144f3 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java @@ -32,9 +32,9 @@ public class DomAdminV4Builder extends DomAdminBuilderBase { private final ConfigModelContext context; public DomAdminV4Builder(ConfigModelContext context, boolean multitenant, List<ConfigServerSpec> configServerSpecs, - Collection<ContainerModel> containerModels, boolean disableFiledistributor) { + Collection<ContainerModel> containerModels) { super(context.getApplicationType(), context.getDeployState().getFileRegistry(), multitenant, - configServerSpecs, disableFiledistributor); + configServerSpecs); this.containerModels = containerModels; this.context = context; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomFileDistributionOptionsBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomFileDistributionOptionsBuilder.java deleted file mode 100644 index 9cbdd00899b..00000000000 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomFileDistributionOptionsBuilder.java +++ /dev/null @@ -1,53 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.model.builder.xml.dom; - -import com.yahoo.binaryprefix.BinaryScaledAmount; -import com.yahoo.text.XML; -import com.yahoo.vespa.model.admin.FileDistributionOptions; -import org.w3c.dom.Element; - -import java.util.Optional; - -/** - * Builds file distribution options. - * - * @author Tony Vaagenes - * @author hmusum - */ -public class DomFileDistributionOptionsBuilder { - private final FileDistributionOptions fileDistributionOptions; - - public DomFileDistributionOptionsBuilder(FileDistributionOptions fileDistributionOptions) { - this.fileDistributionOptions = fileDistributionOptions; - } - - private static void throwExceptionForElementInFileDistribution(String subElement, String reason) { - throw new RuntimeException("In element '" + subElement + "' contained in 'filedistribution': " + reason); - } - - private Optional<BinaryScaledAmount> getAmount(String name, Element fileDistributionElement) { - Element optionElement = XML.getChild(fileDistributionElement, name); - try { - if (optionElement != null) { - String valueString = XML.getValue(optionElement); - return Optional.of(BinaryScaledAmountParser.parse(valueString)); - } - } catch (NumberFormatException e) { - throwExceptionForElementInFileDistribution(name, "Expected a valid number. (Message = " + e.getMessage() + ")."); - } - return Optional.empty(); - } - - public FileDistributionOptions build(Element fileDistributionElement) { - if (fileDistributionElement != null) { - getAmount("uploadbitrate", fileDistributionElement).ifPresent(fileDistributionOptions::uploadBitRate); - getAmount("downloadbitrate", fileDistributionElement).ifPresent(fileDistributionOptions::downloadBitRate); - Element disable = XML.getChild(fileDistributionElement, "disabled"); - if (disable == null) disable = XML.getChild(fileDistributionElement, "disableFiledistributor"); - if (disable != null) { - fileDistributionOptions.disableFiledistributor(Boolean.valueOf(XML.getValue(disable))); - } - } - return fileDistributionOptions; - } -} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/DummyFileDistributionConfigProducer.java b/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/DummyFileDistributionConfigProducer.java index 1b1e66594d1..d4993d01d1c 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/DummyFileDistributionConfigProducer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/DummyFileDistributionConfigProducer.java @@ -1,7 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.filedistribution; -import com.yahoo.cloud.config.filedistribution.FiledistributorConfig; import com.yahoo.cloud.config.filedistribution.FiledistributorrpcConfig; import com.yahoo.cloud.config.filedistribution.FilereferencesConfig; import com.yahoo.config.model.producer.AbstractConfigProducer; @@ -12,7 +11,6 @@ import com.yahoo.config.model.producer.AbstractConfigProducer; * Dummy file distribution config producer, needed for serving file distribution config when there is no FiledistributorService. */ public class DummyFileDistributionConfigProducer extends AbstractConfigProducer implements - FiledistributorConfig.Producer, FiledistributorrpcConfig.Producer, FilereferencesConfig.Producer { @@ -26,11 +24,6 @@ public class DummyFileDistributionConfigProducer extends AbstractConfigProducer } @Override - public void getConfig(FiledistributorConfig.Builder builder) { - configProvider.getConfig(builder); - } - - @Override public void getConfig(FiledistributorrpcConfig.Builder builder) { configProvider.getConfig(builder); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributionConfigProducer.java b/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributionConfigProducer.java index 85b937fee19..9441b787b09 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributionConfigProducer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributionConfigProducer.java @@ -5,7 +5,6 @@ import com.yahoo.config.application.api.FileRegistry; import com.yahoo.config.model.api.ConfigServerSpec; import com.yahoo.config.model.producer.AbstractConfigProducer; import com.yahoo.vespa.model.Host; -import com.yahoo.vespa.model.admin.FileDistributionOptions; import java.util.IdentityHashMap; import java.util.List; @@ -20,37 +19,25 @@ public class FileDistributionConfigProducer extends AbstractConfigProducer { private final Map<Host, AbstractConfigProducer> fileDistributionConfigProducers = new IdentityHashMap<>(); private final FileDistributor fileDistributor; - private final FileDistributionOptions options; - private FileDistributionConfigProducer(AbstractConfigProducer parent, FileDistributor fileDistributor, FileDistributionOptions options) { + private FileDistributionConfigProducer(AbstractConfigProducer parent, FileDistributor fileDistributor) { super(parent, "filedistribution"); this.fileDistributor = fileDistributor; - this.options = options; } public FileDistributor getFileDistributor() { return fileDistributor; } - public FileDistributionOptions getOptions() { - return options; - } - public void addFileDistributionConfigProducer(Host host, AbstractConfigProducer fileDistributionConfigProducer) { fileDistributionConfigProducers.put(host, fileDistributionConfigProducer); } public static class Builder { - private final FileDistributionOptions options; - - public Builder(FileDistributionOptions fileDistributionOptions) { - this.options = fileDistributionOptions; - } - public FileDistributionConfigProducer build(AbstractConfigProducer ancestor, FileRegistry fileRegistry, List<ConfigServerSpec> configServerSpec) { FileDistributor fileDistributor = new FileDistributor(fileRegistry, configServerSpec); - return new FileDistributionConfigProducer(ancestor, fileDistributor, options); + return new FileDistributionConfigProducer(ancestor, fileDistributor); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributionConfigProvider.java b/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributionConfigProvider.java index a78eb1f74f9..ffba56fa17f 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributionConfigProvider.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributionConfigProvider.java @@ -1,45 +1,31 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.filedistribution; -import com.yahoo.cloud.config.filedistribution.FiledistributorConfig; import com.yahoo.cloud.config.filedistribution.FiledistributorrpcConfig; import com.yahoo.cloud.config.filedistribution.FilereferencesConfig; import com.yahoo.config.FileReference; -import com.yahoo.config.model.api.FileDistribution; import com.yahoo.vespa.model.ConfigProxy; import com.yahoo.vespa.model.Host; -import com.yahoo.vespa.model.admin.FileDistributionOptions; import java.util.Collection; public class FileDistributionConfigProvider { private final FileDistributor fileDistributor; - private final FileDistributionOptions fileDistributionOptions; private final boolean sendAllFiles; private final Host host; public FileDistributionConfigProvider(FileDistributor fileDistributor, - FileDistributionOptions fileDistributionOptions, boolean sendAllFiles, Host host) { this.fileDistributor = fileDistributor; - this.fileDistributionOptions = fileDistributionOptions; this.sendAllFiles = sendAllFiles; this.host = host; } - public void getConfig(FiledistributorConfig.Builder builder) { - fileDistributionOptions.getConfig(builder); - builder.torrentport(FileDistributorService.BASEPORT + 1); - builder.stateport(FileDistributorService.BASEPORT + 2); - builder.hostname(host.getHostname()); - builder.filedbpath(FileDistribution.getDefaultFileDBPath().toString()); - } - public void getConfig(FiledistributorrpcConfig.Builder builder) { // If disabled config proxy should act as file distributor, so use config proxy port - int port = (fileDistributionOptions.disableFiledistributor()) ? ConfigProxy.BASEPORT : FileDistributorService.BASEPORT; + int port = ConfigProxy.BASEPORT; builder.connectionspec("tcp/" + host.getHostname() + ":" + port); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributorService.java b/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributorService.java deleted file mode 100644 index 986334125d2..00000000000 --- a/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributorService.java +++ /dev/null @@ -1,73 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.model.filedistribution; - -import com.yahoo.cloud.config.filedistribution.FiledistributorConfig; -import com.yahoo.cloud.config.filedistribution.FiledistributorrpcConfig; -import com.yahoo.cloud.config.filedistribution.FilereferencesConfig; -import com.yahoo.config.model.producer.AbstractConfigProducer; -import com.yahoo.vespa.model.AbstractService; - -/** - * @author Tony Vaagenes - * - * Config is produced by {@link FileDistributionConfigProvider} - */ -public class FileDistributorService extends AbstractService implements - FiledistributorConfig.Producer, - FiledistributorrpcConfig.Producer, - FilereferencesConfig.Producer { - - final static int BASEPORT = 19092; - - private final FileDistributionConfigProvider configProvider; - - public FileDistributorService(AbstractConfigProducer parent, String hostname, FileDistributionConfigProvider configProvider) { - super(parent, hostname); - this.configProvider = configProvider; - portsMeta.on(0).tag("rpc"); - portsMeta.on(1).tag("torrent"); - portsMeta.on(2).tag("http").tag("state"); - setProp("clustertype", "filedistribution"); - setProp("clustername", "admin"); - } - - @Override - public String getStartupCommand() { - return "exec $ROOT/sbin/vespa-filedistributor" + " --configid " + getConfigId(); - } - - @Override - public boolean getAutostartFlag() { - return true; - } - - @Override - public boolean getAutorestartFlag() { - return true; - } - - @Override - public int getPortCount() { - return 3; - } - - @Override - public int getWantedPort() { - return BASEPORT; - } - - @Override - public void getConfig(FiledistributorConfig.Builder builder) { - configProvider.getConfig(builder); - } - - @Override - public void getConfig(FiledistributorrpcConfig.Builder builder) { - configProvider.getConfig(builder); - } - - @Override - public void getConfig(FilereferencesConfig.Builder builder) { - configProvider.getConfig(builder); - } -} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java index 8420a1f4b91..df5c2c9c173 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java @@ -21,7 +21,6 @@ import com.yahoo.vespa.model.builder.xml.dom.VespaDomBuilder; import com.yahoo.vespa.model.content.ContentNode; import com.yahoo.vespa.model.filedistribution.DummyFileDistributionConfigProducer; import com.yahoo.vespa.model.filedistribution.FileDistributionConfigProducer; -import com.yahoo.vespa.model.filedistribution.FileDistributorService; import org.w3c.dom.Element; import java.util.HashMap; @@ -230,11 +229,7 @@ public class SearchNode extends AbstractService implements FileDistributionConfigProducer fileDistribution = getRoot().getFileDistributionConfigProducer(); if (fileDistribution != null) { AbstractConfigProducer configProducer = fileDistribution.getConfigProducer(getHost()); - // TODO: Hack, will be fixed when FileDistributorService is gone - if (configProducer instanceof DummyFileDistributionConfigProducer) - ((DummyFileDistributionConfigProducer) configProducer).getConfig(builder); - else - ((FileDistributorService) configProducer).getConfig(builder); + ((DummyFileDistributionConfigProducer) configProducer).getConfig(builder); } } diff --git a/config-model/src/test/integration/tensorflow/models/mnist/saved/saved_model.pbtxt b/config-model/src/test/integration/tensorflow/models/mnist/saved/saved_model.pbtxt index eb926836576..5528aa99401 100644 --- a/config-model/src/test/integration/tensorflow/models/mnist/saved/saved_model.pbtxt +++ b/config-model/src/test/integration/tensorflow/models/mnist/saved/saved_model.pbtxt @@ -883,54 +883,6 @@ meta_graphs { } } op { - name: "Selu" - input_arg { - name: "features" - type_attr: "T" - } - output_arg { - name: "activations" - type_attr: "T" - } - attr { - name: "T" - type: "type" - allowed_values { - list { - type: DT_HALF - type: DT_FLOAT - type: DT_DOUBLE - } - } - } - } - op { - name: "SeluGrad" - input_arg { - name: "gradients" - type_attr: "T" - } - input_arg { - name: "outputs" - type_attr: "T" - } - output_arg { - name: "backprops" - type_attr: "T" - } - attr { - name: "T" - type: "type" - allowed_values { - list { - type: DT_HALF - type: DT_FLOAT - type: DT_DOUBLE - } - } - } - } - op { name: "Shape" input_arg { name: "input" @@ -2476,8 +2428,72 @@ meta_graphs { } } node { - name: "dnn/hidden2/Selu" - op: "Selu" + name: "dnn/hidden2/Const" + op: "Const" + attr { + key: "_output_shapes" + value { + list { + shape { + dim { + size: 1 + } + } + } + } + } + attr { + key: "dtype" + value { + type: DT_FLOAT + } + } + attr { + key: "value" + value { + tensor { + dtype: DT_FLOAT + tensor_shape { + dim { + size: 1 + } + } + float_val: 0.009999999776482582 + } + } + } + } + node { + name: "dnn/hidden2/mul" + op: "Mul" + input: "dnn/hidden2/Const" + input: "dnn/hidden2/add" + attr { + key: "T" + value { + type: DT_FLOAT + } + } + attr { + key: "_output_shapes" + value { + list { + shape { + dim { + size: -1 + } + dim { + size: 100 + } + } + } + } + } + } + node { + name: "dnn/hidden2/Maximum" + op: "Maximum" + input: "dnn/hidden2/mul" input: "dnn/hidden2/add" attr { key: "T" @@ -2984,7 +3000,7 @@ meta_graphs { node { name: "dnn/outputs/MatMul" op: "MatMul" - input: "dnn/hidden2/Selu" + input: "dnn/hidden2/Maximum" input: "dnn/outputs/weights/read" attr { key: "T" @@ -4258,7 +4274,7 @@ meta_graphs { node { name: "train/gradients/dnn/outputs/MatMul_grad/MatMul_1" op: "MatMul" - input: "dnn/hidden2/Selu" + input: "dnn/hidden2/Maximum" input: "train/gradients/dnn/outputs/add_grad/tuple/control_dependency" attr { key: "T" @@ -4371,10 +4387,127 @@ meta_graphs { } } node { - name: "train/gradients/dnn/hidden2/Selu_grad/SeluGrad" - op: "SeluGrad" + name: "train/gradients/dnn/hidden2/Maximum_grad/Shape" + op: "Shape" + input: "dnn/hidden2/mul" + attr { + key: "T" + value { + type: DT_FLOAT + } + } + attr { + key: "_output_shapes" + value { + list { + shape { + dim { + size: 2 + } + } + } + } + } + attr { + key: "out_type" + value { + type: DT_INT32 + } + } + } + node { + name: "train/gradients/dnn/hidden2/Maximum_grad/Shape_1" + op: "Shape" + input: "dnn/hidden2/add" + attr { + key: "T" + value { + type: DT_FLOAT + } + } + attr { + key: "_output_shapes" + value { + list { + shape { + dim { + size: 2 + } + } + } + } + } + attr { + key: "out_type" + value { + type: DT_INT32 + } + } + } + node { + name: "train/gradients/dnn/hidden2/Maximum_grad/Shape_2" + op: "Shape" input: "train/gradients/dnn/outputs/MatMul_grad/tuple/control_dependency" - input: "dnn/hidden2/Selu" + attr { + key: "T" + value { + type: DT_FLOAT + } + } + attr { + key: "_output_shapes" + value { + list { + shape { + dim { + size: 2 + } + } + } + } + } + attr { + key: "out_type" + value { + type: DT_INT32 + } + } + } + node { + name: "train/gradients/dnn/hidden2/Maximum_grad/zeros/Const" + op: "Const" + attr { + key: "_output_shapes" + value { + list { + shape { + } + } + } + } + attr { + key: "dtype" + value { + type: DT_FLOAT + } + } + attr { + key: "value" + value { + tensor { + dtype: DT_FLOAT + tensor_shape { + } + float_val: 0.0 + } + } + } + } + node { + name: "train/gradients/dnn/hidden2/Maximum_grad/zeros" + op: "Fill" + input: "train/gradients/dnn/hidden2/Maximum_grad/Shape_2" + input: "train/gradients/dnn/hidden2/Maximum_grad/zeros/Const" attr { key: "T" value { @@ -4398,6 +4531,721 @@ meta_graphs { } } node { + name: "train/gradients/dnn/hidden2/Maximum_grad/GreaterEqual" + op: "GreaterEqual" + input: "dnn/hidden2/mul" + input: "dnn/hidden2/add" + attr { + key: "T" + value { + type: DT_FLOAT + } + } + attr { + key: "_output_shapes" + value { + list { + shape { + dim { + size: -1 + } + dim { + size: 100 + } + } + } + } + } + } + node { + name: "train/gradients/dnn/hidden2/Maximum_grad/BroadcastGradientArgs" + op: "BroadcastGradientArgs" + input: "train/gradients/dnn/hidden2/Maximum_grad/Shape" + input: "train/gradients/dnn/hidden2/Maximum_grad/Shape_1" + attr { + key: "T" + value { + type: DT_INT32 + } + } + attr { + key: "_output_shapes" + value { + list { + shape { + dim { + size: -1 + } + } + shape { + dim { + size: -1 + } + } + } + } + } + } + node { + name: "train/gradients/dnn/hidden2/Maximum_grad/Select" + op: "Select" + input: "train/gradients/dnn/hidden2/Maximum_grad/GreaterEqual" + input: "train/gradients/dnn/outputs/MatMul_grad/tuple/control_dependency" + input: "train/gradients/dnn/hidden2/Maximum_grad/zeros" + attr { + key: "T" + value { + type: DT_FLOAT + } + } + attr { + key: "_output_shapes" + value { + list { + shape { + dim { + size: -1 + } + dim { + size: 100 + } + } + } + } + } + } + node { + name: "train/gradients/dnn/hidden2/Maximum_grad/Select_1" + op: "Select" + input: "train/gradients/dnn/hidden2/Maximum_grad/GreaterEqual" + input: "train/gradients/dnn/hidden2/Maximum_grad/zeros" + input: "train/gradients/dnn/outputs/MatMul_grad/tuple/control_dependency" + attr { + key: "T" + value { + type: DT_FLOAT + } + } + attr { + key: "_output_shapes" + value { + list { + shape { + dim { + size: -1 + } + dim { + size: 100 + } + } + } + } + } + } + node { + name: "train/gradients/dnn/hidden2/Maximum_grad/Sum" + op: "Sum" + input: "train/gradients/dnn/hidden2/Maximum_grad/Select" + input: "train/gradients/dnn/hidden2/Maximum_grad/BroadcastGradientArgs" + attr { + key: "T" + value { + type: DT_FLOAT + } + } + attr { + key: "Tidx" + value { + type: DT_INT32 + } + } + attr { + key: "_output_shapes" + value { + list { + shape { + unknown_rank: true + } + } + } + } + attr { + key: "keep_dims" + value { + b: false + } + } + } + node { + name: "train/gradients/dnn/hidden2/Maximum_grad/Reshape" + op: "Reshape" + input: "train/gradients/dnn/hidden2/Maximum_grad/Sum" + input: "train/gradients/dnn/hidden2/Maximum_grad/Shape" + attr { + key: "T" + value { + type: DT_FLOAT + } + } + attr { + key: "Tshape" + value { + type: DT_INT32 + } + } + attr { + key: "_output_shapes" + value { + list { + shape { + dim { + size: -1 + } + dim { + size: 100 + } + } + } + } + } + } + node { + name: "train/gradients/dnn/hidden2/Maximum_grad/Sum_1" + op: "Sum" + input: "train/gradients/dnn/hidden2/Maximum_grad/Select_1" + input: "train/gradients/dnn/hidden2/Maximum_grad/BroadcastGradientArgs:1" + attr { + key: "T" + value { + type: DT_FLOAT + } + } + attr { + key: "Tidx" + value { + type: DT_INT32 + } + } + attr { + key: "_output_shapes" + value { + list { + shape { + unknown_rank: true + } + } + } + } + attr { + key: "keep_dims" + value { + b: false + } + } + } + node { + name: "train/gradients/dnn/hidden2/Maximum_grad/Reshape_1" + op: "Reshape" + input: "train/gradients/dnn/hidden2/Maximum_grad/Sum_1" + input: "train/gradients/dnn/hidden2/Maximum_grad/Shape_1" + attr { + key: "T" + value { + type: DT_FLOAT + } + } + attr { + key: "Tshape" + value { + type: DT_INT32 + } + } + attr { + key: "_output_shapes" + value { + list { + shape { + dim { + size: -1 + } + dim { + size: 100 + } + } + } + } + } + } + node { + name: "train/gradients/dnn/hidden2/Maximum_grad/tuple/group_deps" + op: "NoOp" + input: "^train/gradients/dnn/hidden2/Maximum_grad/Reshape" + input: "^train/gradients/dnn/hidden2/Maximum_grad/Reshape_1" + } + node { + name: "train/gradients/dnn/hidden2/Maximum_grad/tuple/control_dependency" + op: "Identity" + input: "train/gradients/dnn/hidden2/Maximum_grad/Reshape" + input: "^train/gradients/dnn/hidden2/Maximum_grad/tuple/group_deps" + attr { + key: "T" + value { + type: DT_FLOAT + } + } + attr { + key: "_class" + value { + list { + s: "loc:@train/gradients/dnn/hidden2/Maximum_grad/Reshape" + } + } + } + attr { + key: "_output_shapes" + value { + list { + shape { + dim { + size: -1 + } + dim { + size: 100 + } + } + } + } + } + } + node { + name: "train/gradients/dnn/hidden2/Maximum_grad/tuple/control_dependency_1" + op: "Identity" + input: "train/gradients/dnn/hidden2/Maximum_grad/Reshape_1" + input: "^train/gradients/dnn/hidden2/Maximum_grad/tuple/group_deps" + attr { + key: "T" + value { + type: DT_FLOAT + } + } + attr { + key: "_class" + value { + list { + s: "loc:@train/gradients/dnn/hidden2/Maximum_grad/Reshape_1" + } + } + } + attr { + key: "_output_shapes" + value { + list { + shape { + dim { + size: -1 + } + dim { + size: 100 + } + } + } + } + } + } + node { + name: "train/gradients/dnn/hidden2/mul_grad/Shape" + op: "Const" + attr { + key: "_output_shapes" + value { + list { + shape { + dim { + size: 1 + } + } + } + } + } + attr { + key: "dtype" + value { + type: DT_INT32 + } + } + attr { + key: "value" + value { + tensor { + dtype: DT_INT32 + tensor_shape { + dim { + size: 1 + } + } + int_val: 1 + } + } + } + } + node { + name: "train/gradients/dnn/hidden2/mul_grad/Shape_1" + op: "Shape" + input: "dnn/hidden2/add" + attr { + key: "T" + value { + type: DT_FLOAT + } + } + attr { + key: "_output_shapes" + value { + list { + shape { + dim { + size: 2 + } + } + } + } + } + attr { + key: "out_type" + value { + type: DT_INT32 + } + } + } + node { + name: "train/gradients/dnn/hidden2/mul_grad/BroadcastGradientArgs" + op: "BroadcastGradientArgs" + input: "train/gradients/dnn/hidden2/mul_grad/Shape" + input: "train/gradients/dnn/hidden2/mul_grad/Shape_1" + attr { + key: "T" + value { + type: DT_INT32 + } + } + attr { + key: "_output_shapes" + value { + list { + shape { + dim { + size: -1 + } + } + shape { + dim { + size: -1 + } + } + } + } + } + } + node { + name: "train/gradients/dnn/hidden2/mul_grad/mul" + op: "Mul" + input: "train/gradients/dnn/hidden2/Maximum_grad/tuple/control_dependency" + input: "dnn/hidden2/add" + attr { + key: "T" + value { + type: DT_FLOAT + } + } + attr { + key: "_output_shapes" + value { + list { + shape { + dim { + size: -1 + } + dim { + size: 100 + } + } + } + } + } + } + node { + name: "train/gradients/dnn/hidden2/mul_grad/Sum" + op: "Sum" + input: "train/gradients/dnn/hidden2/mul_grad/mul" + input: "train/gradients/dnn/hidden2/mul_grad/BroadcastGradientArgs" + attr { + key: "T" + value { + type: DT_FLOAT + } + } + attr { + key: "Tidx" + value { + type: DT_INT32 + } + } + attr { + key: "_output_shapes" + value { + list { + shape { + unknown_rank: true + } + } + } + } + attr { + key: "keep_dims" + value { + b: false + } + } + } + node { + name: "train/gradients/dnn/hidden2/mul_grad/Reshape" + op: "Reshape" + input: "train/gradients/dnn/hidden2/mul_grad/Sum" + input: "train/gradients/dnn/hidden2/mul_grad/Shape" + attr { + key: "T" + value { + type: DT_FLOAT + } + } + attr { + key: "Tshape" + value { + type: DT_INT32 + } + } + attr { + key: "_output_shapes" + value { + list { + shape { + dim { + size: 1 + } + } + } + } + } + } + node { + name: "train/gradients/dnn/hidden2/mul_grad/mul_1" + op: "Mul" + input: "dnn/hidden2/Const" + input: "train/gradients/dnn/hidden2/Maximum_grad/tuple/control_dependency" + attr { + key: "T" + value { + type: DT_FLOAT + } + } + attr { + key: "_output_shapes" + value { + list { + shape { + dim { + size: -1 + } + dim { + size: 100 + } + } + } + } + } + } + node { + name: "train/gradients/dnn/hidden2/mul_grad/Sum_1" + op: "Sum" + input: "train/gradients/dnn/hidden2/mul_grad/mul_1" + input: "train/gradients/dnn/hidden2/mul_grad/BroadcastGradientArgs:1" + attr { + key: "T" + value { + type: DT_FLOAT + } + } + attr { + key: "Tidx" + value { + type: DT_INT32 + } + } + attr { + key: "_output_shapes" + value { + list { + shape { + unknown_rank: true + } + } + } + } + attr { + key: "keep_dims" + value { + b: false + } + } + } + node { + name: "train/gradients/dnn/hidden2/mul_grad/Reshape_1" + op: "Reshape" + input: "train/gradients/dnn/hidden2/mul_grad/Sum_1" + input: "train/gradients/dnn/hidden2/mul_grad/Shape_1" + attr { + key: "T" + value { + type: DT_FLOAT + } + } + attr { + key: "Tshape" + value { + type: DT_INT32 + } + } + attr { + key: "_output_shapes" + value { + list { + shape { + dim { + size: -1 + } + dim { + size: 100 + } + } + } + } + } + } + node { + name: "train/gradients/dnn/hidden2/mul_grad/tuple/group_deps" + op: "NoOp" + input: "^train/gradients/dnn/hidden2/mul_grad/Reshape" + input: "^train/gradients/dnn/hidden2/mul_grad/Reshape_1" + } + node { + name: "train/gradients/dnn/hidden2/mul_grad/tuple/control_dependency" + op: "Identity" + input: "train/gradients/dnn/hidden2/mul_grad/Reshape" + input: "^train/gradients/dnn/hidden2/mul_grad/tuple/group_deps" + attr { + key: "T" + value { + type: DT_FLOAT + } + } + attr { + key: "_class" + value { + list { + s: "loc:@train/gradients/dnn/hidden2/mul_grad/Reshape" + } + } + } + attr { + key: "_output_shapes" + value { + list { + shape { + dim { + size: 1 + } + } + } + } + } + } + node { + name: "train/gradients/dnn/hidden2/mul_grad/tuple/control_dependency_1" + op: "Identity" + input: "train/gradients/dnn/hidden2/mul_grad/Reshape_1" + input: "^train/gradients/dnn/hidden2/mul_grad/tuple/group_deps" + attr { + key: "T" + value { + type: DT_FLOAT + } + } + attr { + key: "_class" + value { + list { + s: "loc:@train/gradients/dnn/hidden2/mul_grad/Reshape_1" + } + } + } + attr { + key: "_output_shapes" + value { + list { + shape { + dim { + size: -1 + } + dim { + size: 100 + } + } + } + } + } + } + node { + name: "train/gradients/AddN" + op: "AddN" + input: "train/gradients/dnn/hidden2/Maximum_grad/tuple/control_dependency_1" + input: "train/gradients/dnn/hidden2/mul_grad/tuple/control_dependency_1" + attr { + key: "N" + value { + i: 2 + } + } + attr { + key: "T" + value { + type: DT_FLOAT + } + } + attr { + key: "_class" + value { + list { + s: "loc:@train/gradients/dnn/hidden2/Maximum_grad/Reshape_1" + } + } + } + attr { + key: "_output_shapes" + value { + list { + shape { + dim { + size: -1 + } + dim { + size: 100 + } + } + } + } + } + } + node { name: "train/gradients/dnn/hidden2/add_grad/Shape" op: "Shape" input: "dnn/hidden2/MatMul" @@ -4494,7 +5342,7 @@ meta_graphs { node { name: "train/gradients/dnn/hidden2/add_grad/Sum" op: "Sum" - input: "train/gradients/dnn/hidden2/Selu_grad/SeluGrad" + input: "train/gradients/AddN" input: "train/gradients/dnn/hidden2/add_grad/BroadcastGradientArgs" attr { key: "T" @@ -4561,7 +5409,7 @@ meta_graphs { node { name: "train/gradients/dnn/hidden2/add_grad/Sum_1" op: "Sum" - input: "train/gradients/dnn/hidden2/Selu_grad/SeluGrad" + input: "train/gradients/AddN" input: "train/gradients/dnn/hidden2/add_grad/BroadcastGradientArgs:1" attr { key: "T" @@ -5659,7 +6507,7 @@ meta_graphs { } } node { - name: "train/gradients/AddN" + name: "train/gradients/AddN_1" op: "AddN" input: "train/gradients/dnn/hidden1/Maximum_grad/tuple/control_dependency_1" input: "train/gradients/dnn/hidden1/mul_grad/tuple/control_dependency_1" @@ -5796,7 +6644,7 @@ meta_graphs { node { name: "train/gradients/dnn/hidden1/add_grad/Sum" op: "Sum" - input: "train/gradients/AddN" + input: "train/gradients/AddN_1" input: "train/gradients/dnn/hidden1/add_grad/BroadcastGradientArgs" attr { key: "T" @@ -5863,7 +6711,7 @@ meta_graphs { node { name: "train/gradients/dnn/hidden1/add_grad/Sum_1" op: "Sum" - input: "train/gradients/AddN" + input: "train/gradients/AddN_1" input: "train/gradients/dnn/hidden1/add_grad/BroadcastGradientArgs:1" attr { key: "T" @@ -6703,7 +7551,7 @@ meta_graphs { dtype: DT_STRING tensor_shape { } - string_val: "_temp_de3cfc5e8e7e4734ae221577e8fd36a2/part" + string_val: "_temp_af8f34e037d9459f96200d29e33a7078/part" } } } diff --git a/config-model/src/test/integration/tensorflow/models/mnist/saved/variables/variables.data-00000-of-00001 b/config-model/src/test/integration/tensorflow/models/mnist/saved/variables/variables.data-00000-of-00001 Binary files differindex a7ca01888c7..ed4af6c0f8c 100644 --- a/config-model/src/test/integration/tensorflow/models/mnist/saved/variables/variables.data-00000-of-00001 +++ b/config-model/src/test/integration/tensorflow/models/mnist/saved/variables/variables.data-00000-of-00001 diff --git a/config-model/src/test/integration/tensorflow/models/mnist/saved/variables/variables.index b/config-model/src/test/integration/tensorflow/models/mnist/saved/variables/variables.index Binary files differindex 7989c109a3a..c877b02b42a 100644 --- a/config-model/src/test/integration/tensorflow/models/mnist/saved/variables/variables.index +++ b/config-model/src/test/integration/tensorflow/models/mnist/saved/variables/variables.index diff --git a/config-model/src/test/integration/tensorflow/models/mnist/simple_mnist.py b/config-model/src/test/integration/tensorflow/models/mnist/simple_mnist.py index 26529f67919..3d390fed9a1 100644 --- a/config-model/src/test/integration/tensorflow/models/mnist/simple_mnist.py +++ b/config-model/src/test/integration/tensorflow/models/mnist/simple_mnist.py @@ -47,10 +47,12 @@ def neuron_layer(X, n_neurons, name, activation=None): def leaky_relu(z, name=None): return tf.maximum(0.01 * z, z, name=name) +def leaky_relu_with_small_constant(z, name=None): + return tf.maximum(tf.constant(0.01, shape=[1]) * z, z, name=name) with tf.name_scope("dnn"): hidden1 = neuron_layer(input, n_hidden1, name="hidden1", activation=leaky_relu) - hidden2 = neuron_layer(hidden1, n_hidden2, name="hidden2", activation=tf.nn.selu) + hidden2 = neuron_layer(hidden1, n_hidden2, name="hidden2", activation=leaky_relu_with_small_constant) logits = neuron_layer(hidden2, n_outputs, name="outputs") #, activation=tf.nn.sigmoid) with tf.name_scope("loss"): @@ -94,4 +96,4 @@ with tf.Session() as sess: signature_def_map={'serving_default':signature}) builder.save(as_text=True) -file_writer.close()
\ No newline at end of file +file_writer.close() diff --git a/config-model/src/test/java/com/yahoo/config/model/deploy/SystemModelTestCase.java b/config-model/src/test/java/com/yahoo/config/model/deploy/SystemModelTestCase.java index 74eb4caf529..7a2c702a103 100644 --- a/config-model/src/test/java/com/yahoo/config/model/deploy/SystemModelTestCase.java +++ b/config-model/src/test/java/com/yahoo/config/model/deploy/SystemModelTestCase.java @@ -115,8 +115,8 @@ public class SystemModelTestCase { assertEquals(host2, host3); // all three host aliases are for the same host, so the number of services should be 3 + 8 - // (3 simpleservices and logd, configproxy, config sentinel, admin server config server, slobrok, log server and file distribution) - assertEquals(10, host1.getServices().size()); + // (3 simpleservices and logd, configproxy, config sentinel, admin server config server, slobrok and log server) + assertEquals(9, host1.getServices().size()); assertNotNull(host1.getService("simpleservice")); assertNotNull(host1.getService("simpleservice2")); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTensorFlowTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTensorFlowTestCase.java index 83cc3ae418a..4693ac5cf4d 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTensorFlowTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTensorFlowTestCase.java @@ -8,6 +8,7 @@ import com.yahoo.io.GrowableByteBuffer; import com.yahoo.io.IOUtils; import com.yahoo.path.Path; import com.yahoo.search.query.profile.QueryProfileRegistry; +import com.yahoo.searchdefinition.RankProfile; import com.yahoo.searchdefinition.RankingConstant; import com.yahoo.searchdefinition.parser.ParseException; import com.yahoo.searchlib.rankingexpression.evaluation.Value; @@ -252,32 +253,32 @@ public class RankingExpressionWithTensorFlowTestCase { @Test public void testImportingFromStoredExpressionsWithSmallConstants() throws IOException { - final String expression = "join(rename(reduce(join(map(join(rename(reduce(join(join(join(constant(\"dnn_hidden1_mul_x\"), join(rename(reduce(join(input, rename(constant(\"dnn_hidden1_weights\"), (d0, d1), (d1, d3)), f(a,b)(a * b)), sum, d1), d3, d1), rename(constant(\"dnn_hidden1_bias\"), d0, d1), f(a,b)(a + b)), f(a,b)(a * b)), join(rename(reduce(join(input, rename(constant(\"dnn_hidden1_weights\"), (d0, d1), (d1, d3)), f(a,b)(a * b)), sum, d1), d3, d1), rename(constant(\"dnn_hidden1_bias\"), d0, d1), f(a,b)(a + b)), f(a,b)(max(a,b))), rename(constant(\"dnn_hidden2_weights\"), (d0, d1), (d1, d3)), f(a,b)(a * b)), sum, d1), d3, d1), rename(constant(\"dnn_hidden2_bias\"), d0, d1), f(a,b)(a + b)), f(a)(1.050701 * if (a >= 0, a, 1.673263 * (exp(a) - 1)))), rename(constant(\"dnn_outputs_weights\"), (d0, d1), (d1, d3)), f(a,b)(a * b)), sum, d1), d3, d1), rename(constant(\"dnn_outputs_bias\"), d0, d1), f(a,b)(a + b))"; + final String expression = "join(rename(reduce(join(join(join(rename(constant(\"dnn_hidden2_Const\"), d0, d1), join(rename(reduce(join(join(join(0.009999999776482582, join(rename(reduce(join(input, rename(constant(\"dnn_hidden1_weights\"), (d0, d1), (d1, d3)), f(a,b)(a * b)), sum, d1), d3, d1), rename(constant(\"dnn_hidden1_bias\"), d0, d1), f(a,b)(a + b)), f(a,b)(a * b)), join(rename(reduce(join(input, rename(constant(\"dnn_hidden1_weights\"), (d0, d1), (d1, d3)), f(a,b)(a * b)), sum, d1), d3, d1), rename(constant(\"dnn_hidden1_bias\"), d0, d1), f(a,b)(a + b)), f(a,b)(max(a,b))), rename(constant(\"dnn_hidden2_weights\"), (d0, d1), (d1, d3)), f(a,b)(a * b)), sum, d1), d3, d1), rename(constant(\"dnn_hidden2_bias\"), d0, d1), f(a,b)(a + b)), f(a,b)(a * b)), join(rename(reduce(join(join(join(0.009999999776482582, join(rename(reduce(join(input, rename(constant(\"dnn_hidden1_weights\"), (d0, d1), (d1, d3)), f(a,b)(a * b)), sum, d1), d3, d1), rename(constant(\"dnn_hidden1_bias\"), d0, d1), f(a,b)(a + b)), f(a,b)(a * b)), join(rename(reduce(join(input, rename(constant(\"dnn_hidden1_weights\"), (d0, d1), (d1, d3)), f(a,b)(a * b)), sum, d1), d3, d1), rename(constant(\"dnn_hidden1_bias\"), d0, d1), f(a,b)(a + b)), f(a,b)(max(a,b))), rename(constant(\"dnn_hidden2_weights\"), (d0, d1), (d1, d3)), f(a,b)(a * b)), sum, d1), d3, d1), rename(constant(\"dnn_hidden2_bias\"), d0, d1), f(a,b)(a + b)), f(a,b)(max(a,b))), rename(constant(\"dnn_outputs_weights\"), (d0, d1), (d1, d3)), f(a,b)(a * b)), sum, d1), d3, d1), rename(constant(\"dnn_outputs_bias\"), d0, d1), f(a,b)(a + b))"; StoringApplicationPackage application = new StoringApplicationPackage(applicationDir); RankProfileSearchFixture search = fixtureWith("tensor(d0[2],d1[784])(0.0)", - "tensorflow('mnist/saved')", - null, - null, - "input", - application); + "tensorflow('mnist/saved')", + null, + null, + "input", + application); search.assertFirstPhaseExpression(expression, "my_profile"); - assertSmallConstant("dnn_hidden1_mul_x", TensorType.empty, search); + assertSmallConstant("dnn_hidden2_Const", TensorType.fromSpec("tensor(d0[1])"), search); // At this point the expression is stored - copy application to another location which do not have a models dir Path storedApplicationDirectory = applicationDir.getParentPath().append("copy"); try { storedApplicationDirectory.toFile().mkdirs(); IOUtils.copyDirectory(applicationDir.append(ApplicationPackage.MODELS_GENERATED_DIR).toFile(), - storedApplicationDirectory.append(ApplicationPackage.MODELS_GENERATED_DIR).toFile()); + storedApplicationDirectory.append(ApplicationPackage.MODELS_GENERATED_DIR).toFile()); StoringApplicationPackage storedApplication = new StoringApplicationPackage(storedApplicationDirectory); RankProfileSearchFixture searchFromStored = fixtureWith("tensor(d0[2],d1[784])(0.0)", - "tensorflow('mnist/saved')", - null, - null, - "input", - storedApplication); + "tensorflow('mnist/saved')", + null, + null, + "input", + storedApplication); searchFromStored.assertFirstPhaseExpression(expression, "my_profile"); - assertSmallConstant("dnn_hidden1_mul_x", TensorType.empty, search); + assertSmallConstant("dnn_hidden2_Const", TensorType.fromSpec("tensor(d0[1])"), search); } finally { IOUtils.recursiveDeleteDir(storedApplicationDirectory.toFile()); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorTransformTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorTransformTestCase.java index c18cfcfe1aa..b001db69768 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorTransformTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorTransformTestCase.java @@ -64,7 +64,7 @@ public class TensorTransformTestCase extends SearchDefinitionTestCase { assertContainsExpression("min(attribute(tensor_field_1) * attribute(tensor_field_2),x)", "reduce(attribute(tensor_field_1)*attribute(tensor_field_2),min,x)"); assertContainsExpression("min(join(attribute(tensor_field_1),attribute(tensor_field_2),f(x,y)(x*y)),x)", "reduce(join(attribute(tensor_field_1),attribute(tensor_field_2),f(x,y)(x*y)),min,x)"); assertContainsExpression("min(join(tensor_field_1,tensor_field_2,f(x,y)(x*y)),x)", "min(join(tensor_field_1,tensor_field_2,f(x,y)(x*y)),x)"); // because tensor fields are not in attribute(...) - assertContainsExpression("min(join(attribute(tensor_field_1),backend_rank_feature,f(x,y)(x*y)),x)", "min(join(attribute(tensor_field_1),backend_rank_feature,f(x,y)(x*y)),x)"); + assertContainsExpression("min(join(attribute(tensor_field_1),backend_rank_feature,f(x,y)(x*y)),x)", "reduce(join(attribute(tensor_field_1),backend_rank_feature,f(x,y)(x*y)),min,x)"); } @Test diff --git a/config-model/src/test/java/com/yahoo/vespa/model/admin/AdminTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/admin/AdminTestCase.java index 223b50dd180..01e1a0ba9fa 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/admin/AdminTestCase.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/admin/AdminTestCase.java @@ -92,12 +92,11 @@ public class AdminTestCase { SentinelConfig.Builder b = new SentinelConfig.Builder(); vespaModel.getConfig(b, localhostConfigId); SentinelConfig sentinelConfig = new SentinelConfig(b); - assertThat(sentinelConfig.service().size(), is(5)); + assertThat(sentinelConfig.service().size(), is(4)); assertThat(sentinelConfig.service(0).name(), is("logserver")); assertThat(sentinelConfig.service(1).name(), is("slobrok")); assertThat(sentinelConfig.service(2).name(), is("slobrok2")); assertThat(sentinelConfig.service(3).name(), is("logd")); - assertThat(sentinelConfig.service(4).name(), is("filedistributorservice")); } /** @@ -128,11 +127,10 @@ public class AdminTestCase { SentinelConfig.Builder b = new SentinelConfig.Builder(); vespaModel.getConfig(b, localhostConfigId); SentinelConfig sentinelConfig = new SentinelConfig(b); - assertThat(sentinelConfig.service().size(), is(4)); + assertThat(sentinelConfig.service().size(), is(3)); assertThat(sentinelConfig.service(0).name(), is("logserver")); assertThat(sentinelConfig.service(1).name(), is("slobrok")); assertThat(sentinelConfig.service(2).name(), is("logd")); - assertThat(sentinelConfig.service(3).name(), is("filedistributorservice")); assertThat(sentinelConfig.service(0).affinity().cpuSocket(), is(-1)); assertTrue(sentinelConfig.service(0).preShutdownCommand().isEmpty()); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/admin/DedicatedAdminV4Test.java b/config-model/src/test/java/com/yahoo/vespa/model/admin/DedicatedAdminV4Test.java index 999ebd8ad4e..c3ff1a5076b 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/admin/DedicatedAdminV4Test.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/admin/DedicatedAdminV4Test.java @@ -17,7 +17,6 @@ import org.junit.Test; import org.xml.sax.SAXException; import java.io.IOException; -import java.util.List; import java.util.Set; import java.util.stream.Collectors; import java.util.stream.IntStream; @@ -72,12 +71,9 @@ public class DedicatedAdminV4Test { VespaModel model = createModel(hosts, services); assertEquals(3, model.getHosts().size()); - assertHostContainsServices(model, "hosts/myhost0", - "slobrok", "logd", "filedistributorservice"); - assertHostContainsServices(model, "hosts/myhost1", - "slobrok", "logd", "filedistributorservice"); - assertHostContainsServices(model, "hosts/myhost2", - "logserver", "logd", "filedistributorservice"); + assertHostContainsServices(model, "hosts/myhost0", "slobrok", "logd"); + assertHostContainsServices(model, "hosts/myhost1", "slobrok", "logd"); + assertHostContainsServices(model, "hosts/myhost2", "logserver", "logd"); Monitoring monitoring = model.getAdmin().getMonitoring(); assertEquals("vespa.routing", monitoring.getClustername()); @@ -136,14 +132,10 @@ public class DedicatedAdminV4Test { assertEquals(4, model.getHosts().size()); // 4 slobroks, 2 per cluster where possible - assertHostContainsServices(model, "hosts/myhost0", - "slobrok", "logd", "filedistributorservice", "logserver", "qrserver"); - assertHostContainsServices(model, "hosts/myhost1", - "slobrok", "logd", "filedistributorservice", "qrserver"); - assertHostContainsServices(model, "hosts/myhost2", - "slobrok", "logd", "filedistributorservice", "qrserver"); - assertHostContainsServices(model, "hosts/myhost3", - "slobrok", "logd", "filedistributorservice", "qrserver"); + assertHostContainsServices(model, "hosts/myhost0", "slobrok", "logd", "logserver", "qrserver"); + assertHostContainsServices(model, "hosts/myhost1", "slobrok", "logd", "qrserver"); + assertHostContainsServices(model, "hosts/myhost2", "slobrok", "logd", "qrserver"); + assertHostContainsServices(model, "hosts/myhost3", "slobrok", "logd", "qrserver"); } @Test @@ -161,12 +153,9 @@ public class DedicatedAdminV4Test { VespaModel model = createModel(hosts, services); assertEquals(3, model.getHosts().size()); - assertHostContainsServices(model, "hosts/myhost0", - "filedistributorservice", "logd", "logforwarder", "slobrok"); - assertHostContainsServices(model, "hosts/myhost1", - "filedistributorservice", "logd", "logforwarder", "slobrok"); - assertHostContainsServices(model, "hosts/myhost2", - "filedistributorservice", "logd", "logforwarder", "logserver"); + assertHostContainsServices(model, "hosts/myhost0", "logd", "logforwarder", "slobrok"); + assertHostContainsServices(model, "hosts/myhost1", "logd", "logforwarder", "slobrok"); + assertHostContainsServices(model, "hosts/myhost2", "logd", "logforwarder", "logserver"); Set<String> configIds = model.getConfigIds(); // 1 logforwarder on each host diff --git a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java index 043f961f98e..eb51d88c7b5 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java @@ -360,7 +360,7 @@ public class ContentBuilderTest extends DomBuilderTest { "logd", "configproxy", "config-sentinel", "configserver", "logserver", "slobrok", "container-clustercontroller", - "filedistributorservice", "storagenode", "distributor" + "storagenode", "distributor" }; assertServices(h, expectedServices); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2BuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2BuilderTest.java index 446ceb686e8..16a01360a51 100755 --- a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2BuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2BuilderTest.java @@ -31,7 +31,7 @@ public class DomAdminV2BuilderTest extends DomBuilderTest { private static MockRoot root; @Before - public void prepareTest() throws Exception { + public void prepareTest() { root = new MockRoot("root"); } @@ -210,7 +210,7 @@ public class DomAdminV2BuilderTest extends DomBuilderTest { final DomAdminV2Builder domAdminBuilder = new DomAdminV2Builder(ConfigModelContext.ApplicationType.DEFAULT, root.getDeployState().getFileRegistry(), multitenant, - configServerSpecs, root.getDeployState().disableFiledistributor()); + configServerSpecs); Admin admin = domAdminBuilder.build(root, xml); admin.addPerHostServices(root.getHostSystem().getHosts(), new DeployProperties.Builder().build()); return admin; diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/utils/ContentClusterUtils.java b/config-model/src/test/java/com/yahoo/vespa/model/content/utils/ContentClusterUtils.java index 27301ab60b2..1373acdf5d2 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/utils/ContentClusterUtils.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/utils/ContentClusterUtils.java @@ -61,7 +61,7 @@ public class ContentClusterUtils { public static ContentCluster createCluster(String clusterXml, MockRoot root) { Document doc = XML.getDocument(clusterXml); Admin admin = new Admin(root, new DefaultMonitoring("vespa", 60), new Metrics(), Collections.emptyMap(), false, - new FileDistributionConfigProducer.Builder(FileDistributionOptions.defaultOptions()) + new FileDistributionConfigProducer.Builder() .build(root, new MockFileRegistry(), null)); ConfigModelContext context = ConfigModelContext.create(null, root.getDeployState(), null, root, null); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTestCase.java index e597e338314..d3879f1ab36 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTestCase.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTestCase.java @@ -267,7 +267,7 @@ public class VespaModelTestCase { assertThat(hosts.size(), is(1)); //logd, config proxy, sentinel, config server, slobrok, log server, file distributor HostInfo host = hosts.iterator().next(); - assertThat(host.getServices().size(), is(7)); + assertThat(host.getServices().size(), is(6)); new LogdConfig((LogdConfig.Builder) model.getConfig(new LogdConfig.Builder(), "admin/model")); } |