diff options
67 files changed, 1442 insertions, 244 deletions
diff --git a/config-model/src/main/java/com/yahoo/config/model/ConfigModelContext.java b/config-model/src/main/java/com/yahoo/config/model/ConfigModelContext.java index 8b3d983d6c1..78a8c161c3b 100644 --- a/config-model/src/main/java/com/yahoo/config/model/ConfigModelContext.java +++ b/config-model/src/main/java/com/yahoo/config/model/ConfigModelContext.java @@ -5,17 +5,14 @@ import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.producer.AbstractConfigProducer; -import org.w3c.dom.Element; -import java.util.Optional; import java.util.stream.Stream; /** * This class contains a context that is passed to a model builder, and can be used to retrieve the application package, * logger etc. * - * @author lulf - * @since 5.1 + * @author Ulf Lilleengen */ public final class ConfigModelContext { diff --git a/config-model/src/main/java/com/yahoo/config/model/admin/AdminModel.java b/config-model/src/main/java/com/yahoo/config/model/admin/AdminModel.java index 32cb1840e7c..5eb4afcc241 100644 --- a/config-model/src/main/java/com/yahoo/config/model/admin/AdminModel.java +++ b/config-model/src/main/java/com/yahoo/config/model/admin/AdminModel.java @@ -75,7 +75,11 @@ public class AdminModel extends ConfigModel { public void doBuild(AdminModel model, Element adminElement, ConfigModelContext modelContext) { AbstractConfigProducer parent = modelContext.getParentProducer(); DeployProperties properties = modelContext.getDeployState().getProperties(); - DomAdminV2Builder domBuilder = new DomAdminV2Builder(modelContext.getApplicationType(), modelContext.getDeployState().getFileRegistry(), properties.multitenant(), properties.configServerSpecs()); + DomAdminV2Builder domBuilder = new DomAdminV2Builder(modelContext.getApplicationType(), + modelContext.getDeployState().getFileRegistry(), + properties.multitenant(), + properties.configServerSpecs(), + modelContext.getDeployState().disableFiledistributor()); model.admin = domBuilder.build(parent, adminElement); // TODO: Is required since other models depend on admin. if (parent instanceof ApplicationConfigProducerRoot) { @@ -101,7 +105,11 @@ public class AdminModel extends ConfigModel { public void doBuild(AdminModel model, Element adminElement, ConfigModelContext modelContext) { AbstractConfigProducer parent = modelContext.getParentProducer(); DeployProperties properties = modelContext.getDeployState().getProperties(); - DomAdminV4Builder domBuilder = new DomAdminV4Builder(modelContext, properties.multitenant(), properties.configServerSpecs(), model.getContainerModels()); + DomAdminV4Builder domBuilder = new DomAdminV4Builder(modelContext, + properties.multitenant(), + properties.configServerSpecs(), + model.getContainerModels(), + modelContext.getDeployState().disableFiledistributor()); model.admin = domBuilder.build(parent, adminElement); // TODO: Is required since other models depend on admin. if (parent instanceof ApplicationConfigProducerRoot) { diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java b/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java index 25d45dee234..43d3fafdb78 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java @@ -68,8 +68,8 @@ public class DeployState implements ConfigDefinitionStore { private final ValidationOverrides validationOverrides; private final Version wantedNodeVespaVersion; private final Instant now; - private final HostProvisioner provisioner; + private final boolean disableFiledistributor; public static DeployState createTestState() { return new Builder().build(); @@ -82,8 +82,8 @@ public class DeployState implements ConfigDefinitionStore { private DeployState(ApplicationPackage applicationPackage, SearchDocumentModel searchDocumentModel, RankProfileRegistry rankProfileRegistry, FileRegistry fileRegistry, DeployLogger deployLogger, Optional<HostProvisioner> hostProvisioner, DeployProperties properties, Optional<ApplicationPackage> permanentApplicationPackage, Optional<ConfigDefinitionRepo> configDefinitionRepo, - java.util.Optional<Model> previousModel, Set<Rotation> rotations, Zone zone, QueryProfiles queryProfiles, - SemanticRules semanticRules, Instant now, Version wantedNodeVespaVersion) { + java.util.Optional<Model> previousModel, Set<Rotation> rotations, Zone zone, QueryProfiles queryProfiles, + SemanticRules semanticRules, Instant now, Version wantedNodeVespaVersion, boolean disableFiledistributor) { this.logger = deployLogger; this.fileRegistry = fileRegistry; this.rankProfileRegistry = rankProfileRegistry; @@ -102,6 +102,7 @@ public class DeployState implements ConfigDefinitionStore { this.validationOverrides = applicationPackage.getValidationOverrides().map(ValidationOverrides::fromXml).orElse(ValidationOverrides.empty); this.wantedNodeVespaVersion = wantedNodeVespaVersion; this.now = now; + this.disableFiledistributor = disableFiledistributor; } public static HostProvisioner getDefaultModelHostProvisioner(ApplicationPackage applicationPackage) { @@ -169,6 +170,7 @@ public class DeployState implements ConfigDefinitionStore { public ApplicationPackage getApplicationPackage() { return applicationPackage; } + public List<SearchDefinition> getSearchDefinitions() { return searchDefinitions; } @@ -214,6 +216,8 @@ public class DeployState implements ConfigDefinitionStore { public Instant now() { return now; } + public boolean disableFiledistributor() { return disableFiledistributor; } + public static class Builder { private ApplicationPackage applicationPackage = MockApplicationPackage.createEmpty(); @@ -228,6 +232,7 @@ public class DeployState implements ConfigDefinitionStore { private Zone zone = Zone.defaultZone(); private Instant now = Instant.now(); private Version wantedNodeVespaVersion = Vtag.currentVersion; + private boolean disableFiledistributor = false; public Builder applicationPackage(ApplicationPackage applicationPackage) { this.applicationPackage = applicationPackage; @@ -289,13 +294,19 @@ public class DeployState implements ConfigDefinitionStore { return this; } + public Builder disableFiledistributor(boolean disableFiledistributor) { + this.disableFiledistributor = disableFiledistributor; + return this; + } + public DeployState build() { RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); QueryProfiles queryProfiles = new QueryProfilesBuilder().build(applicationPackage); SemanticRules semanticRules = new SemanticRuleBuilder().build(applicationPackage); SearchDocumentModel searchDocumentModel = createSearchDocumentModel(rankProfileRegistry, logger, queryProfiles); return new DeployState(applicationPackage, searchDocumentModel, rankProfileRegistry, fileRegistry, logger, hostProvisioner, - properties, permanentApplicationPackage, configDefinitionRepo, previousModel, rotations, zone, queryProfiles, semanticRules, now, wantedNodeVespaVersion); + properties, permanentApplicationPackage, configDefinitionRepo, previousModel, rotations, + zone, queryProfiles, semanticRules, now, wantedNodeVespaVersion, disableFiledistributor); } private SearchDocumentModel createSearchDocumentModel(RankProfileRegistry rankProfileRegistry, DeployLogger logger, QueryProfiles queryProfiles) { diff --git a/config-model/src/main/java/com/yahoo/config/model/test/MockRoot.java b/config-model/src/main/java/com/yahoo/config/model/test/MockRoot.java index 277e02e43e2..2a3ff1ca905 100644 --- a/config-model/src/main/java/com/yahoo/config/model/test/MockRoot.java +++ b/config-model/src/main/java/com/yahoo/config/model/test/MockRoot.java @@ -11,6 +11,7 @@ import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.builder.xml.XmlHelper; import com.yahoo.config.model.producer.AbstractConfigProducer; import com.yahoo.config.model.producer.AbstractConfigProducerRoot; +import com.yahoo.config.provision.Zone; import com.yahoo.text.XML; import com.yahoo.vespa.model.ConfigProducer; import com.yahoo.vespa.model.HostSystem; @@ -147,7 +148,8 @@ public class MockRoot extends AbstractConfigProducerRoot { try { Document doc = XmlHelper.getDocumentBuilder().parse(new InputSource(new StringReader(servicesXml))); - setAdmin(new DomAdminV2Builder(ConfigModelContext.ApplicationType.DEFAULT, deployState.getFileRegistry(), false, new ArrayList<>()). + setAdmin(new DomAdminV2Builder(ConfigModelContext.ApplicationType.DEFAULT, deployState.getFileRegistry(), + false, new ArrayList<>(), deployState.disableFiledistributor()). build(this, XML.getChildren(doc.getDocumentElement(), "admin").get(0))); } catch (SAXException | IOException e) { throw new RuntimeException(e); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/SystemMetrics.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/SystemMetrics.java index cafdf83608f..3db09f6b566 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/SystemMetrics.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/SystemMetrics.java @@ -11,6 +11,7 @@ import java.util.Set; @SuppressWarnings("UnusedDeclaration") // Used by model amenders public class SystemMetrics { public static final String CPU_UTIL = "cpu.util"; + public static final String CPU_SYS_UTIL = "cpu.sys.util"; public static final String DISK_LIMIT = "disk.limit"; public static final String DISK_USED = "disk.used"; public static final String DISK_UTIL = "disk.util"; @@ -23,6 +24,7 @@ public class SystemMetrics { private static MetricSet createSystemMetricSet() { Set<Metric> dockerNodeMetrics = ImmutableSet.of(new Metric(CPU_UTIL), + new Metric(CPU_SYS_UTIL), new Metric(DISK_LIMIT), new Metric(DISK_USED), new Metric(DISK_UTIL), diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminBuilderBase.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminBuilderBase.java index a3a1a0fbebc..3e825c3912a 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminBuilderBase.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminBuilderBase.java @@ -37,16 +37,19 @@ public abstract class DomAdminBuilderBase extends VespaDomBuilder.DomConfigProdu private final ApplicationType applicationType; private final List<ConfigServerSpec> configServerSpecs; private final FileRegistry fileRegistry; + private final boolean disableFiledistributor; protected final boolean multitenant; - public DomAdminBuilderBase(ApplicationType applicationType, FileRegistry fileRegistry, boolean multitenant, List<ConfigServerSpec> configServerSpecs) { + DomAdminBuilderBase(ApplicationType applicationType, FileRegistry fileRegistry, boolean multitenant, + List<ConfigServerSpec> configServerSpecs, boolean disableFiledistributor) { this.applicationType = applicationType; this.fileRegistry = fileRegistry; this.multitenant = multitenant; this.configServerSpecs = configServerSpecs; + this.disableFiledistributor = disableFiledistributor; } - protected List<Configserver> getConfigServersFromSpec(AbstractConfigProducer parent) { + List<Configserver> getConfigServersFromSpec(AbstractConfigProducer parent) { List<Configserver> configservers = new ArrayList<>(); for (ConfigServerSpec spec : configServerSpecs) { HostSystem hostSystem = parent.getHostSystem(); @@ -76,7 +79,9 @@ public abstract class DomAdminBuilderBase extends VespaDomBuilder.DomConfigProdu new ModelConfigProvider(admin); - FileDistributionOptions fileDistributionOptions = new DomFileDistributionOptionsBuilder().build(XML.getChild(adminElement, "filedistribution")); + FileDistributionOptions fileDistributionOptions = FileDistributionOptions.defaultOptions(); + fileDistributionOptions.disabled(disableFiledistributor); + fileDistributionOptions = new DomFileDistributionOptionsBuilder(fileDistributionOptions).build(XML.getChild(adminElement, "filedistribution")); admin.setFileDistribution(new FileDistributionConfigProducer.Builder(fileDistributionOptions).build(parent, fileRegistry)); return admin; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2Builder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2Builder.java index dd1d4e36255..d966f3b49f6 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2Builder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2Builder.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.builder.xml.dom; +import com.yahoo.config.application.api.FileRegistry; import com.yahoo.config.model.ConfigModelContext; import com.yahoo.config.model.api.ConfigServerSpec; import com.yahoo.config.model.producer.AbstractConfigProducer; @@ -18,7 +19,6 @@ import com.yahoo.vespa.model.builder.xml.dom.VespaDomBuilder.DomConfigProducerBu import com.yahoo.vespa.model.container.Container; import com.yahoo.vespa.model.container.ContainerCluster; import com.yahoo.vespa.model.container.xml.ContainerModelBuilder; -import com.yahoo.config.application.api.FileRegistry; import org.w3c.dom.Element; import java.util.List; @@ -37,8 +37,9 @@ public class DomAdminV2Builder extends DomAdminBuilderBase { public DomAdminV2Builder(ConfigModelContext.ApplicationType applicationType, FileRegistry fileRegistry, boolean multitenant, - List<ConfigServerSpec> configServerSpecs) { - super(applicationType, fileRegistry, multitenant, configServerSpecs); + List<ConfigServerSpec> configServerSpecs, + boolean disableFiledistributor) { + super(applicationType, fileRegistry, multitenant, configServerSpecs, disableFiledistributor); } @Override diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java index cb8ec205395..a7a785cea43 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java @@ -30,8 +30,10 @@ public class DomAdminV4Builder extends DomAdminBuilderBase { private final Collection<ContainerModel> containerModels; private final ConfigModelContext context; - public DomAdminV4Builder(ConfigModelContext context, boolean multitenant, List<ConfigServerSpec> configServerSpecs, Collection<ContainerModel> containerModels) { - super(context.getApplicationType(), context.getDeployState().getFileRegistry(), multitenant, configServerSpecs); + public DomAdminV4Builder(ConfigModelContext context, boolean multitenant, List<ConfigServerSpec> configServerSpecs, + Collection<ContainerModel> containerModels, boolean disableFiledistributor) { + super(context.getApplicationType(), context.getDeployState().getFileRegistry(), multitenant, + configServerSpecs, disableFiledistributor); this.containerModels = containerModels; this.context = context; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomFileDistributionOptionsBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomFileDistributionOptionsBuilder.java index 8a5d6846a64..90d71f186ae 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomFileDistributionOptionsBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomFileDistributionOptionsBuilder.java @@ -15,6 +15,11 @@ import java.util.Optional; * @author hmusum */ public class DomFileDistributionOptionsBuilder { + private final FileDistributionOptions fileDistributionOptions; + + public DomFileDistributionOptionsBuilder(FileDistributionOptions fileDistributionOptions) { + this.fileDistributionOptions = fileDistributionOptions; + } private static void throwExceptionForElementInFileDistribution(String subElement, String reason) { throw new RuntimeException("In element '" + subElement + "' contained in 'filedistribution': " + reason); @@ -34,15 +39,14 @@ public class DomFileDistributionOptionsBuilder { } public FileDistributionOptions build(Element fileDistributionElement) { - FileDistributionOptions options = FileDistributionOptions.defaultOptions(); if (fileDistributionElement != null) { - getAmount("uploadbitrate", fileDistributionElement).ifPresent(options::uploadBitRate); - getAmount("downloadbitrate", fileDistributionElement).ifPresent(options::downloadBitRate); + getAmount("uploadbitrate", fileDistributionElement).ifPresent(fileDistributionOptions::uploadBitRate); + getAmount("downloadbitrate", fileDistributionElement).ifPresent(fileDistributionOptions::downloadBitRate); Element disable = XML.getChild(fileDistributionElement, "disabled"); if (disable != null) { - options.disabled(Boolean.valueOf(XML.getValue(disable))); + fileDistributionOptions.disabled(Boolean.valueOf(XML.getValue(disable))); } } - return options; + return fileDistributionOptions; } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/ConfigserverCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/ConfigserverCluster.java index 62828b314d0..bce78017bdd 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/ConfigserverCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/ConfigserverCluster.java @@ -144,6 +144,9 @@ public class ConfigserverCluster extends AbstractConfigProducer if (options.loadBalancerAddress().isPresent()) { builder.loadBalancerAddress(options.loadBalancerAddress().get()); } + if (options.disableFiledistributor().isPresent()) { + builder.disableFiledistributor(options.disableFiledistributor().get()); + } } private String[] getConfigModelPluginDirs() { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/option/CloudConfigOptions.java b/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/option/CloudConfigOptions.java index aeb86ae9d59..866bae6666a 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/option/CloudConfigOptions.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/option/CloudConfigOptions.java @@ -4,7 +4,7 @@ package com.yahoo.vespa.model.container.configserver.option; import java.util.Optional; /** - * @author tonytv + * @author Tony Vaagenes */ public interface CloudConfigOptions { @@ -44,4 +44,5 @@ public interface CloudConfigOptions { Optional<String> dockerRegistry(); Optional<String> dockerVespaBaseImage(); Optional<String> loadBalancerAddress(); + Optional<Boolean> disableFiledistributor(); } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2BuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2BuilderTest.java index 8bd62789d52..91f6dd65bda 100755 --- a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2BuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2BuilderTest.java @@ -207,7 +207,10 @@ public class DomAdminV2BuilderTest extends DomBuilderTest { } private Admin buildAdmin(Element xml, boolean multitenant, List<ConfigServerSpec> configServerSpecs) { - final DomAdminV2Builder domAdminBuilder = new DomAdminV2Builder(ConfigModelContext.ApplicationType.DEFAULT, root.getDeployState().getFileRegistry(), multitenant, configServerSpecs); + final DomAdminV2Builder domAdminBuilder = + new DomAdminV2Builder(ConfigModelContext.ApplicationType.DEFAULT, + root.getDeployState().getFileRegistry(), multitenant, + configServerSpecs, root.getDeployState().disableFiledistributor()); Admin admin = domAdminBuilder.build(root, xml); admin.addPerHostServices(root.getHostSystem().getHosts(), new DeployProperties.Builder().build()); return admin; diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/configserver/TestOptions.java b/config-model/src/test/java/com/yahoo/vespa/model/container/configserver/TestOptions.java index e2c8f2e2c52..1784fe0e974 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/configserver/TestOptions.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/configserver/TestOptions.java @@ -6,8 +6,7 @@ import com.yahoo.vespa.model.container.configserver.option.CloudConfigOptions; import java.util.Optional; /** - * @author lulf - * @since 5. + * @author Ulf Lilleengen */ public class TestOptions implements CloudConfigOptions { private Optional<Integer> rpcPort = Optional.empty(); @@ -20,6 +19,7 @@ public class TestOptions implements CloudConfigOptions { private Optional<Boolean> useVespaVersionInRequest = Optional.empty(); private Optional<Boolean> hostedVespa = Optional.empty(); private Optional<Integer> numParallelTenantLoaders = Optional.empty(); + private Optional<Boolean> disableFiledistributor = Optional.empty(); @Override public Optional<Integer> rpcPort() { @@ -118,6 +118,9 @@ public class TestOptions implements CloudConfigOptions { @Override public Optional<String> loadBalancerAddress() { return Optional.empty(); } + @Override + public Optional<Boolean> disableFiledistributor() { return disableFiledistributor; } + public TestOptions numParallelTenantLoaders(int numLoaders) { this.numParallelTenantLoaders = Optional.of(numLoaders); return this; diff --git a/config/src/tests/configagent/configagent.cpp b/config/src/tests/configagent/configagent.cpp index f6593890975..7eb9442f492 100644 --- a/config/src/tests/configagent/configagent.cpp +++ b/config/src/tests/configagent/configagent.cpp @@ -22,17 +22,17 @@ public: bool abort() override { return false; } bool isAborted() const override { return false; } void setError(int errorCode) override { (void) errorCode; } - bool verifyState(const ConfigState &) const override { return false; } const ConfigKey _key; }; class MyConfigResponse : public ConfigResponse { public: - MyConfigResponse(const ConfigKey & key, const ConfigValue & value, bool valid, int64_t timestamp, - const vespalib::string & md5, const std::string & errorMsg, int errorC0de, bool iserror) + MyConfigResponse(const ConfigKey & key, const ConfigValue & value, bool isUpdated, bool valid, + int64_t timestamp, const vespalib::string & md5, const std::string & errorMsg, int errorC0de, bool iserror) : _key(key), _value(value), + _isUpdated(isUpdated), _fillCalled(false), _valid(valid), _state(md5, timestamp), @@ -54,6 +54,7 @@ public: const ConfigKey _key; const ConfigValue _value; + bool _isUpdated; bool _fillCalled; bool _valid; const ConfigState _state; @@ -63,19 +64,24 @@ public: Trace _trace; - static ConfigResponse::UP createOKResponse(const ConfigKey & key, const ConfigValue & value, uint64_t timestamp = 10, const vespalib::string & md5 = "a") +/** + MyConfigResponse(const ConfigKey & key, const ConfigValue & value, bool isUpdated, bool valid, + int64_t timestamp, const vespalib::string & md5, int64_t prevTimestamp, const vespalib::string &prevMd5, + const std::string & errorMsg, int errorC0de, bool iserror) +*/ + static ConfigResponse::UP createOKResponse(const ConfigKey & key, const ConfigValue & value) { - return ConfigResponse::UP(new MyConfigResponse(key, value, true, timestamp, md5, "", 0, false)); + return ConfigResponse::UP(new MyConfigResponse(key, value, true, true, 10, "a", "", 0, false)); } static ConfigResponse::UP createServerErrorResponse(const ConfigKey & key, const ConfigValue & value) { - return ConfigResponse::UP(new MyConfigResponse(key, value, true, 10, "a", "whinewhine", 2, true)); + return ConfigResponse::UP(new MyConfigResponse(key, value, false, true, 10, "a", "whinewhine", 2, true)); } static ConfigResponse::UP createConfigErrorResponse(const ConfigKey & key, const ConfigValue & value) { - return ConfigResponse::UP(new MyConfigResponse(key, value, false, 10, "a", "", 0, false)); + return ConfigResponse::UP(new MyConfigResponse(key, value, false, false, 10, "a", "", 0, false)); } }; @@ -148,40 +154,12 @@ TEST("require that successful request is delivered to holder") { handler.handleResponse(MyConfigRequest(testKey), MyConfigResponse::createOKResponse(testKey, testValue)); ASSERT_TRUE(latch->poll()); ConfigUpdate::UP update(latch->provide()); - ASSERT_TRUE(update); + ASSERT_TRUE(update.get() != NULL); ASSERT_TRUE(update->hasChanged()); MyConfig cfg(update->getValue()); ASSERT_EQUAL("l33t", cfg.myField); } -TEST("require that important(the change) request is delivered to holder even if it was not the last") { - const ConfigKey testKey(ConfigKey::create<MyConfig>("mykey")); - const ConfigValue testValue1(createValue("l33t", "a")); - const ConfigValue testValue2(createValue("l34t", "b")); - IConfigHolder::SP latch(new MyHolder()); - - FRTConfigAgent handler(latch, testTimingValues); - handler.handleResponse(MyConfigRequest(testKey), - MyConfigResponse::createOKResponse(testKey, testValue1, 1, testValue1.getMd5())); - ASSERT_TRUE(latch->poll()); - ConfigUpdate::UP update(latch->provide()); - ASSERT_TRUE(update); - ASSERT_TRUE(update->hasChanged()); - MyConfig cfg(update->getValue()); - ASSERT_EQUAL("l33t", cfg.myField); - - handler.handleResponse(MyConfigRequest(testKey), - MyConfigResponse::createOKResponse(testKey, testValue2, 2, testValue2.getMd5())); - handler.handleResponse(MyConfigRequest(testKey), - MyConfigResponse::createOKResponse(testKey, testValue2, 3, testValue2.getMd5())); - ASSERT_TRUE(latch->poll()); - update = latch->provide(); - ASSERT_TRUE(update); - ASSERT_TRUE(update->hasChanged()); - MyConfig cfg2(update->getValue()); - ASSERT_EQUAL("l34t", cfg2.myField); -} - TEST("require that successful request sets correct wait time") { const ConfigKey testKey(ConfigKey::create<MyConfig>("mykey")); const ConfigValue testValue(createValue("l33t", "a")); diff --git a/config/src/tests/frt/frt.cpp b/config/src/tests/frt/frt.cpp index 5f4b7dee215..1fe91af885b 100644 --- a/config/src/tests/frt/frt.cpp +++ b/config/src/tests/frt/frt.cpp @@ -6,6 +6,10 @@ #include <vespa/config/common/configdefinition.h> #include <vespa/config/frt/connection.h> #include <vespa/config/frt/frtsource.h> +#include <vespa/config/frt/frtconfigresponse.h> +#include <vespa/config/frt/frtconfigrequest.h> +#include <vespa/config/frt/frtconfigrequestv2.h> +#include <vespa/config/frt/frtconfigresponsev2.h> #include <vespa/config/frt/frtconfigrequestv3.h> #include <vespa/config/frt/frtconfigresponsev3.h> #include <vespa/vespalib/data/slime/slime.h> @@ -228,27 +232,107 @@ namespace { TEST_F("require that empty config response does not validate", RPCFixture()) { - FRTConfigResponseV3 fail1(f1.createEmptyRequest()); + FRTConfigResponseV1 fail1(f1.createEmptyRequest()); ASSERT_FALSE(fail1.validateResponse()); ASSERT_FALSE(fail1.hasValidResponse()); ASSERT_TRUE(fail1.isError()); } TEST_F("require that response containing errors does not validate", RPCFixture()) { - FRTConfigResponseV3 fail1(f1.createErrorRequest()); + FRTConfigResponseV1 fail1(f1.createErrorRequest()); ASSERT_FALSE(fail1.validateResponse()); ASSERT_FALSE(fail1.hasValidResponse()); ASSERT_TRUE(fail1.isError()); ASSERT_TRUE(fail1.errorCode() != 0); } +TEST_F("require that valid response validates", RPCFixture()) { + std::vector<vespalib::string> vec; + vec.push_back("bar \"foo\""); + FRTConfigResponseV1 ok(f1.createOKResponse("foo", "baz", "bim", "boo", 12, 15, vec, "mn")); + ASSERT_TRUE(ok.validateResponse()); + ASSERT_TRUE(ok.hasValidResponse()); +} + TEST_F("require that response contains all values", RPCFixture()) { - FRTConfigResponseV3 ok(f1.createOKResponse("foo", "baz", "bim", "boo", 12, 15)); + FRTConfigResponseV1 ok(f1.createOKResponse("foo", "baz", "bim", "boo", 12, 15)); ASSERT_FALSE(ok.validateResponse()); ASSERT_FALSE(ok.hasValidResponse()); } -TEST_FF("require that request is config task is scheduled", SourceFixture(), FRTFixture(f1)) +TEST_F("require that valid response returns values after fill", RPCFixture()) { + std::vector<vespalib::string> vec; + vec.push_back("bar \"foo\""); + FRTConfigResponseV1 ok(f1.createOKResponse("foo", "baz", "bim", "boo", 12, 15, vec, "mn")); + ASSERT_TRUE(ok.validateResponse()); + ASSERT_TRUE(ok.hasValidResponse()); + + // Should not be valid + ASSERT_TRUE(ConfigKey() == ok.getKey()); + ASSERT_TRUE(ConfigValue() == ok.getValue()); + + ok.fill(); + ConfigKey key(ok.getKey()); + ASSERT_EQUAL("foo", key.getDefName()); + ASSERT_EQUAL("baz", key.getDefMd5()); + ASSERT_EQUAL("bim", key.getConfigId()); + ASSERT_EQUAL("mn", key.getDefNamespace()); + + ConfigValue value(ok.getValue()); + ASSERT_TRUE(vec == value.getLines()); +} + +TEST("require that request parameters are correctly initialized") { + ConnectionMock conn; + std::vector<vespalib::string> schema; + schema.push_back("foo"); + schema.push_back("bar"); + ConfigKey key("foo", "bar", "bim", "boo", schema); + FRTConfigRequestV1 frtReq(key, &conn, "mymd5", 1337, 8); + + FRT_RPCRequest * req = frtReq.getRequest(); + FRT_Values & params(*req->GetParams()); + ASSERT_EQUAL("bar", std::string(params[0]._string._str)); + ASSERT_EQUAL("", std::string(params[1]._string._str)); + ASSERT_EQUAL("boo", std::string(params[2]._string._str)); + ASSERT_EQUAL("foo", std::string(params[3]._string._str)); + ASSERT_EQUAL("mymd5", std::string(params[4]._string._str)); + ASSERT_EQUAL(1337u, params[5]._intval64); + ASSERT_EQUAL(8u, params[6]._intval64); + ASSERT_EQUAL("bim", std::string(params[7]._string._str)); + ASSERT_EQUAL(2u, params[8]._string_array._len); + ASSERT_EQUAL("foo", std::string(params[8]._string_array._pt[0]._str)); + ASSERT_EQUAL("bar", std::string(params[8]._string_array._pt[1]._str)); + ASSERT_EQUAL(1u, params[9]._intval32); +} + +TEST("require that request is aborted") { + MyAbortHandler handler; + ConnectionMock conn; + ConfigKey key("foo", "bar", "bim", "boo"); + FRTConfigRequestV1 frtReq(key, &conn, "mymd5", 1337, 8); + frtReq.getRequest()->SetAbortHandler(&handler); + ASSERT_FALSE(frtReq.isAborted()); + ASSERT_TRUE(frtReq.abort()); +} + +TEST_FF("require that request is invoked", SourceFixture(), + FRTFixture(f1)) +{ + f2.result.state = ConfigState("foo", 3); + f2.src.getConfig(); + ASSERT_TRUE(f2.src.getCurrentRequest().verifyKey(f1.key)); + ASSERT_FALSE(f2.src.getCurrentRequest().verifyKey(ConfigKey("foo", "bal", "bim", "boo", std::vector<vespalib::string>()))); + ASSERT_FALSE(f2.src.getCurrentRequest().verifyState(ConfigState("foo", 0))); + ASSERT_FALSE(f2.src.getCurrentRequest().verifyState(ConfigState("foo", 1))); + ASSERT_FALSE(f2.src.getCurrentRequest().verifyState(ConfigState("bar", 1))); + ASSERT_TRUE(f2.src.getCurrentRequest().verifyState(ConfigState("foo", 3))); + ASSERT_TRUE(f2.result.notified); + f2.src.close(); +} + +TEST_FF("require that request is config task is scheduled", SourceFixture(), + FRTFixture(f1)) { f2.src.getConfig(); ASSERT_TRUE(f2.result.notified); @@ -265,7 +349,7 @@ TEST_FF("require that request is config task is scheduled", SourceFixture(), FRT f2.src.close(); } -TEST("require that v3 request is correctly initialized") { +TEST("require that v2 request is correctly initialized") { ConnectionMock conn; ConfigKey key = ConfigKey::create<MyConfig>("foobi"); vespalib::string md5 = "mymd5"; @@ -275,13 +359,97 @@ TEST("require that v3 request is correctly initialized") { int64_t timeout = 3000; Trace traceIn(3); traceIn.trace(2, "Hei"); - FRTConfigRequestV3 v3req(&conn, key, md5, currentGeneration, wantedGeneration, hostName, - timeout, traceIn, VespaVersion::fromString("1.2.3"), CompressionType::LZ4); - ASSERT_TRUE(v3req.verifyState(ConfigState(md5, 3))); - ASSERT_FALSE(v3req.verifyState(ConfigState(md5, 2))); - ASSERT_FALSE(v3req.verifyState(ConfigState("xxx", 3))); - ASSERT_FALSE(v3req.verifyState(ConfigState("xxx", 2))); + FRTConfigRequestV2 v2req(&conn, key, md5, currentGeneration, wantedGeneration, hostName, timeout, traceIn); + ConfigDefinition origDef(MyConfig::CONFIG_DEF_SCHEMA); + FRT_RPCRequest * req = v2req.getRequest(); + ASSERT_TRUE(req != NULL); + FRT_Values & params(*req->GetParams()); + std::string json(params[0]._string._str); + Slime slime; + JsonFormat::decode(Memory(json), slime); + Inspector & root(slime.get()); + EXPECT_EQUAL(2, root[REQUEST_VERSION].asLong()); + EXPECT_EQUAL(key.getDefName(), root[REQUEST_DEF_NAME].asString().make_string()); + EXPECT_EQUAL(key.getDefNamespace(), root[REQUEST_DEF_NAMESPACE].asString().make_string()); + EXPECT_EQUAL(key.getDefMd5(), root[REQUEST_DEF_MD5].asString().make_string()); + EXPECT_EQUAL(key.getConfigId(), root[REQUEST_CLIENT_CONFIGID].asString().make_string()); + EXPECT_EQUAL(hostName, root[REQUEST_CLIENT_HOSTNAME].asString().make_string()); + EXPECT_EQUAL(currentGeneration, root[REQUEST_CURRENT_GENERATION].asLong()); + EXPECT_EQUAL(wantedGeneration, root[REQUEST_WANTED_GENERATION].asLong()); + EXPECT_EQUAL(md5, root[REQUEST_CONFIG_MD5].asString().make_string()); + EXPECT_EQUAL(timeout, root[REQUEST_TIMEOUT].asLong()); + Trace trace; + trace.deserialize(root[REQUEST_TRACE]); + EXPECT_TRUE(trace.shouldTrace(2)); + EXPECT_TRUE(trace.shouldTrace(3)); + EXPECT_FALSE(trace.shouldTrace(4)); + EXPECT_EQUAL(timeout, root[REQUEST_TIMEOUT].asLong()); + ConfigDefinition def; + def.deserialize(root[REQUEST_DEF_CONTENT]); + EXPECT_EQUAL(origDef.asString(), def.asString()); + ConfigResponse::UP response(v2req.createResponse(req)); + req->GetReturn()->AddString("foobar"); + EXPECT_TRUE(response->validateResponse()); +} + +TEST("require that v2 reponse is correctly initialized") { + ConnectionMock conn; + Slime slime; + ConfigKey key = ConfigKey::create<MyConfig>("foobi"); + vespalib::string md5 = "mymd5"; + int64_t generation = 3; + vespalib::string hostname = "myhhost"; + Trace traceIn(3); + traceIn.trace(2, "Hei!"); + Cursor & root(slime.setObject()); + root.setLong(RESPONSE_VERSION, 2ul); + root.setString(RESPONSE_DEF_NAME, Memory(key.getDefName())); + root.setString(RESPONSE_DEF_NAMESPACE, Memory(key.getDefNamespace())); + root.setString(RESPONSE_DEF_MD5, Memory(key.getDefMd5())); + root.setString(RESPONSE_CONFIGID, Memory(key.getConfigId())); + root.setString(RESPONSE_CLIENT_HOSTNAME, Memory(hostname)); + root.setString(RESPONSE_CONFIG_MD5, Memory(md5)); + root.setLong(RESPONSE_CONFIG_GENERATION, generation); + traceIn.serialize(root.setObject(RESPONSE_TRACE)); + Cursor & payload(root.setObject(RESPONSE_PAYLOAD)); + payload.setString("myField", "foobiar"); + SimpleBuffer buf; + JsonFormat::encode(slime, buf, true); + FRT_RPCRequest * req = conn.allocRPCRequest(); + req->GetReturn()->AddString(buf.get().make_string().c_str()); + FRTConfigResponseV2 response(req); + ASSERT_TRUE(response.validateResponse()); + response.fill(); + Trace trace(response.getTrace()); + EXPECT_TRUE(trace.shouldTrace(3)); + EXPECT_FALSE(trace.shouldTrace(4)); + ConfigKey responseKey(response.getKey()); + EXPECT_EQUAL(key.getDefName(), responseKey.getDefName()); + EXPECT_EQUAL(key.getDefNamespace(), responseKey.getDefNamespace()); + EXPECT_EQUAL(key.getDefMd5(), responseKey.getDefMd5()); + EXPECT_EQUAL(key.getConfigId(), responseKey.getConfigId()); + EXPECT_EQUAL(hostname, response.getHostName()); + ConfigState state(response.getConfigState()); + EXPECT_EQUAL(md5, state.md5); + EXPECT_EQUAL(generation, state.generation); + ConfigValue value(response.getValue()); + MyConfig::UP config(value.newInstance<MyConfig>()); + EXPECT_EQUAL("foobiar", config->myField); + req->SubRef(); +} + +TEST("require that v3 request is correctly initialized") { + ConnectionMock conn; + ConfigKey key = ConfigKey::create<MyConfig>("foobi"); + vespalib::string md5 = "mymd5"; + int64_t currentGeneration = 3; + int64_t wantedGeneration = 4; + vespalib::string hostName = "myhost"; + int64_t timeout = 3000; + Trace traceIn(3); + traceIn.trace(2, "Hei"); + FRTConfigRequestV3 v3req(&conn, key, md5, currentGeneration, wantedGeneration, hostName, timeout, traceIn, VespaVersion::fromString("1.2.3"), CompressionType::LZ4); ConfigDefinition origDef(MyConfig::CONFIG_DEF_SCHEMA); FRT_RPCRequest * req = v3req.getRequest(); diff --git a/config/src/vespa/config/common/configholder.cpp b/config/src/vespa/config/common/configholder.cpp index 6eeb1bd79a2..e94510dcdea 100644 --- a/config/src/vespa/config/common/configholder.cpp +++ b/config/src/vespa/config/common/configholder.cpp @@ -39,7 +39,7 @@ bool ConfigHolder::poll() { vespalib::MonitorGuard guard(_monitor); - return static_cast<bool>(_current); + return (_current.get() != NULL); } void diff --git a/config/src/vespa/config/common/configrequest.h b/config/src/vespa/config/common/configrequest.h index efe1e56cdd7..395b09abcb0 100644 --- a/config/src/vespa/config/common/configrequest.h +++ b/config/src/vespa/config/common/configrequest.h @@ -24,13 +24,15 @@ public: ConfigRequest() { } virtual ~ConfigRequest() { } + virtual const ConfigKey & getKey() const = 0; + /** Abort a request. */ virtual bool abort() = 0; + virtual bool isAborted() const = 0; - virtual void setError(int errorCode) = 0; - virtual bool verifyState(const ConfigState & state) const = 0; + virtual void setError(int errorCode) = 0; }; } diff --git a/config/src/vespa/config/frt/CMakeLists.txt b/config/src/vespa/config/frt/CMakeLists.txt index 879b32684a7..9d0966bb09a 100644 --- a/config/src/vespa/config/frt/CMakeLists.txt +++ b/config/src/vespa/config/frt/CMakeLists.txt @@ -8,7 +8,9 @@ vespa_add_library(config_frt OBJECT frtconfigresponse.cpp frtsourcefactory.cpp frtconfigagent.cpp + frtconfigrequestv2.cpp frtconfigrequestfactory.cpp + frtconfigresponsev2.cpp protocol.cpp slimeconfigrequest.cpp slimeconfigresponse.cpp diff --git a/config/src/vespa/config/frt/frtconfigagent.cpp b/config/src/vespa/config/frt/frtconfigagent.cpp index e4bae234a01..ff16ef77a1b 100644 --- a/config/src/vespa/config/frt/frtconfigagent.cpp +++ b/config/src/vespa/config/frt/frtconfigagent.cpp @@ -1,6 +1,5 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "frtconfigagent.h" -#include "frtconfigrequestv3.h" #include <vespa/config/common/trace.h> #include <vespa/log/log.h> @@ -30,14 +29,14 @@ FRTConfigAgent::handleResponse(const ConfigRequest & request, ConfigResponse::UP LOG(spam, "current state for %s: generation %ld md5 %s", key.toString().c_str(), _configState.generation, _configState.md5.c_str()); } if (response->validateResponse() && !response->isError()) { - handleOKResponse(request, std::move(response)); + handleOKResponse(std::move(response)); } else { handleErrorResponse(request, std::move(response)); } } void -FRTConfigAgent::handleOKResponse(const ConfigRequest & request, ConfigResponse::UP response) +FRTConfigAgent::handleOKResponse(ConfigResponse::UP response) { _failedRequests = 0; response->fill(); @@ -46,7 +45,8 @@ FRTConfigAgent::handleOKResponse(const ConfigRequest & request, ConfigResponse:: } ConfigState newState = response->getConfigState(); - if ( ! request.verifyState(newState)) { + bool isNewGeneration = newState.isNewerGenerationThan(_configState); + if (isNewGeneration) { handleUpdatedGeneration(response->getKey(), newState, response->getValue()); } setWaitTime(_timingValues.successDelay, 1); @@ -59,15 +59,21 @@ FRTConfigAgent::handleUpdatedGeneration(const ConfigKey & key, const ConfigState if (LOG_WOULD_LOG(spam)) { LOG(spam, "new generation %ld for key %s", newState.generation, key.toString().c_str()); } - _latest = configValue; - + _configState.generation = newState.generation; + bool hasDifferentPayload = newState.hasDifferentPayloadFrom(_configState); + if (hasDifferentPayload) { + if (LOG_WOULD_LOG(spam)) { + LOG(spam, "new payload for key %s, existing md5(%s), new md5(%s)", key.toString().c_str(), _configState.md5.c_str(), newState.md5.c_str()); + } + _configState.md5 = newState.md5; + _latest = configValue; + } if (LOG_WOULD_LOG(spam)) { - LOG(spam, "updating holder for key %s,", key.toString().c_str()); + LOG(spam, "updating holder for key %s, payload changed: %d", key.toString().c_str(), hasDifferentPayload ? 1 : 0); } - _holder->handle(ConfigUpdate::UP(new ConfigUpdate(_latest, true, newState.generation))); + _holder->handle(ConfigUpdate::UP(new ConfigUpdate(_latest, hasDifferentPayload, _configState.generation))); _numConfigured++; - _configState = newState; } void diff --git a/config/src/vespa/config/frt/frtconfigagent.h b/config/src/vespa/config/frt/frtconfigagent.h index 97edefbded7..e3c362b5278 100644 --- a/config/src/vespa/config/frt/frtconfigagent.h +++ b/config/src/vespa/config/frt/frtconfigagent.h @@ -33,7 +33,7 @@ public: const ConfigState & getConfigState() const override; private: void handleUpdatedGeneration(const ConfigKey & key, const ConfigState & newState, const ConfigValue & configValue); - void handleOKResponse(const ConfigRequest & request, ConfigResponse::UP response); + void handleOKResponse(ConfigResponse::UP response); void handleErrorResponse(const ConfigRequest & request, ConfigResponse::UP response); void setWaitTime(uint64_t delay, int multiplier); diff --git a/config/src/vespa/config/frt/frtconfigrequest.cpp b/config/src/vespa/config/frt/frtconfigrequest.cpp index 0845f7408de..aa792126aca 100644 --- a/config/src/vespa/config/frt/frtconfigrequest.cpp +++ b/config/src/vespa/config/frt/frtconfigrequest.cpp @@ -2,7 +2,9 @@ #include "frtconfigrequest.h" #include "frtconfigresponse.h" #include "connection.h" -#include <vespa/fnet/frt/rpcrequest.h> +#include <vespa/fnet/frt/frt.h> +#include <vespa/config/common/configkey.h> +#include <vespa/config/common/configstate.h> namespace config { @@ -43,4 +45,52 @@ FRTConfigRequest::isAborted() const return (_request->GetErrorCode() == FRTE_RPC_ABORT); } +const vespalib::string FRTConfigRequestV1::REQUEST_TYPES = "sssssllsSi"; + +FRTConfigRequestV1::FRTConfigRequestV1(const ConfigKey & key, + Connection * connection, + const vespalib::string & configMd5, + int64_t generation, + int64_t serverTimeout) + : FRTConfigRequest(connection, key) +{ + _request->SetMethodName("config.v1.getConfig"); + _parameters.AddString(key.getDefName().c_str()); + _parameters.AddString(""); + _parameters.AddString(key.getDefMd5().c_str()); + _parameters.AddString(key.getConfigId().c_str()); + _parameters.AddString(configMd5.c_str()); + _parameters.AddInt64(generation); + _parameters.AddInt64(serverTimeout); + _parameters.AddString(key.getDefNamespace().c_str()); + const std::vector<vespalib::string> & schema(key.getDefSchema()); + FRT_StringValue * schemaValue = _parameters.AddStringArray(schema.size()); + for (size_t i = 0; i < schema.size(); i++) { + _parameters.SetString(&schemaValue[i], schema[i].c_str()); + } + _parameters.AddInt32(1); +} + +bool +FRTConfigRequestV1::verifyKey(const ConfigKey & key) const +{ + return (key.getDefName().compare(_parameters[0]._string._str) == 0 && + key.getDefNamespace().compare(_parameters[7]._string._str) == 0 && + key.getConfigId().compare(_parameters[3]._string._str) == 0 && + key.getDefMd5().compare(_parameters[2]._string._str) == 0); +} + +bool +FRTConfigRequestV1::verifyState(const ConfigState & state) const +{ + return (state.md5.compare(_parameters[4]._string._str) == 0 && + state.generation == static_cast<int64_t>(_parameters[5]._intval64)); +} + +ConfigResponse::UP +FRTConfigRequestV1::createResponse(FRT_RPCRequest * request) const +{ + return ConfigResponse::UP(new FRTConfigResponseV1(request)); +} + } // namespace config diff --git a/config/src/vespa/config/frt/frtconfigrequest.h b/config/src/vespa/config/frt/frtconfigrequest.h index 061151d5f39..e1d6b5590bd 100644 --- a/config/src/vespa/config/frt/frtconfigrequest.h +++ b/config/src/vespa/config/frt/frtconfigrequest.h @@ -22,6 +22,8 @@ public: typedef std::unique_ptr<FRTConfigRequest> UP; FRTConfigRequest(Connection * connection, const ConfigKey & key); ~FRTConfigRequest(); + virtual bool verifyKey(const ConfigKey & key) const = 0; + virtual bool verifyState(const ConfigState & state) const = 0; bool abort() override; bool isAborted() const override; @@ -31,11 +33,25 @@ public: FRT_RPCRequest* getRequest() { return _request; } virtual ConfigResponse::UP createResponse(FRT_RPCRequest * request) const = 0; protected: - FRT_RPCRequest * _request; - FRT_Values & _parameters; + FRT_RPCRequest *_request; + FRT_Values & _parameters; private: - Connection * _connection; - const ConfigKey _key; + Connection * _connection; + const ConfigKey _key; +}; + +class FRTConfigRequestV1 : public FRTConfigRequest { +public: + FRTConfigRequestV1(const ConfigKey & key, + Connection * connection, + const vespalib::string & configMd5, + int64_t generation, + int64_t serverTimeout); + bool verifyKey(const ConfigKey & key) const override; + bool verifyState(const ConfigState & state) const override; + ConfigResponse::UP createResponse(FRT_RPCRequest * request) const override; +private: + static const vespalib::string REQUEST_TYPES; }; } diff --git a/config/src/vespa/config/frt/frtconfigrequestfactory.cpp b/config/src/vespa/config/frt/frtconfigrequestfactory.cpp index 1f1ddb196b0..9fe3e073b65 100644 --- a/config/src/vespa/config/frt/frtconfigrequestfactory.cpp +++ b/config/src/vespa/config/frt/frtconfigrequestfactory.cpp @@ -1,9 +1,14 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "frtconfigrequestfactory.h" +#include "frtconfigrequest.h" +#include "frtconfigrequestv2.h" #include "frtconfigrequestv3.h" +#include <vespa/config/common/trace.h> +#include <vespa/config/common/compressiontype.h> #include <vespa/vespalib/util/host_name.h> +#include <unistd.h> +#include <limits.h> -using std::make_unique; namespace config { @@ -23,11 +28,14 @@ FRTConfigRequestFactory::~FRTConfigRequestFactory() { } FRTConfigRequest::UP -FRTConfigRequestFactory::createConfigRequest(const ConfigKey & key, Connection * connection, - const ConfigState & state, int64_t serverTimeout) const +FRTConfigRequestFactory::createConfigRequest(const ConfigKey & key, Connection * connection, const ConfigState & state, int64_t serverTimeout) const { - return make_unique<FRTConfigRequestV3>(connection, key, state.md5, state.generation, 0u, _hostName, - serverTimeout, Trace(_traceLevel), _vespaVersion, _compressionType); + if (1 == _protocolVersion) { + return FRTConfigRequest::UP(new FRTConfigRequestV1(key, connection, state.md5, state.generation, serverTimeout)); + } else if (2 == _protocolVersion) { + return FRTConfigRequest::UP(new FRTConfigRequestV2(connection, key, state.md5, state.generation, 0u, _hostName, serverTimeout, Trace(_traceLevel))); + } + return FRTConfigRequest::UP(new FRTConfigRequestV3(connection, key, state.md5, state.generation, 0u, _hostName, serverTimeout, Trace(_traceLevel), _vespaVersion, _compressionType)); } } // namespace config diff --git a/config/src/vespa/config/frt/frtconfigrequestv2.cpp b/config/src/vespa/config/frt/frtconfigrequestv2.cpp new file mode 100644 index 00000000000..ddc79b830b3 --- /dev/null +++ b/config/src/vespa/config/frt/frtconfigrequestv2.cpp @@ -0,0 +1,32 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include "frtconfigrequestv2.h" +#include "frtconfigresponsev2.h" +#include "connection.h" +#include <vespa/config/common/trace.h> +#include <vespa/config/common/vespa_version.h> + +using namespace config::protocol; + +namespace config { + +FRTConfigRequestV2::FRTConfigRequestV2(Connection * connection, + const ConfigKey & key, + const vespalib::string & configMd5, + int64_t currentGeneration, + int64_t wantedGeneration, + const vespalib::string & hostName, + int64_t serverTimeout, + const Trace & trace) + : SlimeConfigRequest(connection, key, configMd5, currentGeneration, wantedGeneration, hostName, serverTimeout, trace, VespaVersion::getCurrentVersion(), 2, CompressionType::UNCOMPRESSED, "config.v2.getConfig") +{ +} + + + +ConfigResponse::UP +FRTConfigRequestV2::createResponse(FRT_RPCRequest * request) const +{ + return ConfigResponse::UP(new FRTConfigResponseV2(request)); +} + +} diff --git a/config/src/vespa/config/frt/frtconfigrequestv2.h b/config/src/vespa/config/frt/frtconfigrequestv2.h new file mode 100644 index 00000000000..5f055153b8c --- /dev/null +++ b/config/src/vespa/config/frt/frtconfigrequestv2.h @@ -0,0 +1,29 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include "slimeconfigrequest.h" + +class FRT_Values; +class FRT_RPCRequest; + +namespace config { + +class ConfigKey; +class Connection; +class Trace; + +class FRTConfigRequestV2 : public SlimeConfigRequest { +public: + FRTConfigRequestV2(Connection * connection, + const ConfigKey & key, + const vespalib::string & configMd5, + int64_t currentGeneration, + int64_t wantedGeneration, + const vespalib::string & hostName, + int64_t serverTimeout, + const Trace & trace); + ConfigResponse::UP createResponse(FRT_RPCRequest * request) const override; +}; + +} + diff --git a/config/src/vespa/config/frt/frtconfigresponse.cpp b/config/src/vespa/config/frt/frtconfigresponse.cpp index 046416bcfd6..98922ad4729 100644 --- a/config/src/vespa/config/frt/frtconfigresponse.cpp +++ b/config/src/vespa/config/frt/frtconfigresponse.cpp @@ -1,7 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "frtconfigresponse.h" -#include <vespa/fnet/frt/rpcrequest.h> +#include <vespa/fnet/frt/frt.h> namespace config { @@ -42,4 +42,52 @@ vespalib::string FRTConfigResponse::errorMessage() const { return _request->GetE int FRTConfigResponse::errorCode() const { return _request->GetErrorCode(); } bool FRTConfigResponse::isError() const { return _request->IsError(); } +// +// V1 Implementation +// +const vespalib::string FRTConfigResponseV1::RESPONSE_TYPES = "sssssilSs"; + +FRTConfigResponseV1::FRTConfigResponseV1(FRT_RPCRequest * request) + : FRTConfigResponse(request), + _key(), + _value() +{ +} + +FRTConfigResponseV1::~FRTConfigResponseV1() {} + +const vespalib::string & +FRTConfigResponseV1::getResponseTypes() const +{ + return RESPONSE_TYPES; +} + +void +FRTConfigResponseV1::fill() +{ + const std::vector<vespalib::string> payload(getPayLoad()); + _value = ConfigValue(payload, calculateContentMd5(payload)); + _key = readKey(); + _state = ConfigState(vespalib::string((*_returnValues)[4]._string._str), (*_returnValues)[6]._intval64); +} + +const ConfigKey +FRTConfigResponseV1::readKey() const +{ + return ConfigKey((*_returnValues)[3]._string._str, (*_returnValues)[0]._string._str, (*_returnValues)[8]._string._str, (*_returnValues)[2]._string._str); +} + +const std::vector<vespalib::string> +FRTConfigResponseV1::getPayLoad() const +{ + uint32_t numStrings = (*_returnValues)[7]._string_array._len; + FRT_StringValue *s = (*_returnValues)[7]._string_array._pt; + std::vector<vespalib::string> payload; + payload.reserve(numStrings); + for (uint32_t i = 0; i < numStrings; i++) { + payload.push_back(vespalib::string(s[i]._str)); + } + return payload; +} + } // namespace config diff --git a/config/src/vespa/config/frt/frtconfigresponse.h b/config/src/vespa/config/frt/frtconfigresponse.h index 31811c6a38f..f829044e698 100644 --- a/config/src/vespa/config/frt/frtconfigresponse.h +++ b/config/src/vespa/config/frt/frtconfigresponse.h @@ -39,5 +39,33 @@ protected: FRT_Values * _returnValues; }; +class FRTConfigResponseV1 : public FRTConfigResponse { +private: + FRTConfigResponseV1& operator=(const FRTConfigResponseV1&); +public: + FRTConfigResponseV1(FRT_RPCRequest * request); + ~FRTConfigResponseV1(); + + const ConfigKey & getKey() const override { return _key; } + const ConfigValue & getValue() const override { return _value; } + const Trace & getTrace() const override { return _trace; } + + const ConfigState & getConfigState() const override { return _state; } + + void fill() override; + +private: + static const vespalib::string RESPONSE_TYPES; + + const std::vector<vespalib::string> getPayLoad() const; + const ConfigKey readKey() const; + const vespalib::string & getResponseTypes() const override; + + ConfigKey _key; + ConfigValue _value; + ConfigState _state; + Trace _trace; +}; + } // namespace config diff --git a/config/src/vespa/config/frt/frtconfigresponsev2.cpp b/config/src/vespa/config/frt/frtconfigresponsev2.cpp new file mode 100644 index 00000000000..1527700f99f --- /dev/null +++ b/config/src/vespa/config/frt/frtconfigresponsev2.cpp @@ -0,0 +1,44 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include "frtconfigresponsev2.h" +#include <vespa/fnet/frt/frt.h> + +using namespace vespalib; +using namespace vespalib::slime; +using namespace vespalib::slime::convenience; +using namespace config::protocol::v2; + +namespace config { + +class V2Payload : public protocol::Payload { +public: + V2Payload(const SlimePtr & data) + : _data(data) + {} + const Inspector & getSlimePayload() const override { + return extractPayload(*_data); + } +private: + SlimePtr _data; +}; + +const vespalib::string FRTConfigResponseV2::RESPONSE_TYPES = "s"; + +FRTConfigResponseV2::FRTConfigResponseV2(FRT_RPCRequest * request) + : SlimeConfigResponse(request) +{ +} + +const vespalib::string & +FRTConfigResponseV2::getResponseTypes() const +{ + return RESPONSE_TYPES; +} + +const ConfigValue +FRTConfigResponseV2::readConfigValue() const +{ + vespalib::string md5(_data->get()[RESPONSE_CONFIG_MD5].asString().make_string()); + return ConfigValue(PayloadPtr(new V2Payload(_data)), md5); +} + +} // namespace config diff --git a/config/src/vespa/config/frt/frtconfigresponsev2.h b/config/src/vespa/config/frt/frtconfigresponsev2.h new file mode 100644 index 00000000000..89d1a9157e3 --- /dev/null +++ b/config/src/vespa/config/frt/frtconfigresponsev2.h @@ -0,0 +1,28 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include "slimeconfigresponse.h" +#include <vespa/config/common/configvalue.h> + +class FRT_RPCRequest; +class FRT_Values; + +namespace config { + +/** + * Baseclass for config responses. + */ +class FRTConfigResponseV2 : public SlimeConfigResponse { +private: + FRTConfigResponseV2& operator=(const FRTConfigResponseV2&); +public: + FRTConfigResponseV2(FRT_RPCRequest * request); + +private: + static const vespalib::string RESPONSE_TYPES; + const vespalib::string & getResponseTypes() const override; + const ConfigValue readConfigValue() const override; +}; + +} // namespace config + diff --git a/config/src/vespa/config/frt/slimeconfigrequest.cpp b/config/src/vespa/config/frt/slimeconfigrequest.cpp index 27ac39ae56a..12dbbab2eb7 100644 --- a/config/src/vespa/config/frt/slimeconfigrequest.cpp +++ b/config/src/vespa/config/frt/slimeconfigrequest.cpp @@ -40,10 +40,19 @@ SlimeConfigRequest::SlimeConfigRequest(Connection * connection, } bool +SlimeConfigRequest::verifyKey(const ConfigKey & key) const +{ + return (key.getDefName().compare(_parameters[0]._string._str) == 0 && + key.getDefNamespace().compare(_parameters[7]._string._str) == 0 && + key.getConfigId().compare(_parameters[3]._string._str) == 0 && + key.getDefMd5().compare(_parameters[2]._string._str) == 0); +} + +bool SlimeConfigRequest::verifyState(const ConfigState & state) const { - return (state.md5.compare(_data[REQUEST_CONFIG_MD5].asString().make_stringref()) == 0 && - state.generation == _data[REQUEST_CURRENT_GENERATION].asLong()); + return (state.md5.compare(_parameters[4]._string._str) == 0 && + state.generation == static_cast<int64_t>(_parameters[5]._intval64)); } void diff --git a/config/src/vespa/config/frt/slimeconfigrequest.h b/config/src/vespa/config/frt/slimeconfigrequest.h index 625d70c094b..d616876780e 100644 --- a/config/src/vespa/config/frt/slimeconfigrequest.h +++ b/config/src/vespa/config/frt/slimeconfigrequest.h @@ -30,6 +30,7 @@ public: const CompressionType & compressionType, const vespalib::string & methodName); ~SlimeConfigRequest() {} + bool verifyKey(const ConfigKey & key) const override; bool verifyState(const ConfigState & state) const override; virtual ConfigResponse::UP createResponse(FRT_RPCRequest * request) const override = 0; private: diff --git a/configdefinitions/src/vespa/configserver.def b/configdefinitions/src/vespa/configserver.def index c13665342ef..3c99875f978 100644 --- a/configdefinitions/src/vespa/configserver.def +++ b/configdefinitions/src/vespa/configserver.def @@ -1,5 +1,6 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. namespace=cloud.config + rpcport int default=19070 httpport int default=19071 numthreads int default=16 @@ -43,4 +44,7 @@ dockerRegistry string default="" dockerVespaBaseImage string default="" # Athenz config -loadBalancerAddress string default=""
\ No newline at end of file +loadBalancerAddress string default="" + +# File distributions +disableFiledistributor bool default=false diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDirectory.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDirectory.java index 7d0ba6cd9bd..d8a560c6159 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDirectory.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDirectory.java @@ -5,16 +5,21 @@ package com.yahoo.vespa.config.server.filedistribution; import com.yahoo.config.FileReference; import com.yahoo.config.model.api.FileDistribution; import com.yahoo.io.IOUtils; +import com.yahoo.log.LogLevel; import com.yahoo.text.Utf8; import net.jpountz.xxhash.XXHash64; import net.jpountz.xxhash.XXHashFactory; import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; import java.io.FilenameFilter; import java.io.IOException; import java.nio.ByteBuffer; +import java.nio.channels.FileChannel; import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.attribute.BasicFileAttributes; import java.util.logging.Logger; public class FileDirectory { @@ -94,6 +99,7 @@ public class FileDirectory { public FileReference addFile(File source, FileReference reference) { ensureRootExist(); try { + logfileInfo(source); File destinationDir = new File(root, reference.value()); if (!destinationDir.exists()) { destinationDir.mkdir(); @@ -102,7 +108,7 @@ public class FileDirectory { if (source.isDirectory()) IOUtils.copyDirectory(source, destination); else - IOUtils.copy(source, destination); + copyFile(source, destination); if (!destinationDir.exists()) { if ( ! tempDestinationDir.toFile().renameTo(destinationDir)) { log.warning("Failed moving '" + tempDestinationDir.toFile().getAbsolutePath() + "' to '" + destination.getAbsolutePath() + "'."); @@ -117,4 +123,18 @@ public class FileDirectory { throw new IllegalArgumentException(e); } } + + private void logfileInfo(File file ) throws IOException { + BasicFileAttributes basicFileAttributes = Files.readAttributes(file.toPath(), BasicFileAttributes.class); + log.log(LogLevel.DEBUG, "Adding file " + file.getAbsolutePath() + " (created " + basicFileAttributes.creationTime() + + ", modified " + basicFileAttributes.lastModifiedTime() + + ", size " + basicFileAttributes.size() + ")"); + } + + private static void copyFile(File source, File dest) throws IOException { + try (FileChannel sourceChannel = new FileInputStream(source).getChannel(); + FileChannel destChannel = new FileOutputStream(dest).getChannel()) { + destChannel.transferFrom(sourceChannel, 0, sourceChannel.size()); + } + } } diff --git a/configserver/src/main/sh/start-filedistribution b/configserver/src/main/sh/start-filedistribution index bb8599f2bc9..1995e59de4e 100755 --- a/configserver/src/main/sh/start-filedistribution +++ b/configserver/src/main/sh/start-filedistribution @@ -63,7 +63,7 @@ ROOT=${VESPA_HOME%/} VESPA_CONFIG_ID="dir:${ROOT}/conf/filedistributor" export VESPA_CONFIG_ID -if [ "$multitenant" = "true" ]; then +if [ "$multitenant" = "true" ] && [ "$disable_filedistributor" = "" ] || [ "$disable_filedistributor" = "false" ]; then foo=`${ROOT}/libexec/vespa/vespa-config.pl -mkfiledistributorconfig` PIDFILE_FILEDISTRIBUTOR=var/run/filedistributor.pid LOGFILE="${ROOT}/logs/vespa/vespa.log" diff --git a/container-dependencies-enforcer/pom.xml b/container-dependencies-enforcer/pom.xml index 1b0a7f9e958..61495384744 100644 --- a/container-dependencies-enforcer/pom.xml +++ b/container-dependencies-enforcer/pom.xml @@ -62,8 +62,9 @@ <rules> <bannedDependencies> <excludes> - <!-- Only allow explicitly listed deps in provided scope --> + <!-- Only allow explicitly listed deps in provided and compile scope --> <exclude>*:*:*:jar:provided:*</exclude> + <exclude>*:*:*:jar:compile:*</exclude> </excludes> <includes> <include>com.yahoo.vespa</include> diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index d7324450d4c..82c607b89fc 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -21,7 +21,6 @@ import com.yahoo.slime.Cursor; import com.yahoo.slime.Inspector; import com.yahoo.slime.Slime; import com.yahoo.vespa.config.SlimeUtils; -import com.yahoo.vespa.curator.Lock; import com.yahoo.vespa.hosted.controller.AlreadyExistsException; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Controller; @@ -53,7 +52,6 @@ import com.yahoo.vespa.hosted.controller.api.identifiers.UserGroup; import com.yahoo.vespa.hosted.controller.api.identifiers.UserId; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Log; -import com.yahoo.vespa.hosted.controller.api.integration.organization.IssueId; import com.yahoo.vespa.hosted.controller.api.integration.organization.User; import com.yahoo.vespa.hosted.controller.api.integration.routing.RotationStatus; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; @@ -783,17 +781,17 @@ public class ApplicationApiHandler extends LoggingRequestHandler { Inspector deployOptions = SlimeUtils.jsonToSlime(dataParts.get("deployOptions")).get(); + ApplicationPackage applicationPackage = new ApplicationPackage(dataParts.get("applicationZip")); DeployAuthorizer deployAuthorizer = new DeployAuthorizer(controller.zoneRegistry(), athenzClientFactory); Tenant tenant = controller.tenants().tenant(new TenantId(tenantName)).orElseThrow(() -> new NotExistsException(new TenantId(tenantName))); Principal principal = authorizer.getPrincipal(request); - deployAuthorizer.throwIfUnauthorizedForDeploy(principal, Environment.from(environment), tenant, applicationId); + deployAuthorizer.throwIfUnauthorizedForDeploy(principal, Environment.from(environment), tenant, applicationId, applicationPackage); // TODO: get rid of the json object DeployOptions deployOptionsJsonClass = new DeployOptions(screwdriverBuildJobFromSlime(deployOptions.field("screwdriverBuildJob")), optional("vespaVersion", deployOptions).map(Version::new), deployOptions.field("ignoreValidationErrors").asBool(), deployOptions.field("deployCurrentVersion").asBool()); - ApplicationPackage applicationPackage = new ApplicationPackage(dataParts.get("applicationZip")); controller.applications().validate(applicationPackage.deploymentSpec()); ActivateResult result = controller.applications().deployApplication(applicationId, zone, diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/DeployAuthorizer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/DeployAuthorizer.java index 71126259417..2e627676766 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/DeployAuthorizer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/DeployAuthorizer.java @@ -6,6 +6,7 @@ import com.yahoo.config.provision.Environment; import com.yahoo.vespa.hosted.controller.api.Tenant; import com.yahoo.vespa.hosted.controller.api.identifiers.AthenzDomain; import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry; +import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; import com.yahoo.vespa.hosted.controller.athenz.ApplicationAction; import com.yahoo.vespa.hosted.controller.athenz.AthenzClientFactory; import com.yahoo.vespa.hosted.controller.athenz.AthenzPrincipal; @@ -15,6 +16,7 @@ import com.yahoo.vespa.hosted.controller.athenz.ZmsException; import javax.ws.rs.ForbiddenException; import javax.ws.rs.NotAuthorizedException; import java.security.Principal; +import java.util.Objects; import java.util.logging.Logger; import static com.yahoo.vespa.hosted.controller.restapi.application.Authorizer.environmentRequiresAuthorization; @@ -38,7 +40,21 @@ public class DeployAuthorizer { public void throwIfUnauthorizedForDeploy(Principal principal, Environment environment, Tenant tenant, - ApplicationId applicationId) { + ApplicationId applicationId, + ApplicationPackage applicationPackage) { + // Validate that domain in identity configuration (deployment.xml) is same as tenant domain + applicationPackage.deploymentSpec().athenzDomain().ifPresent(identityDomain -> { + AthenzDomain tenantDomain = tenant.getAthensDomain().orElseThrow(() -> new IllegalArgumentException("Identity provider only available to Athenz onboarded tenants")); + if (! Objects.equals(tenantDomain.id(), identityDomain.value())) { + throw new ForbiddenException( + String.format( + "Athenz domain in deployment.xml: [%s] must match tenant domain: [%s]", + identityDomain.value(), + tenantDomain.id() + )); + } + }); + if (!environmentRequiresAuthorization(environment)) { return; } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java index 72bfa238094..a2b24864d1e 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java @@ -2,6 +2,8 @@ package com.yahoo.vespa.hosted.controller.deployment; import com.yahoo.config.application.api.ValidationId; +import com.yahoo.config.provision.AthenzDomain; +import com.yahoo.config.provision.AthenzService; import com.yahoo.config.provision.Environment; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; @@ -29,6 +31,7 @@ public class ApplicationPackageBuilder { private final StringBuilder environmentBody = new StringBuilder(); private final StringBuilder validationOverridesBody = new StringBuilder(); private final StringBuilder blockChange = new StringBuilder(); + private String athenzIdentityAttributes = null; private String searchDefinition = "search test { }"; public ApplicationPackageBuilder upgradePolicy(String upgradePolicy) { @@ -83,6 +86,11 @@ public class ApplicationPackageBuilder { return this; } + public ApplicationPackageBuilder athenzIdentity(AthenzDomain domain, AthenzService service) { + this.athenzIdentityAttributes = String.format("athenz-domain='%s' athenz-service='%s'", domain.value(), service.value()); + return this; + } + /** Sets the content of the search definition test.sd */ public ApplicationPackageBuilder searchDefinition(String testSearchDefinition) { this.searchDefinition = testSearchDefinition; @@ -90,7 +98,12 @@ public class ApplicationPackageBuilder { } private byte[] deploymentSpec() { - StringBuilder xml = new StringBuilder("<deployment version='1.0'>\n"); + StringBuilder xml = new StringBuilder(); + xml.append("<deployment version='1.0' "); + if(athenzIdentityAttributes != null) { + xml.append(athenzIdentityAttributes); + } + xml.append(">\n"); if (upgradePolicy != null) { xml.append("<upgrade policy='"); xml.append(upgradePolicy); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerControllerTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerControllerTester.java index 6c5120df515..71ad6560126 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerControllerTester.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerControllerTester.java @@ -23,9 +23,11 @@ import com.yahoo.vespa.hosted.controller.api.identifiers.TenantId; import com.yahoo.vespa.hosted.controller.api.identifiers.UserId; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.DeploymentJobs; +import com.yahoo.vespa.hosted.controller.athenz.ApplicationAction; import com.yahoo.vespa.hosted.controller.athenz.AthenzPrincipal; -import com.yahoo.vespa.hosted.controller.athenz.mock.AthenzDbMock; +import com.yahoo.vespa.hosted.controller.athenz.AthenzUtils; import com.yahoo.vespa.hosted.controller.athenz.mock.AthenzClientFactoryMock; +import com.yahoo.vespa.hosted.controller.athenz.mock.AthenzDbMock; import com.yahoo.vespa.hosted.controller.maintenance.JobControl; import com.yahoo.vespa.hosted.controller.maintenance.Upgrader; import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; @@ -121,4 +123,16 @@ public class ContainerControllerTester { containerTester.assertResponse(request, expectedResponse, expectedStatusCode); } + /* + * Authorize action on tenantDomain/application for a given screwdriverId + */ + public void authorize(AthenzDomain tenantDomain, ScrewdriverId screwdriverId, ApplicationAction action, Application application) { + AthenzClientFactoryMock mock = (AthenzClientFactoryMock) containerTester.container().components() + .getComponent(AthenzClientFactoryMock.class.getName()); + + mock.getSetup() + .domains.get(tenantDomain) + .applications.get(new com.yahoo.vespa.hosted.controller.api.identifiers.ApplicationId(application.id().application().value())) + .addRoleMember(action, AthenzUtils.createPrincipal(screwdriverId)); + } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java index bf4586f9fd0..7902e49288c 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java @@ -3,12 +3,14 @@ package com.yahoo.vespa.hosted.controller.restapi.application; import com.yahoo.application.container.handler.Request; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.AthenzService; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Environment; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.ConfigServerClientMock; import com.yahoo.vespa.hosted.controller.api.identifiers.AthenzDomain; import com.yahoo.vespa.hosted.controller.api.identifiers.PropertyId; +import com.yahoo.vespa.hosted.controller.api.identifiers.ScrewdriverId; import com.yahoo.vespa.hosted.controller.api.identifiers.UserId; import com.yahoo.vespa.hosted.controller.api.integration.MetricsService.ApplicationMetrics; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException; @@ -21,10 +23,11 @@ import com.yahoo.vespa.hosted.controller.application.ClusterUtilization; import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.DeploymentJobs; import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics; +import com.yahoo.vespa.hosted.controller.athenz.ApplicationAction; import com.yahoo.vespa.hosted.controller.athenz.AthenzPrincipal; import com.yahoo.vespa.hosted.controller.athenz.AthenzUtils; -import com.yahoo.vespa.hosted.controller.athenz.mock.AthenzDbMock; import com.yahoo.vespa.hosted.controller.athenz.mock.AthenzClientFactoryMock; +import com.yahoo.vespa.hosted.controller.athenz.mock.AthenzDbMock; import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; import com.yahoo.vespa.hosted.controller.restapi.ContainerControllerTester; import com.yahoo.vespa.hosted.controller.restapi.ContainerTester; @@ -598,6 +601,55 @@ public class ApplicationApiTest extends ControllerContainerTest { 403); } + @Test + public void deployment_fails_on_illegal_domain_in_deployment_spec() throws IOException { + ContainerControllerTester controllerTester = new ContainerControllerTester(container, responseFiles); + ContainerTester tester = controllerTester.containerTester(); + ApplicationPackage applicationPackage = new ApplicationPackageBuilder() + .upgradePolicy("default") + .athenzIdentity(com.yahoo.config.provision.AthenzDomain.from("invalid.domain"), AthenzService.from("service")) + .environment(Environment.prod) + .region("us-west-1") + .build(); + long screwdriverProjectId = 123; + AthenzDomain domain = addTenantAthenzDomain(athenzUserDomain, "mytenant"); + + Application application = controllerTester.createApplication(athenzUserDomain, "tenant1", "application1"); + controllerTester.authorize(domain, new ScrewdriverId(Long.toString(screwdriverProjectId)), ApplicationAction.deploy, application); + + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/test/region/us-east-1/instance/default/", POST) + .data(createApplicationDeployData(applicationPackage, Optional.of(screwdriverProjectId))) + .domain(athenzScrewdriverDomain).user("sd" + screwdriverProjectId), + "{\"error-code\":\"FORBIDDEN\",\"message\":\"Athenz domain in deployment.xml: [invalid.domain] must match tenant domain: [domain1]\"}", + 403); + + } + + @Test + public void deployment_succeeds_when_correct_domain_is_used() throws IOException { + ContainerControllerTester controllerTester = new ContainerControllerTester(container, responseFiles); + ContainerTester tester = controllerTester.containerTester(); + ApplicationPackage applicationPackage = new ApplicationPackageBuilder() + .upgradePolicy("default") + .athenzIdentity(com.yahoo.config.provision.AthenzDomain.from("domain1"), AthenzService.from("service")) + .environment(Environment.prod) + .region("us-west-1") + .build(); + long screwdriverProjectId = 123; + AthenzDomain domain = addTenantAthenzDomain(athenzUserDomain, "mytenant"); + + Application application = controllerTester.createApplication(athenzUserDomain, "tenant1", "application1"); + controllerTester.authorize(domain, new ScrewdriverId(Long.toString(screwdriverProjectId)), ApplicationAction.deploy, application); + + // Allow systemtest to succeed by notifying completion of system test + controllerTester.notifyJobCompletion(application.id(), screwdriverProjectId, true, DeploymentJobs.JobType.component); + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/test/region/us-east-1/instance/default/", POST) + .data(createApplicationDeployData(applicationPackage, Optional.of(screwdriverProjectId))) + .domain(athenzScrewdriverDomain).user("sd" + screwdriverProjectId), + new File("deploy-result.json")); + + } + private HttpEntity createApplicationDeployData(ApplicationPackage applicationPackage, Optional<Long> screwdriverJobId) { MultipartEntityBuilder builder = MultipartEntityBuilder.create(); builder.addTextBody("deployOptions", deployOptions(screwdriverJobId), ContentType.APPLICATION_JSON); diff --git a/filedistribution/pom.xml b/filedistribution/pom.xml index 41622792e43..10b77b540e7 100644 --- a/filedistribution/pom.xml +++ b/filedistribution/pom.xml @@ -19,11 +19,6 @@ <dependencies> <dependency> <groupId>com.yahoo.vespa</groupId> - <artifactId>config-lib</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> <artifactId>vespajlib</artifactId> <version>${project.version}</version> </dependency> diff --git a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/RpcTester.java b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/RpcTester.java new file mode 100644 index 00000000000..28935c203fe --- /dev/null +++ b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/RpcTester.java @@ -0,0 +1,98 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.vespa.filedistribution; + +import com.yahoo.config.FileReference; +import com.yahoo.io.IOUtils; +import com.yahoo.jrt.DataValue; +import com.yahoo.jrt.Int32Value; +import com.yahoo.jrt.Int64Value; +import com.yahoo.jrt.Request; +import com.yahoo.jrt.Spec; +import com.yahoo.jrt.StringValue; +import com.yahoo.jrt.Supervisor; +import com.yahoo.jrt.Target; +import com.yahoo.jrt.Transport; +import com.yahoo.log.LogLevel; +import net.jpountz.xxhash.XXHash64; +import net.jpountz.xxhash.XXHashFactory; + +import java.io.File; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.logging.Logger; + +public class RpcTester { + + private static final Logger log = Logger.getLogger(RpcTester.class.getName()); + + private final Target target; + + private RpcTester(Target target) { + this.target = target; + } + + private void call(String fileReference, String filename, byte[] blob) { + new FileReceiver(target).receive(new FileReference(fileReference), filename, blob); + } + + public static void main(String[] args) { + //String fileReference = args[0]; + String fileReference = "59f93f445438c9db7ccbf1629f583c2aa004a68b"; + String filename = "com.yahoo.vespatest.ExtraHitSearcher-1.0.0-deploy.jar"; + File file = new File(String.format("/tmp/%s/%s", fileReference, filename)); + byte[] blob = null; + + try { + blob = IOUtils.readFileBytes(file); + } catch (IOException e) { + e.printStackTrace(); + } + + log.log(LogLevel.INFO, "Read blob from " + file.getAbsolutePath()); + + + Supervisor supervisor = new Supervisor(new Transport()); + + Spec spec = new Spec("tcp/localhost:19090"); + log.log(LogLevel.INFO, "Connecting to " + spec); + Target target = supervisor.connect(spec); + if (! target.isValid()) { + log.log(LogLevel.INFO, "Could not connect"); + System.exit(1); + } else { + log.log(LogLevel.INFO, "Connected to " + spec); + } + + new RpcTester(target).call(fileReference, filename, blob); + } + + class FileReceiver { + + Target target; + + FileReceiver(Target target) { + this.target = target; + } + + void receive(FileReference reference, String filename, byte[] content) { + + log.log(LogLevel.INFO, "Preparing receive call for " + reference.value() + " and file " + filename); + + XXHash64 hasher = XXHashFactory.fastestInstance().hash64(); + Request fileBlob = new Request("filedistribution.receiveFile"); + + log.log(LogLevel.INFO, "Calling " + fileBlob.methodName() + " with target " + target); + + fileBlob.parameters().add(new StringValue(reference.value())); + fileBlob.parameters().add(new StringValue(filename)); + fileBlob.parameters().add(new DataValue(content)); + fileBlob.parameters().add(new Int64Value(hasher.hash(ByteBuffer.wrap(content), 0))); + fileBlob.parameters().add(new Int32Value(0)); + fileBlob.parameters().add(new StringValue("OK")); + log.log(LogLevel.INFO, "Doing invokeSync"); + target.invokeSync(fileBlob, 5); + log.log(LogLevel.INFO, "Done with invokeSync"); + } + } +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java index a3a647e1d14..763291bf86b 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java @@ -139,7 +139,7 @@ public class StorageMaintainer { Process duCommand = new ProcessBuilder().command(command).start(); if (!duCommand.waitFor(60, TimeUnit.SECONDS)) { duCommand.destroy(); - throw new RuntimeException("Disk usage command timedout, aborting."); + throw new RuntimeException("Disk usage command timed out, aborting."); } String output = IOUtils.readAll(new InputStreamReader(duCommand.getInputStream())); String[] results = output.split("\t"); diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java index 09eb14039e8..d01692f1f05 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java @@ -524,6 +524,7 @@ public class NodeAgentImpl implements NodeAgent { Docker.ContainerStats stats = containerStats.get(); final String APP = MetricReceiverWrapper.APPLICATION_NODE; final int totalNumCpuCores = ((List<Number>) ((Map) stats.getCpuStats().get("cpu_usage")).get("percpu_usage")).size(); + final long cpuContainerKernelTime = ((Number) ((Map) stats.getCpuStats().get("cpu_usage")).get("usage_in_kernelmode")).longValue(); final long cpuContainerTotalTime = ((Number) ((Map) stats.getCpuStats().get("cpu_usage")).get("total_usage")).longValue(); final long cpuSystemTotalTime = ((Number) stats.getCpuStats().get("system_cpu_usage")).longValue(); final long memoryTotalBytes = ((Number) stats.getMemoryStats().get("limit")).longValue(); @@ -532,26 +533,28 @@ public class NodeAgentImpl implements NodeAgent { final long diskTotalBytes = (long) (nodeSpec.minDiskAvailableGb * BYTES_IN_GB); final Optional<Long> diskTotalBytesUsed = storageMaintainer.getDiskUsageFor(containerName); - // CPU usage by a container as percentage of total host CPU, cpuPercentageOfHost, is given by dividing used - // CPU time by the container with CPU time used by the entire system. - // CPU usage by a container as percentage of total CPU allocated to it is given by dividing the - // cpuPercentageOfHost with the ratio of container minCpuCores by total number of CPU cores. - double cpuPercentageOfHost = lastCpuMetric.getCpuUsagePercentage(cpuContainerTotalTime, cpuSystemTotalTime); - double cpuPercentageOfAllocated = totalNumCpuCores * cpuPercentageOfHost / nodeSpec.minCpuCores; + lastCpuMetric.updateCpuDeltas(cpuSystemTotalTime, cpuContainerTotalTime, cpuContainerKernelTime); + + // Ratio of CPU cores allocated to this container to total number of CPU cores on this host + final double allocatedCpuRatio = nodeSpec.minCpuCores / totalNumCpuCores; + double cpuUsageRatioOfAllocated = lastCpuMetric.getCpuUsageRatio() / allocatedCpuRatio; + double cpuKernelUsageRatioOfAllocated = lastCpuMetric.getCpuKernelUsageRatio() / allocatedCpuRatio; + long memoryTotalBytesUsed = memoryTotalBytesUsage - memoryTotalBytesCache; - double memoryPercentUsed = 100.0 * memoryTotalBytesUsed / memoryTotalBytes; - Optional<Double> diskPercentUsed = diskTotalBytesUsed.map(used -> 100.0 * used / diskTotalBytes); + double memoryUsageRatio = (double) memoryTotalBytesUsed / memoryTotalBytes; + Optional<Double> diskUsageRatio = diskTotalBytesUsed.map(used -> (double) used / diskTotalBytes); List<DimensionMetrics> metrics = new ArrayList<>(); DimensionMetrics.Builder systemMetricsBuilder = new DimensionMetrics.Builder(APP, dimensions) .withMetric("mem.limit", memoryTotalBytes) .withMetric("mem.used", memoryTotalBytesUsed) - .withMetric("mem.util", memoryPercentUsed) - .withMetric("cpu.util", cpuPercentageOfAllocated) + .withMetric("mem.util", 100 * memoryUsageRatio) + .withMetric("cpu.util", 100 * cpuUsageRatioOfAllocated) + .withMetric("cpu.sys.util", 100 * cpuKernelUsageRatioOfAllocated) .withMetric("disk.limit", diskTotalBytes); diskTotalBytesUsed.ifPresent(diskUsed -> systemMetricsBuilder.withMetric("disk.used", diskUsed)); - diskPercentUsed.ifPresent(diskUtil -> systemMetricsBuilder.withMetric("disk.util", diskUtil)); + diskUsageRatio.ifPresent(diskRatio -> systemMetricsBuilder.withMetric("disk.util", 100 * diskRatio)); metrics.add(systemMetricsBuilder.build()); stats.getNetworks().forEach((interfaceName, interfaceStats) -> { @@ -612,17 +615,35 @@ public class NodeAgentImpl implements NodeAgent { } class CpuUsageReporter { + private long containerKernelUsage = 0; private long totalContainerUsage = 0; private long totalSystemUsage = 0; - double getCpuUsagePercentage(long currentContainerUsage, long currentSystemUsage) { - long deltaSystemUsage = currentSystemUsage - totalSystemUsage; - double cpuUsagePct = (deltaSystemUsage == 0 || totalSystemUsage == 0) ? - 0 : 100.0 * (currentContainerUsage - totalContainerUsage) / deltaSystemUsage; + private long deltaContainerKernelUsage; + private long deltaContainerUsage; + private long deltaSystemUsage; + + private void updateCpuDeltas(long totalSystemUsage, long totalContainerUsage, long containerKernelUsage) { + deltaSystemUsage = totalSystemUsage - this.totalSystemUsage; + deltaContainerUsage = totalContainerUsage - this.totalContainerUsage; + deltaContainerKernelUsage = containerKernelUsage - this.containerKernelUsage; + + this.totalSystemUsage = totalSystemUsage; + this.totalContainerUsage = totalContainerUsage; + this.containerKernelUsage = containerKernelUsage; + } + + /** + * Returns the CPU usage ratio for the docker container that this NodeAgent is managing + * in the time between the last two times updateCpuDeltas() was called. This is calculated + * by dividing the CPU time used by the container with the CPU time used by the entire system. + */ + double getCpuUsageRatio() { + return deltaSystemUsage == 0 ? 0 : (double) deltaContainerUsage / deltaSystemUsage; + } - totalContainerUsage = currentContainerUsage; - totalSystemUsage = currentSystemUsage; - return cpuUsagePct; + double getCpuKernelUsageRatio() { + return deltaSystemUsage == 0 ? 0 : (double) deltaContainerKernelUsage / deltaSystemUsage; } } diff --git a/node-admin/src/test/resources/docker.stats.json b/node-admin/src/test/resources/docker.stats.json index 3b1087b9202..ff4a2fde943 100644 --- a/node-admin/src/test/resources/docker.stats.json +++ b/node-admin/src/test/resources/docker.stats.json @@ -36,7 +36,7 @@ 44567860460, 39049895962 ], - "usage_in_kernelmode":44050000000, + "usage_in_kernelmode":44106083850, "usage_in_usermode":158950000000 }, "system_cpu_usage":5876882680000000, diff --git a/node-admin/src/test/resources/expected.container.system.metrics.txt b/node-admin/src/test/resources/expected.container.system.metrics.txt index 8a4d696b08e..023d3958c60 100644 --- a/node-admin/src/test/resources/expected.container.system.metrics.txt +++ b/node-admin/src/test/resources/expected.container.system.metrics.txt @@ -11,6 +11,7 @@ s: "mem.limit": 4294967296, "mem.used": 1073741824, "disk.used": 39625000000, + "cpu.sys.util": 3.402, "disk.util": 15.85, "cpu.util": 5.4, "mem.util": 25.0, diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java index bcc18cfd876..7de138fa954 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java @@ -45,8 +45,12 @@ public class MockNodeRepository extends NodeRepository { */ public MockNodeRepository(MockCurator curator, NodeFlavors flavors) throws Exception { super(flavors, curator, Clock.fixed(Instant.ofEpochMilli(123), ZoneId.of("Z")), Zone.defaultZone(), - new MockNameResolver().mockAnyLookup(), new DockerImage("docker-registry.domain.tld:8080/dist/vespa")); + new MockNameResolver() + .addRecord("test-container-1", "::2") + .mockAnyLookup(), + new DockerImage("docker-registry.domain.tld:8080/dist/vespa")); this.flavors = flavors; + curator.setConnectionSpec("cfg1:1234,cfg2:1234,cfg3:1234"); populate(); } @@ -133,4 +137,4 @@ public class MockNodeRepository extends NodeRepository { provisioner.activate(transaction, application, hosts); transaction.commit(); } -}
\ No newline at end of file +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java index 37b2f54da4d..54e0c59ff00 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java @@ -13,7 +13,6 @@ import com.yahoo.vespa.config.SlimeUtils; import com.yahoo.vespa.hosted.provision.testutils.ContainerConfig; import org.junit.After; import org.junit.Before; -import org.junit.Ignore; import org.junit.Test; import java.io.File; @@ -40,9 +39,20 @@ public class RestApiTest { private final static String responsesPath = "src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/"; + private JDisc container; + + @Before + public void startContainer() { + container = JDisc.fromServicesXml(ContainerConfig.servicesXmlV2(0), Networking.disable); + } + + @After + public void stopContainer() { + container.close(); + } + /** This test gives examples of all the requests that can be made to nodes/v2 */ @Test - @Ignore /** TODO re-enable this and verify correctness */ public void test_requests() throws Exception { // GET assertFile(new Request("http://localhost:8080/nodes/v2/"), "root.json"); @@ -54,28 +64,28 @@ public class RestApiTest { assertFile(new Request("http://localhost:8080/nodes/v2/node/host2.yahoo.com"), "node2.json"); // GET with filters - assertFile(new Request("http://localhost:8080/nodes/v2/node/?recursive=true&hostname=host2.yahoo.com%20host1.yahoo.com"), "application2-nodes.json"); - assertFile(new Request("http://localhost:8080/nodes/v2/node/?recursive=true&clusterType=content"), "active-nodes.json"); + assertFile(new Request("http://localhost:8080/nodes/v2/node/?recursive=true&hostname=host6.yahoo.com%20host2.yahoo.com"), "application2-nodes.json"); + assertFile(new Request("http://localhost:8080/nodes/v2/node/?recursive=true&clusterType=content"), "content-nodes.json"); assertFile(new Request("http://localhost:8080/nodes/v2/node/?recursive=true&clusterId=id2"), "application2-nodes.json"); assertFile(new Request("http://localhost:8080/nodes/v2/node/?recursive=true&application=tenant2.application2.instance2"), "application2-nodes.json"); - assertFile(new Request("http://localhost:8080/nodes/v2/node/?recursive=true&parentHost=parent1.yahoo.com,parent.host.yahoo.com"), "parent-nodes.json"); + assertFile(new Request("http://localhost:8080/nodes/v2/node/?recursive=true&parentHost=dockerhost1.yahoo.com"), "child-nodes.json"); // POST restart command assertRestart(1, new Request("http://localhost:8080/nodes/v2/command/restart?hostname=host2.yahoo.com", new byte[0], Request.Method.POST)); assertRestart(2, new Request("http://localhost:8080/nodes/v2/command/restart?application=tenant2.application2.instance2", new byte[0], Request.Method.POST)); - assertRestart(4, new Request("http://localhost:8080/nodes/v2/command/restart", + assertRestart(9, new Request("http://localhost:8080/nodes/v2/command/restart", new byte[0], Request.Method.POST)); assertResponseContains(new Request("http://localhost:8080/nodes/v2/node/host2.yahoo.com"), "\"restartGeneration\":3"); // POST reboot command - assertReboot(5, new Request("http://localhost:8080/nodes/v2/command/reboot?state=failed%20active", + assertReboot(10, new Request("http://localhost:8080/nodes/v2/command/reboot?state=failed%20active", new byte[0], Request.Method.POST)); assertReboot(2, new Request("http://localhost:8080/nodes/v2/command/reboot?application=tenant2.application2.instance2", new byte[0], Request.Method.POST)); - assertReboot(10, new Request("http://localhost:8080/nodes/v2/command/reboot", + assertReboot(15, new Request("http://localhost:8080/nodes/v2/command/reboot", new byte[0], Request.Method.POST)); assertResponseContains(new Request("http://localhost:8080/nodes/v2/node/host2.yahoo.com"), "\"rebootGeneration\":4"); @@ -106,9 +116,9 @@ public class RestApiTest { assertFile(new Request("http://localhost:8080/nodes/v2/node/parent2.yahoo.com"), "parent2.json"); // DELETE a provisioned node - assertResponse(new Request("http://localhost:8080/nodes/v2/node/host11.yahoo.com", + assertResponse(new Request("http://localhost:8080/nodes/v2/node/host9.yahoo.com", new byte[0], Request.Method.DELETE), - "{\"message\":\"Removed host11.yahoo.com\"}"); + "{\"message\":\"Removed host9.yahoo.com\"}"); // PUT nodes ready assertResponse(new Request("http://localhost:8080/nodes/v2/state/dirty/host8.yahoo.com", @@ -125,15 +135,15 @@ public class RestApiTest { "{\"message\":\"Moved host8.yahoo.com to ready\"}"); // PUT a node in failed ... - assertResponse(new Request("http://localhost:8080/nodes/v2/state/failed/host3.yahoo.com", + assertResponse(new Request("http://localhost:8080/nodes/v2/state/failed/host2.yahoo.com", new byte[0], Request.Method.PUT), - "{\"message\":\"Moved host3.yahoo.com to failed\"}"); - assertResponseContains(new Request("http://localhost:8080/nodes/v2/node/host3.yahoo.com"), + "{\"message\":\"Moved host2.yahoo.com to failed\"}"); + assertResponseContains(new Request("http://localhost:8080/nodes/v2/node/host2.yahoo.com"), "\"state\":\"failed\""); // ... and put it back in active (after fixing). This is useful to restore data when multiple nodes fail. - assertResponse(new Request("http://localhost:8080/nodes/v2/state/active/host3.yahoo.com", + assertResponse(new Request("http://localhost:8080/nodes/v2/state/active/host2.yahoo.com", new byte[0], Request.Method.PUT), - "{\"message\":\"Moved host3.yahoo.com to active\"}"); + "{\"message\":\"Moved host2.yahoo.com to active\"}"); // PUT a node in parked ... assertResponse(new Request("http://localhost:8080/nodes/v2/state/parked/host8.yahoo.com", @@ -147,30 +157,29 @@ public class RestApiTest { "{\"message\":\"Removed host8.yahoo.com\"}"); // or, PUT a node in failed ... - assertResponse(new Request("http://localhost:8080/nodes/v2/state/failed/host6.yahoo.com", + assertResponse(new Request("http://localhost:8080/nodes/v2/state/failed/test-container-1", new byte[0], Request.Method.PUT), - "{\"message\":\"Moved host6.yahoo.com to failed\"}"); - assertResponseContains(new Request("http://localhost:8080/nodes/v2/node/host6.yahoo.com"), + "{\"message\":\"Moved test-container-1 to failed\"}"); + assertResponseContains(new Request("http://localhost:8080/nodes/v2/node/test-container-1"), "\"state\":\"failed\""); // ... and deallocate it such that it moves to dirty and is recycled - assertResponse(new Request("http://localhost:8080/nodes/v2/state/dirty/host6.yahoo.com", + assertResponse(new Request("http://localhost:8080/nodes/v2/state/dirty/test-container-1", new byte[0], Request.Method.PUT), - "{\"message\":\"Moved host6.yahoo.com to dirty\"}"); + "{\"message\":\"Moved test-container-1 to dirty\"}"); // ... and set it back to ready as if this was from the node-admin with the temporary state rest api - assertResponse(new Request("http://localhost:8080/nodes/v2/state/availablefornewallocations/host6.yahoo.com", + assertResponse(new Request("http://localhost:8080/nodes/v2/state/availablefornewallocations/test-container-1", new byte[0], Request.Method.PUT), - "{\"message\":\"Moved host6.yahoo.com to ready\"}"); + "{\"message\":\"Marked following nodes as available for new allocation: test-container-1\"}"); // Put a host in failed and make sure it's children are also failed - assertResponse(new Request("http://localhost:8080/nodes/v2/state/failed/parent1.yahoo.com", new byte[0], Request.Method.PUT), - "{\"message\":\"Moved host10.yahoo.com, host5.yahoo.com, parent1.yahoo.com to failed\"}"); + assertResponse(new Request("http://localhost:8080/nodes/v2/state/failed/dockerhost1.yahoo.com", new byte[0], Request.Method.PUT), + "{\"message\":\"Moved dockerhost1.yahoo.com, host4.yahoo.com to failed\"}"); assertResponse(new Request("http://localhost:8080/nodes/v2/state/failed"), "{\"nodes\":[" + - "{\"url\":\"http://localhost:8080/nodes/v2/node/parent1.yahoo.com\"}," + + "{\"url\":\"http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com\"}," + "{\"url\":\"http://localhost:8080/nodes/v2/node/host5.yahoo.com\"}," + - "{\"url\":\"http://localhost:8080/nodes/v2/node/host10.yahoo.com\"}]}"); - + "{\"url\":\"http://localhost:8080/nodes/v2/node/host4.yahoo.com\"}]}"); // Update (PATCH) a node (multiple fields can also be sent in one request body) assertResponse(new Request("http://localhost:8080/nodes/v2/node/host4.yahoo.com", @@ -514,13 +523,6 @@ public class RestApiTest { } } - private JDisc container; - @Before - public void startContainer() { - container = JDisc.fromServicesXml(ContainerConfig.servicesXmlV2(0), Networking.disable); } - @After - public void stopContainer() { container.close(); } - private String asDockerNodeJson(String hostname, String parentHostname, int additionalIpCount, String... ipAddress) { return "{\"hostname\":\"" + hostname + "\", \"parentHostname\":\"" + parentHostname + "\"," + createIpAddresses(ipAddress) + diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/active-nodes.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/active-nodes.json index d1df5b83f24..c67ba904f9a 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/active-nodes.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/active-nodes.json @@ -1,8 +1,13 @@ { "nodes": [ + @include(docker-node4.json), + @include(docker-node5.json), + @include(docker-node2.json), + @include(docker-node1.json), + @include(docker-node3.json), @include(node6.json), - @include(node3.json), @include(node2.json), - @include(node1.json) + @include(docker-container1.json), + @include(node4.json) ] -}
\ No newline at end of file +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/application2-nodes.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/application2-nodes.json index 1d4d97315cd..4581ecba73d 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/application2-nodes.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/application2-nodes.json @@ -1,6 +1,6 @@ { "nodes": [ - @include(node2.json), - @include(node1.json) + @include(node6.json), + @include(node2.json) ] -}
\ No newline at end of file +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/child-nodes.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/child-nodes.json new file mode 100644 index 00000000000..dae92aae091 --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/child-nodes.json @@ -0,0 +1,5 @@ +{ + "nodes": [ + @include(node4.json) + ] +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/content-nodes.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/content-nodes.json new file mode 100644 index 00000000000..47a2c012b17 --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/content-nodes.json @@ -0,0 +1,8 @@ +{ + "nodes": [ + @include(node6.json), + @include(node2.json), + @include(docker-container1.json), + @include(node4.json) + ] +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-container1.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-container1.json new file mode 100644 index 00000000000..7823ed0431d --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-container1.json @@ -0,0 +1,56 @@ +{ + "url": "http://localhost:8080/nodes/v2/node/test-container-1", + "id": "test-container-1", + "state": "active", + "type": "tenant", + "hostname": "test-container-1", + "parentHostname": "dockerhost3.yahoo.com", + "openStackId": "fake-test-container-1", + "flavor": "docker", + "canonicalFlavor": "docker", + "minDiskAvailableGb": 100.0, + "minMainMemoryAvailableGb": 0.5, + "description": "Flavor-name-is-docker", + "minCpuCores": 0.2, + "fastDisk": true, + "environment": "DOCKER_CONTAINER", + "owner": { + "tenant": "tenant3", + "application": "application3", + "instance": "instance3" + }, + "membership": { + "clustertype": "content", + "clusterid": "id3", + "group": "0", + "index": 1, + "retired": false + }, + "restartGeneration": 0, + "currentRestartGeneration": 0, + "wantedDockerImage": "docker-registry.domain.tld:8080/dist/vespa:6.42.0", + "wantedVespaVersion": "6.42.0", + "allowedToBeDown": false, + "rebootGeneration": 0, + "currentRebootGeneration": 0, + "failCount": 0, + "hardwareFailure": false, + "wantToRetire": false, + "wantToDeprovision": false, + "history": [ + { + "event": "reserved", + "at": 123, + "agent": "system" + }, + { + "event": "activated", + "at": 123, + "agent": "application" + } + ], + "ipAddresses": [ + "::2" + ], + "additionalIpAddresses": [] +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node1.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node1.json new file mode 100644 index 00000000000..a13dfae927f --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node1.json @@ -0,0 +1,70 @@ +{ + "url": "http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com", + "id": "dockerhost1.yahoo.com", + "state": "active", + "type": "host", + "hostname": "dockerhost1.yahoo.com", + "openStackId": "dockerhost1", + "flavor": "large", + "canonicalFlavor": "large", + "minDiskAvailableGb": 1600.0, + "minMainMemoryAvailableGb": 32.0, + "description": "Flavor-name-is-large", + "minCpuCores": 4.0, + "fastDisk": true, + "environment": "BARE_METAL", + "owner": { + "tenant": "zoneapp", + "application": "zoneapp", + "instance": "zoneapp" + }, + "membership": { + "clustertype": "container", + "clusterid": "node-admin", + "group": "0", + "index": 0, + "retired": false + }, + "restartGeneration": 0, + "currentRestartGeneration": 0, + "wantedDockerImage": "docker-registry.domain.tld:8080/dist/vespa:6.42.0", + "wantedVespaVersion": "6.42.0", + "allowedToBeDown": false, + "rebootGeneration": 1, + "currentRebootGeneration": 0, + "failCount": 0, + "hardwareFailure": false, + "wantToRetire": false, + "wantToDeprovision": false, + "history": [ + { + "event": "provisioned", + "at": 123, + "agent": "system" + }, + { + "event": "readied", + "at": 123, + "agent": "system" + }, + { + "event": "reserved", + "at": 123, + "agent": "application" + }, + { + "event": "activated", + "at": 123, + "agent": "application" + } + ], + "ipAddresses": [ + "::1", + "127.0.0.1" + ], + "additionalIpAddresses": [ + "::2", + "::3", + "::4" + ] +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node2.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node2.json new file mode 100644 index 00000000000..f7a1d6ab9a9 --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node2.json @@ -0,0 +1,70 @@ +{ + "url": "http://localhost:8080/nodes/v2/node/dockerhost2.yahoo.com", + "id": "dockerhost2.yahoo.com", + "state": "active", + "type": "host", + "hostname": "dockerhost2.yahoo.com", + "openStackId": "dockerhost2", + "flavor": "large", + "canonicalFlavor": "large", + "minDiskAvailableGb": 1600.0, + "minMainMemoryAvailableGb": 32.0, + "description": "Flavor-name-is-large", + "minCpuCores": 4.0, + "fastDisk": true, + "environment": "BARE_METAL", + "owner": { + "tenant": "zoneapp", + "application": "zoneapp", + "instance": "zoneapp" + }, + "membership": { + "clustertype": "container", + "clusterid": "node-admin", + "group": "0", + "index": 1, + "retired": false + }, + "restartGeneration": 0, + "currentRestartGeneration": 0, + "wantedDockerImage": "docker-registry.domain.tld:8080/dist/vespa:6.42.0", + "wantedVespaVersion": "6.42.0", + "allowedToBeDown": false, + "rebootGeneration": 1, + "currentRebootGeneration": 0, + "failCount": 0, + "hardwareFailure": false, + "wantToRetire": false, + "wantToDeprovision": false, + "history": [ + { + "event": "provisioned", + "at": 123, + "agent": "system" + }, + { + "event": "readied", + "at": 123, + "agent": "system" + }, + { + "event": "reserved", + "at": 123, + "agent": "application" + }, + { + "event": "activated", + "at": 123, + "agent": "application" + } + ], + "ipAddresses": [ + "::1", + "127.0.0.1" + ], + "additionalIpAddresses": [ + "::2", + "::3", + "::4" + ] +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node3.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node3.json new file mode 100644 index 00000000000..f877d33672f --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node3.json @@ -0,0 +1,70 @@ +{ + "url": "http://localhost:8080/nodes/v2/node/dockerhost3.yahoo.com", + "id": "dockerhost3.yahoo.com", + "state": "active", + "type": "host", + "hostname": "dockerhost3.yahoo.com", + "openStackId": "dockerhost3", + "flavor": "large", + "canonicalFlavor": "large", + "minDiskAvailableGb": 1600.0, + "minMainMemoryAvailableGb": 32.0, + "description": "Flavor-name-is-large", + "minCpuCores": 4.0, + "fastDisk": true, + "environment": "BARE_METAL", + "owner": { + "tenant": "zoneapp", + "application": "zoneapp", + "instance": "zoneapp" + }, + "membership": { + "clustertype": "container", + "clusterid": "node-admin", + "group": "0", + "index": 2, + "retired": false + }, + "restartGeneration": 0, + "currentRestartGeneration": 0, + "wantedDockerImage": "docker-registry.domain.tld:8080/dist/vespa:6.42.0", + "wantedVespaVersion": "6.42.0", + "allowedToBeDown": false, + "rebootGeneration": 1, + "currentRebootGeneration": 0, + "failCount": 0, + "hardwareFailure": false, + "wantToRetire": false, + "wantToDeprovision": false, + "history": [ + { + "event": "provisioned", + "at": 123, + "agent": "system" + }, + { + "event": "readied", + "at": 123, + "agent": "system" + }, + { + "event": "reserved", + "at": 123, + "agent": "application" + }, + { + "event": "activated", + "at": 123, + "agent": "application" + } + ], + "ipAddresses": [ + "::1", + "127.0.0.1" + ], + "additionalIpAddresses": [ + "::2", + "::3", + "::4" + ] +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node4.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node4.json new file mode 100644 index 00000000000..913cf9852aa --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node4.json @@ -0,0 +1,70 @@ +{ + "url": "http://localhost:8080/nodes/v2/node/dockerhost4.yahoo.com", + "id": "dockerhost4.yahoo.com", + "state": "active", + "type": "host", + "hostname": "dockerhost4.yahoo.com", + "openStackId": "dockerhost4", + "flavor": "large", + "canonicalFlavor": "large", + "minDiskAvailableGb": 1600.0, + "minMainMemoryAvailableGb": 32.0, + "description": "Flavor-name-is-large", + "minCpuCores": 4.0, + "fastDisk": true, + "environment": "BARE_METAL", + "owner": { + "tenant": "zoneapp", + "application": "zoneapp", + "instance": "zoneapp" + }, + "membership": { + "clustertype": "container", + "clusterid": "node-admin", + "group": "0", + "index": 3, + "retired": false + }, + "restartGeneration": 0, + "currentRestartGeneration": 0, + "wantedDockerImage": "docker-registry.domain.tld:8080/dist/vespa:6.42.0", + "wantedVespaVersion": "6.42.0", + "allowedToBeDown": false, + "rebootGeneration": 1, + "currentRebootGeneration": 0, + "failCount": 0, + "hardwareFailure": false, + "wantToRetire": false, + "wantToDeprovision": false, + "history": [ + { + "event": "provisioned", + "at": 123, + "agent": "system" + }, + { + "event": "readied", + "at": 123, + "agent": "system" + }, + { + "event": "reserved", + "at": 123, + "agent": "application" + }, + { + "event": "activated", + "at": 123, + "agent": "application" + } + ], + "ipAddresses": [ + "::1", + "127.0.0.1" + ], + "additionalIpAddresses": [ + "::2", + "::3", + "::4" + ] +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node5.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node5.json new file mode 100644 index 00000000000..685b0a52b15 --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node5.json @@ -0,0 +1,70 @@ +{ + "url": "http://localhost:8080/nodes/v2/node/dockerhost5.yahoo.com", + "id": "dockerhost5.yahoo.com", + "state": "active", + "type": "host", + "hostname": "dockerhost5.yahoo.com", + "openStackId": "dockerhost5", + "flavor": "large", + "canonicalFlavor": "large", + "minDiskAvailableGb": 1600.0, + "minMainMemoryAvailableGb": 32.0, + "description": "Flavor-name-is-large", + "minCpuCores": 4.0, + "fastDisk": true, + "environment": "BARE_METAL", + "owner": { + "tenant": "zoneapp", + "application": "zoneapp", + "instance": "zoneapp" + }, + "membership": { + "clustertype": "container", + "clusterid": "node-admin", + "group": "0", + "index": 4, + "retired": false + }, + "restartGeneration": 0, + "currentRestartGeneration": 0, + "wantedDockerImage": "docker-registry.domain.tld:8080/dist/vespa:6.42.0", + "wantedVespaVersion": "6.42.0", + "allowedToBeDown": false, + "rebootGeneration": 1, + "currentRebootGeneration": 0, + "failCount": 0, + "hardwareFailure": false, + "wantToRetire": false, + "wantToDeprovision": false, + "history": [ + { + "event": "provisioned", + "at": 123, + "agent": "system" + }, + { + "event": "readied", + "at": 123, + "agent": "system" + }, + { + "event": "reserved", + "at": 123, + "agent": "application" + }, + { + "event": "activated", + "at": 123, + "agent": "application" + } + ], + "ipAddresses": [ + "::1", + "127.0.0.1" + ], + "additionalIpAddresses": [ + "::2", + "::3", + "::4" + ] +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/maintenance.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/maintenance.json index fea4fb8d4d2..c09829e7f85 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/maintenance.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/maintenance.json @@ -28,6 +28,9 @@ "name":"OperatorChangeApplicationMaintainer" }, { + "name":"ProvisionedExpirer" + }, + { "name":"RetiredEarlyExpirer" }, { @@ -43,4 +46,4 @@ "inactive":[ "NodeFailer" ] -}
\ No newline at end of file +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node4-after-changes.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node4-after-changes.json index f311c240b1d..cb250e2033b 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node4-after-changes.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node4-after-changes.json @@ -1,7 +1,7 @@ { "url": "http://localhost:8080/nodes/v2/node/host4.yahoo.com", "id": "host4.yahoo.com", - "state": "reserved", + "state": "failed", "type": "tenant", "hostname": "host4.yahoo.com", "parentHostname": "parent.yahoo.com", @@ -12,36 +12,36 @@ "minMainMemoryAvailableGb": 12.0, "description": "Flavor-name-is-medium-disk", "minCpuCores": 6.0, - "fastDisk":true, + "fastDisk": true, "environment": "BARE_METAL", "owner": { - "tenant": "tenant1", - "application": "application1", - "instance": "instance1" + "tenant": "tenant3", + "application": "application3", + "instance": "instance3" }, "membership": { - "clustertype": "container", - "clusterid": "id1", + "clustertype": "content", + "clusterid": "id3", "group": "0", - "index": 1, + "index": 0, "retired": false }, - "restartGeneration": 0, + "restartGeneration": 1, "currentRestartGeneration": 1, "wantedDockerImage": "docker-registry.domain.tld:8080/dist/vespa:6.42.0", "wantedVespaVersion": "6.42.0", "allowedToBeDown": false, - "rebootGeneration": 2, + "rebootGeneration": 3, "currentRebootGeneration": 1, "vespaVersion": "6.43.0", "currentDockerImage": "docker-registry.domain.tld:8080/dist/vespa:6.43.0", "hostedVersion": "6.43.0", "convergedStateVersion": "6.43.0", - "failCount": 0, + "failCount": 1, "hardwareFailure": true, "hardwareFailureDescription": "memory_mcelog", - "wantToRetire" : true, - "wantToDeprovision" : true, + "wantToRetire": true, + "wantToDeprovision": true, "history": [ { "event": "provisioned", @@ -51,7 +51,7 @@ { "event": "readied", "at": 123, - "agent": "system" + "agent": "system" }, { "event": "reserved", @@ -59,11 +59,24 @@ "agent": "application" }, { + "event": "activated", + "at": 123, + "agent": "application" + }, + { + "event": "failed", + "at": 123, + "agent": "operator" + }, + { "event": "rebooted", "at": 123, "agent": "system" } ], - "ipAddresses":["127.0.0.1", "::1"], - "additionalIpAddresses":[] + "ipAddresses": [ + "127.0.0.1", + "::1" + ], + "additionalIpAddresses": [] } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/nodes-recursive.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/nodes-recursive.json index a9feed81674..475b914989b 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/nodes-recursive.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/nodes-recursive.json @@ -1,14 +1,19 @@ { "nodes": [ @include(node7.json), - @include(parent1.json), + @include(node3.json), @include(node10.json), - @include(node4.json), + @include(node1.json), + @include(docker-node4.json), @include(node6.json), - @include(node3.json), + @include(docker-node5.json), + @include(docker-node2.json), @include(node2.json), - @include(node1.json), + @include(docker-node1.json), + @include(docker-node3.json), + @include(docker-container1.json), + @include(node4.json), @include(node55.json), @include(node5.json) ] -}
\ No newline at end of file +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/nodes.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/nodes.json index 67b65259f8a..3bfaa95d5ee 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/nodes.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/nodes.json @@ -4,31 +4,46 @@ "url": "http://localhost:8080/nodes/v2/node/host7.yahoo.com" }, { - "url":"http://localhost:8080/nodes/v2/node/parent1.yahoo.com" + "url": "http://localhost:8080/nodes/v2/node/host3.yahoo.com" }, { - "url":"http://localhost:8080/nodes/v2/node/host10.yahoo.com" + "url": "http://localhost:8080/nodes/v2/node/host10.yahoo.com" }, { - "url":"http://localhost:8080/nodes/v2/node/host4.yahoo.com" + "url": "http://localhost:8080/nodes/v2/node/host1.yahoo.com" }, { - "url":"http://localhost:8080/nodes/v2/node/host6.yahoo.com" + "url": "http://localhost:8080/nodes/v2/node/dockerhost4.yahoo.com" }, { - "url":"http://localhost:8080/nodes/v2/node/host3.yahoo.com" + "url": "http://localhost:8080/nodes/v2/node/host6.yahoo.com" }, { - "url":"http://localhost:8080/nodes/v2/node/host2.yahoo.com" + "url": "http://localhost:8080/nodes/v2/node/dockerhost5.yahoo.com" }, { - "url":"http://localhost:8080/nodes/v2/node/host1.yahoo.com" + "url": "http://localhost:8080/nodes/v2/node/dockerhost2.yahoo.com" }, { - "url":"http://localhost:8080/nodes/v2/node/host55.yahoo.com" + "url": "http://localhost:8080/nodes/v2/node/host2.yahoo.com" }, { - "url":"http://localhost:8080/nodes/v2/node/host5.yahoo.com" + "url": "http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com" + }, + { + "url": "http://localhost:8080/nodes/v2/node/dockerhost3.yahoo.com" + }, + { + "url": "http://localhost:8080/nodes/v2/node/test-container-1" + }, + { + "url": "http://localhost:8080/nodes/v2/node/host4.yahoo.com" + }, + { + "url": "http://localhost:8080/nodes/v2/node/host55.yahoo.com" + }, + { + "url": "http://localhost:8080/nodes/v2/node/host5.yahoo.com" } ] -}
\ No newline at end of file +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/parent-nodes.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/parent-nodes.json deleted file mode 100644 index 81ca0465c4b..00000000000 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/parent-nodes.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "nodes": [ - @include(node10.json), - @include(node5.json) - ] -}
\ No newline at end of file diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/states-recursive.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/states-recursive.json index 4ee1d5ed9b9..183f81ee3e1 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/states-recursive.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/states-recursive.json @@ -9,23 +9,28 @@ "ready": { "url": "http://localhost:8080/nodes/v2/state/ready", "nodes": [ - @include(parent1.json) + @include(node3.json) ] }, "reserved": { "url": "http://localhost:8080/nodes/v2/state/reserved", "nodes": [ @include(node10.json), - @include(node4.json) + @include(node1.json) ] }, "active": { "url": "http://localhost:8080/nodes/v2/state/active", "nodes": [ + @include(docker-node4.json), + @include(docker-node5.json), + @include(docker-node2.json), + @include(docker-node1.json), + @include(docker-node3.json), @include(node6.json), - @include(node3.json), @include(node2.json), - @include(node1.json) + @include(docker-container1.json), + @include(node4.json) ] }, "inactive": { diff --git a/standalone-container/src/main/scala/com/yahoo/container/standalone/CloudConfigYinstVariables.scala b/standalone-container/src/main/scala/com/yahoo/container/standalone/CloudConfigYinstVariables.scala index fe1af676b54..41e6b66b986 100644 --- a/standalone-container/src/main/scala/com/yahoo/container/standalone/CloudConfigYinstVariables.scala +++ b/standalone-container/src/main/scala/com/yahoo/container/standalone/CloudConfigYinstVariables.scala @@ -10,8 +10,8 @@ import scala.language.implicitConversions import scala.util.Try /** - * @author tonytv - */ + * @author Tony Vaagenes + */ class CloudConfigYinstVariables extends CloudConfigOptions { import CloudConfigYinstVariables._ @@ -39,6 +39,7 @@ class CloudConfigYinstVariables extends CloudConfigOptions { override val dockerRegistry = optionalYinstVar[java.lang.String]("docker_registry") override val dockerVespaBaseImage = optionalYinstVar[java.lang.String]("docker_vespa_base_image") override val loadBalancerAddress = optionalYinstVar[java.lang.String]("load_balancer_address") + override val disableFiledistributor = optionalYinstVar[java.lang.Boolean]("disable_filedistributor") } object CloudConfigYinstVariables { diff --git a/vespajlib/src/main/java/com/yahoo/io/IOUtils.java b/vespajlib/src/main/java/com/yahoo/io/IOUtils.java index 2572842b213..febe02cb33e 100644 --- a/vespajlib/src/main/java/com/yahoo/io/IOUtils.java +++ b/vespajlib/src/main/java/com/yahoo/io/IOUtils.java @@ -3,6 +3,7 @@ package com.yahoo.io; import java.io.*; +import java.nio.channels.FileChannel; import java.nio.file.Files; import java.nio.file.NoSuchFileException; import java.util.List; @@ -155,33 +156,22 @@ public abstract class IOUtils { /** * Copies a file to another file. * If the out file exists it will be overwritten. - * NOTE: Not an optimal implementation currently. * * @throws IOException if copying fails */ public static void copy(String inFile, String outFile) throws IOException { - BufferedReader reader=null; - BufferedWriter writer=null; - - try { - reader = createReader(inFile); - writer = createWriter(outFile, false); - int c; - while (-1 != (c = reader.read()) ) - writer.write(c); - } finally { - closeReader(reader); - closeWriter(writer); - } + copy(new File(inFile), new File(outFile)); } /** * Copies a file to another file. * If the out file exists it will be overwritten. - * NOTE: Not an optimal implementation currently. */ public static void copy(File inFile, File outFile) throws IOException { - copy(inFile.toString(),outFile.toString()); + try (FileChannel sourceChannel = new FileInputStream(inFile).getChannel(); + FileChannel destChannel = new FileOutputStream(outFile).getChannel()) { + destChannel.transferFrom(sourceChannel, 0, sourceChannel.size()); + } } /** @@ -275,8 +265,8 @@ public abstract class IOUtils { } /** - * Returns the number of line in a file. - * If the files does not exists, 0 is returned + * Returns the number of lines in a file. + * If the file does not exists, 0 is returned */ public static int countLines(String file) { BufferedReader reader = null; @@ -292,7 +282,6 @@ public abstract class IOUtils { } finally { closeReader(reader); } - } /** diff --git a/vespajlib/src/test/java/com/yahoo/io/IOUtilsTestCase.java b/vespajlib/src/test/java/com/yahoo/io/IOUtilsTestCase.java index 3a8b0dde1c1..8955bd9ea05 100644 --- a/vespajlib/src/test/java/com/yahoo/io/IOUtilsTestCase.java +++ b/vespajlib/src/test/java/com/yahoo/io/IOUtilsTestCase.java @@ -1,15 +1,23 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.io; +import org.junit.Test; + import java.io.*; import java.util.Arrays; import java.util.List; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + /** * @author bratseth */ -public class IOUtilsTestCase extends junit.framework.TestCase { +public class IOUtilsTestCase { + @Test public void testCloseNUllDoesNotFail() { IOUtils.closeWriter(null); IOUtils.closeReader(null); @@ -17,12 +25,14 @@ public class IOUtilsTestCase extends junit.framework.TestCase { IOUtils.closeOutputStream(null); } + @Test public void testFileWriter() throws IOException { IOUtils.writeFile("temp1.txt", "hello",false); assertEquals("hello", IOUtils.readFile(new File("temp1.txt"))); new File("temp1.txt").delete(); } + @Test public void testFileWriterWithoutEncoding() throws IOException { BufferedWriter writer=null; try { @@ -36,6 +46,7 @@ public class IOUtilsTestCase extends junit.framework.TestCase { new File("temp2.txt").delete(); } + @Test public void testFileWriterWithoutEncodingFromFileName() throws IOException { BufferedWriter writer=null; try { @@ -49,12 +60,14 @@ public class IOUtilsTestCase extends junit.framework.TestCase { new File("temp3.txt").delete(); } + @Test public void testFileCounting() throws IOException { IOUtils.writeFile("temp4.txt","hello\nworld",false); assertEquals(2,IOUtils.countLines("temp4.txt")); new File("temp4.txt").delete(); } + @Test public void testFileCopy() throws IOException { IOUtils.writeFile("temp5.txt","hello",false); IOUtils.copy(new File("temp5.txt"), new File("temp5copy.txt")); @@ -63,6 +76,7 @@ public class IOUtilsTestCase extends junit.framework.TestCase { new File("temp5copy.txt").delete(); } + @Test public void testFileCopyWithLineCap() throws IOException { IOUtils.writeFile("temp6.txt","hello\nyou\nworld",false); IOUtils.copy("temp6.txt","temp6copy.txt",2); @@ -71,6 +85,7 @@ public class IOUtilsTestCase extends junit.framework.TestCase { new File("temp6copy.txt").delete(); } + @Test public void testGetLines() throws IOException { IOUtils.writeFile("temp7.txt","hello\nworld",false); List<String> lines=IOUtils.getLines("temp7.txt"); @@ -80,6 +95,7 @@ public class IOUtilsTestCase extends junit.framework.TestCase { new File("temp7.txt").delete(); } + @Test public void testFileWriterAppend() throws IOException { boolean append=true; IOUtils.writeFile("temp8.txt", "hello",!append); @@ -95,6 +111,7 @@ public class IOUtilsTestCase extends junit.framework.TestCase { new File("temp8.txt").delete(); } + @Test public void testCloseAllReaders() throws IOException { StringReader reader1=new StringReader("hello"); StringReader reader2=new StringReader("world"); @@ -115,6 +132,7 @@ public class IOUtilsTestCase extends junit.framework.TestCase { } } + @Test public void testDirCopying() throws IOException { IOUtils.writeFile("temp1/temp1.txt","hello",false); IOUtils.writeFile("temp1/temp2.txt","world",false); @@ -127,6 +145,7 @@ public class IOUtilsTestCase extends junit.framework.TestCase { assertTrue(!new File("temp2").exists()); } + @Test public void testDirCopyingWithFilter() throws IOException { IOUtils.writeFile("temp1/temp1.txt","hello",false); IOUtils.writeFile("temp1/temp2.txt","world",false); |